cgse 0.13.1__tar.gz → 0.14.1__tar.gz

{cgse-0.13.1 → cgse-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse
-Version: 0.13.1
+Version: 0.14.1
 Summary: Generic Common-EGSE: Commanding and monitoring lab equipment
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

cgse-0.14.1/docs/libs/cgse-core/notifyhub.md

@@ -0,0 +1,168 @@
+# The Notification Hub
+
+## Overview
+
+The Notification Hub serves as a centralized event distribution system for the core services and control servers. 
+Instead of having services directly subscribe to each other, all services publish their events to the hub, which 
+then redistributes them to interested subscribers via ZeroMQ PUB-SUB sockets.
+
+## Benefits
+
+- **Decoupling**: Services don't need to know about each other directly
+- **Scalability**: Easy to add new subscribers without modifying existing services
+- **Reliability**: Single point of event distribution with consistent delivery
+- **Simplicity**: One subscription endpoint for all system events
+
+# Socket Endpoints
+
+The Notification Hub exposes three ZeroMQ sockets:
+
+| Purpose            |    Pattern     |  Your Socket  | Endpoint                    |
+|--------------------|:--------------:|:-------------:|-----------------------------|
+| Event Publishing   |   PUSH-PULL    |     PUSH      | tcp://notification-hub:4245 |
+| Event Subscription |    PUB-SUB     |      SUB      | tcp://notification-hub:4246 |
+| Health Checks      | ROUTER-DEALER  |    DEALER     | tcp://notification-hub:4247 |
+
+Note: port numbers can be changed in the local settings YAML file.
+
+## Events
+
+```python
+from egse.notifyhub.event import NotificationEvent
+
+event = NotificationEvent(
+    event_type="new_setup",
+    source_service="cm_cs",
+    data={"setup_id": "0001234"},
+)
+```
+A `NotificationEvent` also has a `timestamp` and a `correlation_id` 
+associated. These fields are automatically filled on creation of the event 
+and must not be provided. An event can be converted into a plain dictionary 
+with the `.as_dict()` method.
+
+```
+>>> print(event.as_dict())
+{
+  'event_type': 'new_setup', 
+  'source_service': 'cm_cs', 
+  'data': {'setup_id': '0001234'}, 
+  'timestamp': 1756709627.25942, 
+  'correlation_id': '23b44f16-34f9-498a-85f7-5c57658ea363'
+}
+```
+
+## Publishing Events
+
+Any core service, control server, or script can publish events to the notification hub using the EventPublisher class. For convenience, EventPublisher supports usage as a context manager, making it easy to send single events.
+
+=== "Synchronous"
+
+    ```python
+    from egse.notifyhub.services import EventPublisher
+    
+    with EventPublisher() as publisher:
+        publisher.publish(event)
+    ```
+
+=== "Asynchronous"
+
+    ```python
+    from egse.notifyhub.services import AsyncEventPublisher
+    
+    async with AsyncEventPublisher() as publisher:
+        await publisher.publish(event)
+    ```
+
+
+## Subscribing to Events
+
+Listen to events from the notification hub with the EventSubscriber class. 
+The usage for this class is quite different for synchronous and asynchronous 
+contexts. 
+
+=== "Synchronous"
+
+    In a synchronous context, you will need to add code to poll the socket 
+    and handle the event inside your own event loop. The subscriber socket 
+    can be retrieved with the `subscriber.socket` property, you can then add 
+    the socket to a Poller object.
+
+    ```python
+    from egse.notifyhub.services import EventSubscriber
+    
+    def load_setup(event_data: dict):
+        ...
+
+    subscriber = EventSubscriber(["new_setup"])
+    subscriber.register_handler("new_setup", load_setup)
+    subscriber.connect()
+    
+    while True:
+
+        ...
+
+        if subscriber.poll():
+            subscriber.handle_event()
+        
+    subscriber.disconnect()
+    ```
+
+=== "Asynchronous"
+
+    ```python
+    from egse.notifyhub.services import AsyncEventSubscriber
+
+    async def load_setup(event_data: dict):
+        ...
+
+    subscriber = AsyncEventSubscriber(["new_setup"])
+    subscriber.register_handler("new_setup", load_setup)
+    await subscriber.connect()
+
+    event_listener = asyncio.create_task(subscriber.start_listening())
+
+    ...
+
+    subscriber.disconnect()
+    await event_listener  # add a wait_for with a timeout if needed
+    ```
+
+## Health Checks
+
+The Notification Hub provides a health check interface to monitor the status and availability of the hub. This allows services and scripts to verify connectivity and basic hub functionality.
+To perform a health check, use the NotificationHubClient class. The client 
+connects to the hub's ROUTER-DEALER socket and sends a health check request. 
+If the hub is available, it responds True. In case of an error or when the 
+hub is not available, False is returned.
+
+=== "Synchronous"
+
+    ```python
+    from egse.notifyhub.client import NotificationHubClient
+    
+    with NotificationHubClient() as client:
+        if not client.health_check():
+            ... # notification hub not available
+    ```
+
+=== "Asynchronous"
+
+    ```python
+    from egse.notifyhub.client import AsyncNotificationHubClient
+    
+    with AsyncNotificationHubClient() as client:
+        if not await client.health_check():
+            ... # notification hub not available
+    ```
+
+The health check has a default timeout of 5 seconds. If this is too long for 
+your needs, provide a `request_timeout` argument as a float in seconds to 
+the `NotificationHubClient` call. 
+
+
+## Monitoring
+
+Every 30s, the hub provides basic connection and event statistics to the log.
+
+TODO: this should also go on the PUB channel as a StatsEvent.

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.13.1"
+version = "0.14.1"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/cgse_common/cgse.py

@@ -184,6 +184,7 @@ def main(ctx: typer.Context, verbose: bool = False):
         # print("Try 'cgse --help' for a list of commands.")
         typer.echo(ctx.get_help())
 
+
 build_app()
 
 

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/calibration.py

@@ -5,8 +5,8 @@ This module provides functions to calibrate sensor values.
 from __future__ import annotations
 
 import numpy as np
-
-from egse.setup import navdict, Setup, SetupError
+from navdict import navdict
+from egse.setup import Setup, SetupError
 
 
 def apply_gain_offset(counts: float, gain: float, offset: float) -> float:

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/config.py

@@ -3,27 +3,15 @@ This module provides convenience functions to properly configure the CGSE
 and to find paths and resources.
 """
 
-from __future__ import annotations
-
-import errno
 import fnmatch
-import logging
 import os
-from functools import lru_cache
-from os.path import exists
-from os.path import join
 from pathlib import Path
 from pathlib import PurePath
-from typing import List
-from typing import Optional
+from typing import Generator
 from typing import Tuple
 from typing import Union
-from typing import Generator
 
-import git
-from egse.decorators import deprecate
-
-_logger = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def find_first_occurrence_of_dir(pattern: str, root: Path | str) -> Path | None:
@@ -252,18 +240,6 @@ def find_root(
     return Path(default) if default is not None else None
 
 
-def set_logger_levels(logger_levels: List[Tuple] = None):
-    """
-    Set the logging level for the given loggers.
-
-    """
-    logger_levels = logger_levels or []
-
-    for name, level in logger_levels:
-        a_logger = logging.getLogger(name)
-        a_logger.setLevel(level)
-
-
 class WorkingDirectory:
     """
     WorkingDirectory is a context manager to temporarily change the working directory while
@@ -301,7 +277,7 @@ class WorkingDirectory:
         try:
             os.chdir(self._current_dir)
         except OSError as exc:
-            _logger.warning(f"Change back to previous directory failed: {exc}")
+            logger.warning(f"Change back to previous directory failed: {exc}")
 
     @property
     def path(self):

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/counter.py

@@ -2,8 +2,6 @@
 This module manages files that have a counter in their filename.
 """
 
-from __future__ import annotations
-
 __all__ = [
     "counter_exists",
     "counter_filename",
@@ -11,12 +9,10 @@ __all__ = [
     "new_counter",
 ]
 
-import logging
 from pathlib import Path
 
 from egse.config import find_files
-
-_LOGGER = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def counter_filename(location: Path, filename: Path | str) -> Path:
@@ -133,7 +129,7 @@ def get_next_counter(filename: Path) -> int:
     return counter
 
 
-def determine_counter_from_dir_list(location: str, pattern: str, index: int = -1) -> int:
+def determine_counter_from_dir_list(location: Path | str, pattern: str, index: int = -1) -> int:
     """
     Determine counter for a new file at the given location and with the given pattern.
     The next counter is determined from the sorted list of files that match the given pattern.
@@ -170,9 +166,9 @@ def determine_counter_from_dir_list(location: str, pattern: str, index: int = -1
         #  <anything here>_counter>.<extension>
 
         counter = int(parts[index].split(".")[0]) + 1
-        _LOGGER.debug(f"{counter = }")
+        logger.debug(f"{counter = }")
         return counter
 
     except ValueError:
-        _LOGGER.warning("ValueError", exc_info=True)
+        logger.warning("ValueError", exc_info=True)
         return 1

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/decorators.py

@@ -17,8 +17,7 @@ import rich
 
 from egse.settings import Settings
 from egse.system import get_caller_info
-
-_LOGGER = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def static_vars(**kwargs):
@@ -161,14 +160,14 @@ def average_time(*, name: str = "average_time", level: int = logging.INFO, preci
         def _report_average_time():
             if func._call_count:
                 average_time = func._run_time / func._call_count
-                _LOGGER.log(
+                logger.log(
                     level,
                     f"{name}: "
                     f"average runtime of {func.__name__!r} is {average_time:.{precision}f}s, "
                     f"#calls = {func._call_count}.",
                 )
             else:
-                _LOGGER.log(level, f"{name}: function {func.__name__!r} was never called.")
+                logger.log(level, f"{name}: function {func.__name__!r} was never called.")
 
             return func._run_time, func._call_count
 
@@ -210,7 +209,7 @@ def timer(*, name: str = "timer", level: int = logging.INFO, precision: int = 4)
             value = func(*args, **kwargs)
             end_time = time.perf_counter()
             run_time = end_time - start_time
-            _LOGGER.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
+            logger.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
             return value
 
         return wrapper_timer
@@ -235,7 +234,7 @@ def async_timer(*, name: str = "timer", level: int = logging.INFO, precision: in
             value = await func(*args, **kwargs)
             end_time = time.perf_counter()
             run_time = end_time - start_time
-            _LOGGER.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
+            logger.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
             return value
 
         return wrapper_timer
@@ -307,9 +306,9 @@ def debug(func):
             args_repr = [repr(a) for a in args]
             kwargs_repr = [f"{k}={v!r}" for k, v in kwargs.items()]
             signature = ", ".join(args_repr + kwargs_repr)
-            _LOGGER.debug(f"Calling {func.__name__}({signature})")
+            logger.debug(f"Calling {func.__name__}({signature})")
             value = func(*args, **kwargs)
-            _LOGGER.debug(f"{func.__name__!r} returned {value!r}")
+            logger.debug(f"{func.__name__!r} returned {value!r}")
         else:
             value = func(*args, **kwargs)
         return value
@@ -523,7 +522,7 @@ def to_be_implemented(func):
 
     @functools.wraps(func)
     def wrapper_tbi(*args, **kwargs):
-        _LOGGER.warning(f"The function/method {func.__name__} is not yet implemented.")
+        logger.warning(f"The function/method {func.__name__} is not yet implemented.")
         return func(*args, **kwargs)
 
     return wrapper_tbi
@@ -736,12 +735,12 @@ def retry_with_exponential_backoff(
             while attempt < max_attempts:
                 try:
                     response = func(*args, **kwargs)  # Attempt to call the function
-                    _LOGGER.info(f"{func.__name__} successfully executed.")
+                    logger.info(f"{func.__name__} successfully executed.")
                     return response
                 except tuple(exceptions) as exc:
                     last_exception = exc
                     attempt += 1
-                    _LOGGER.info(
+                    logger.info(
                         f"Retry {attempt}: {func.__name__} will be executing again in {wait_time * backoff_factor}s. "
                         f"Received a {last_exception!r}."
                     )
@@ -805,7 +804,7 @@ def retry(times: int = 3, wait: float = 10.0, exceptions: List = None) -> Callab
                 except tuple(exceptions) as exc:
                     previous_exception = exc
                 if n < times:
-                    _LOGGER.info(
+                    logger.info(
                         f"Retry {n + 1}: {func.__name__} will be executing again in {wait}s. "
                         f"Received a {previous_exception!r}."
                     )

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/device.py

@@ -3,7 +3,6 @@ This module defines the generic interfaces to connect devices.
 """
 
 from enum import IntEnum
-from typing import List
 
 from egse.decorators import dynamic_interface
 from egse.exceptions import Error

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/dicts.py

@@ -9,14 +9,14 @@ __all__ = [
 ]
 import logging
 
-from egse.system import capture_rich_output
-from egse.system import flatten_dict
 from rich.table import Table
 
-LOGGER = logging.getLogger(__name__)
+from egse.log import logger
+from egse.system import capture_rich_output
+from egse.system import flatten_dict
 
 
-def log_differences(dict_1, dict_2):
+def log_differences(dict_1, dict_2, log_level: int = logging.INFO):
     """
     Takes two flattened dictionaries and compares them. This function only compares those
     keys that are common to both dictionaries. The key-value pairs that are unique to one
@@ -49,13 +49,14 @@ def log_differences(dict_1, dict_2):
         for name in sorted(mismatched):
             table.add_row(name, str(dict_1[name]), str(dict_2[name]))
 
-        LOGGER.info(capture_rich_output(table))
-        # rich.print(table)
+        logger.log(log_level, capture_rich_output(table))
     else:
-        LOGGER.info(f"No differences between the two flattened dictionaries, {len(all_keys)} values compared.")
+        logger.log(
+            log_level, f"No differences between the two flattened dictionaries, {len(all_keys)} values compared."
+        )
 
 
-def log_key_differences(dict_1, dict_2):
+def log_key_differences(dict_1, dict_2, log_level: int = logging.INFO):
     """
     Takes two dictionaries and compares the top-level keys. The differences are logged in a Rich Table at level=INFO.
     Keys that are present on both dictionaries are not logged.
@@ -81,7 +82,7 @@ def log_key_differences(dict_1, dict_2):
     not_in_s1 = s2 - s1
 
     if not not_in_s1 and not not_in_s2:
-        LOGGER.info("Both dictionaries contains the same keys.")
+        logger.log(log_level, "Both dictionaries contains the same keys.")
 
     table = Table("Dictionary 1", "Dictionary 2", title="Key differences", title_justify="left")
 
@@ -91,11 +92,11 @@ def log_key_differences(dict_1, dict_2):
     for key in not_in_s1:
         table.add_row("", str(key))
 
-    LOGGER.info(capture_rich_output(table))
+    logger.log(log_level, capture_rich_output(table))
 
 
 if __name__ == "__main__":
-    import egse.logger  # noqa activates the logger
+    logging.basicConfig(level=logging.DEBUG)
 
     d1 = {
         "A": 1,
@@ -115,7 +116,7 @@ if __name__ == "__main__":
         "C": 3,
     }
 
-    log_differences(d1, d2)
-    log_key_differences(d1, d2)
+    log_differences(d1, d2, logging.DEBUG)
+    log_key_differences(d1, d2, logging.DEBUG)
 
-    log_differences(d1, d3)
+    log_differences(d1, d3, logging.DEBUG)

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/env.py

@@ -63,11 +63,11 @@ __all__ = [
 ]
 
 import contextlib
-import logging
 import os
 import warnings
 from pathlib import Path
 
+from egse.log import logger
 from egse.system import all_logging_disabled
 from egse.system import get_caller_info
 from egse.system import ignore_m_warning
@@ -75,7 +75,6 @@ from egse.system import ignore_m_warning
 from rich.console import Console
 
 console = Console(width=100)
-logger = logging.getLogger("egse.env")
 
 # Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
 # create the other environment variables that are specific to the project.
@@ -484,7 +483,7 @@ def get_local_settings_path() -> str or None:
 
 def has_conf_repo_location() -> bool:
     location = _env.get("CONF_REPO_LOCATION")
-    return True if location else False
+    return True if location in (None, NoValue) else False
 
 
 def get_conf_repo_location_env_name() -> str:

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/heartbeat.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import contextlib
-import logging
 import pickle
 import queue
 import random
@@ -13,8 +12,6 @@ import zmq
 from egse.system import format_datetime
 from egse.zmq_ser import MessageIdentifier
 
-_LOGGER = logging.getLogger(__name__)
-
 
 def _g_tick(period: float):
     """Generator for ticks every period [s]."""

{cgse-0.13.1 → cgse-0.14.1}/libs/cgse-common/src/egse/hk.py

@@ -10,7 +10,6 @@ __all__ = [
 ]
 import csv
 import datetime
-import logging
 from enum import Enum
 from pathlib import Path
 from typing import Optional
@@ -19,9 +18,11 @@ from typing import Union
 import dateutil.parser as date_parser
 import numpy as np
 import pandas as pd
+
 from egse.config import find_files
 from egse.env import get_data_storage_location
 from egse.env import get_site_id
+from egse.log import logger
 from egse.obsid import ObservationIdentifier
 from egse.obsid import obsid_from_storage
 from egse.setup import Setup
@@ -33,8 +34,6 @@ from egse.system import read_last_lines
 from egse.system import str_to_datetime
 from egse.system import time_since_epoch_1958
 
-_LOGGER = logging.getLogger(__name__)
-
 
 class TmDictionaryColumns(str, Enum):
     """Enumeration of the relevant columns in the TM dictionary spreadsheet.
@@ -438,7 +437,7 @@ def _get_housekeeping_daily(hk_name: str, data_dir, time_window: int = None, set
         # (those will have to be read entirely)
 
         else:
-            _LOGGER.warning(f"No HK available for {origin} on {start_time.day}/{start_time.month}/{start_time.year}")
+            logger.warning(f"No HK available for {origin} on {start_time.day}/{start_time.month}/{start_time.year}")
 
         day = (start_time + datetime.timedelta(days=1)).date()  # The day after the first day
         last_day = datetime.date(now.year, now.month, now.day)  # Today
@@ -463,7 +462,7 @@ def _get_housekeeping_daily(hk_name: str, data_dir, time_window: int = None, set
                         hk_array = np.append(hk_array, row[hk_index])
 
             else:
-                _LOGGER.warning(f"No HK available for {origin} on {day.day}/{day.month}/{day.year}")
+                logger.warning(f"No HK available for {origin} on {day.day}/{day.month}/{day.year}")
 
             day += datetime.timedelta(days=1)
 
@@ -673,7 +672,7 @@ def read_conversion_dict(storage_mnemonic: str, use_site: bool = False, setup: O
 
     else:
         if len(original_name_col) != len(correct_name_col):
-            _LOGGER.error(
+            logger.error(
                 f"Name columns in TM dictionary have different length: "
                 f"{len(original_name_col)} != {len(correct_name_col)}"
             )