cgse 0.12.1__tar.gz → 0.13.0__tar.gz

{cgse-0.12.1 → cgse-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse
-Version: 0.12.1
+Version: 0.13.0
 Summary: Generic Common-EGSE: Commanding and monitoring lab equipment
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse-0.12.1 → cgse-0.13.0}/docs/libs/cgse-core/registry.md

@@ -44,11 +44,12 @@ A microservice can register as follows:
 from egse.registry.client import AsyncRegistryClient
 
 registry_client = AsyncRegistryClient()
+registry_client.connect()
 
 service_id = await registry_client.register(
     name=service_name,
-    host=get_host_ip(),
-    port=get_port(),
+    host=hostname,
+    port=port_number,
     service_type=service_type,
     metadata={},
     ttl=30,

{cgse-0.12.1 → cgse-0.13.0}/docs/roadmap.md

@@ -31,20 +31,20 @@ Provide a `cgse` command that is extensible with new commands and command groups
 
 - [ ] Reference Frames and coordinate transformations -> Graphs
 - [ ] Metrics for all devices will be handled using InfluxDB
-- [ ] Use of Grafana to visualize the metrics
+- [x] Use of Grafana to visualize the metrics
 
 ## Core Services
 
 - [ ] Process Manager needs to be added with optimised design and performance
 - [ ] Synoptic Manager
 - [x] Distributed Service Registry
-- [ ] dynamic port assignment for all services
+- [x] dynamic port assignment for all services -> service registry
 
 ## Devices
 
 - [x] The Symétrie Hexapods: PUNA, ZONDA, JORAN
 - [x] The Keithley Data Acquisition Multimeter
-- [ ] The Lakeshore temperature controller
+- [x] The Lakeshore temperature controller
 
 ## Projects
 
@@ -72,7 +72,8 @@ Provide a `cgse` command that is extensible with new commands and command groups
 
 ## Removals
 
-- [ ] The `get_common_egse_root()` is of no use anymore and needs to be removed or replaced in some
+- [x] The `get_common_egse_root()` is of no use anymore and needs to be 
+  removed or replaced in some
   cases.
 
 ## Testing
@@ -80,4 +81,6 @@ Provide a `cgse` command that is extensible with new commands and command groups
 - [x] Add unit testing with `nox` running tests for Python 3.9, 3.10, 3.11, and 3.12
 - [ ] Add proper unit tests for all packages – using `pytest`
 - [ ] Add a CI test suite
-- [ ] Use GitHub Actions for running tests before merging
+- [ ] Add GitHub Action to check proper formatting of all the code in a pull 
+  request
+- [ ] Add GitHub Actions for running tests before merging

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.12.1"
+version = "0.13.0"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}
@@ -28,8 +28,7 @@ dependencies = [
     "pyyaml>=6.0.2",
     "rich>=13.9.4",
     "typer>=0.15.1",
-    "navdict>=0.2.4",
-
+    "navdict>=0.5.9",
     # Python 3.9 specific dependencies
     "numpy==1.26.4; python_version == '3.9'",
     "pandas==1.5.1; python_version == '3.9'",
@@ -38,10 +37,12 @@ dependencies = [
     "numpy>=2.1; python_version >= '3.10'",
     "pandas>=2.2.0; python_version >= '3.10'",
     "pyzmq>=25.1.0; python_version >= '3.10'",
-
     # Python 3.10+ entrypoints interface changed
     "importlib_metadata>=4.6.0; python_version <= '3.11'",
     "influxdb3-python",
+    "duckdb>=1.3.1",
+    "requests>=2.32.4",
+    "psycopg>=3.2.9",
 ]
 
 [project.scripts]

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/src/egse/command.py

@@ -145,46 +145,6 @@ def stringify_function_call(function_info: dict) -> str:
     return result
 
 
-def dry_run(func: Callable) -> Callable:
-    """This decorator prepares the function to handle a dry run.
-
-    A dry run is used to check the logic of an instrument commanding script without
-    actually executing the instrument commands. The commands are instead added to the
-    command sequence in the global state.
-
-    Args:
-        func: the function that needs to be executed
-
-    Returns:
-        A wrapper around the given function.
-    """
-
-    @functools.wraps(func)
-    def func_wrapper(self, *args, **kwargs):
-        from egse.state import GlobalState  # prevent circular import
-
-        if GlobalState.dry_run:
-            if callable(func) and func.__name__ == "client_call":
-                # This client_call function takes an additional argument which is the Proxy.
-                # the Proxy is not part of the CommandExecution signature and shall be removed
-                # FIXME: do we introduce a memory leak here by adding 'self' to this GlobalState?
-                args = args[1:]
-                try:
-                    self.validate_arguments(*args, **kwargs)
-                except CommandError as e_ce:
-                    GlobalState.add_command(InvalidCommandExecution(e_ce, self, *args, **kwargs))
-                else:
-                    GlobalState.add_command(CommandExecution(self, *args, **kwargs))
-            else:
-                FunctionExecution = namedtuple("FunctionExecution", ["name", "args", "kwargs"])
-                GlobalState.add_command(FunctionExecution(func.__name__, args, kwargs))
-            return Success("Command execution appended to command sequence, function not executed in dry_run.")
-        else:
-            return func(self, *args, **kwargs)
-
-    return func_wrapper
-
-
 def parse_format_string(fstring):
     """
     Parse and decode the format string.
@@ -479,7 +439,6 @@ class Command:
 
 
 class ClientServerCommand(Command):
-    @dry_run
     def client_call(self, other: type, *args: tuple, **kwargs: dict) -> Any:
         """
         This method is called at the client side. It is used by the Proxy

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/src/egse/control.py

@@ -22,12 +22,15 @@ from urllib3.exceptions import NewConnectionError
 from egse.decorators import retry
 from egse.decorators import retry_with_exponential_backoff
 from egse.listener import Listeners
+from egse.metrics import get_metrics_repo
 from egse.registry.client import RegistryClient
 from egse.signal import FileBasedSignaling
 from egse.system import SignalCatcher
 from egse.system import camel_to_kebab
 from egse.system import camel_to_snake
+from egse.system import str_to_datetime
 from egse.system import time_in_ms
+from egse.system import type_name
 from egse.zmq_ser import get_port_number
 
 try:
@@ -47,9 +50,6 @@ from egse.system import get_average_execution_times
 from egse.system import get_full_classname
 from egse.system import get_host_ip
 from egse.system import save_average_execution_time
-from influxdb_client_3 import InfluxDBClient3
-from influxdb_client_3.write_client.domain.write_precision import WritePrecision
-from influxdb_client_3 import Point
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -179,12 +179,15 @@ class ControlServer(metaclass=abc.ABCMeta):
 
         token = os.getenv("INFLUXDB3_AUTH_TOKEN")
         project = os.getenv("PROJECT")
-        self.metrics_time_precision = WritePrecision.MS
 
         if project and token:
-            self.client = InfluxDBClient3(database=project, host="http://localhost:8181", token=token)
+            self.metrics_client = get_metrics_repo(
+                "influxdb", {"host": "http://localhost:8181", "database": project, "token": token}
+            )
+            # self.metrics_client = get_metrics_repo("duckdb", {"db_path": "duckdb_metrics.db", "table_name": "cs_timeseries"})
+            self.metrics_client.connect()
         else:
-            self.client = None
+            self.metrics_client = None
             _LOGGER.warning(
                 "INFLUXDB3_AUTH_TOKEN and/or PROJECT environment variable is not set. "
                 "Metrics will not be propagated to InfluxDB."
@@ -545,7 +548,7 @@ class ControlServer(metaclass=abc.ABCMeta):
                 except Exception as exc:
                     _LOGGER.error(
                         textwrap.dedent(
-                            f"""\
+                            f"""{type_name(exc)}
                             An Exception occurred while collecting housekeeping from the device to be stored in {self.get_storage_mnemonic()}.
                             This might be a temporary problem, still needs to be looked into:
         
@@ -659,20 +662,21 @@ class ControlServer(metaclass=abc.ABCMeta):
             return
 
         try:
-            if self.client:
-                metrics_dictionary = {
-                    "measurement": origin.lower(),  # Table name
-                    "tags": {"site_id": SITE_ID, "origin": origin},  # Site ID, Origin
-                    "fields": dict((hk_name.lower(), hk[hk_name]) for hk_name in hk if hk_name != "timestamp"),
-                    "time": hk["timestamp"],
+            if self.metrics_client:
+                point = {
+                    "measurement": origin.lower(),
+                    "tags": {"site_id": SITE_ID, "origin": origin},
+                    "fields": {hk_name.lower(): hk[hk_name] for hk_name in hk if hk_name != "timestamp"},
+                    "time": str_to_datetime(hk["timestamp"]),
                 }
-                point = Point.from_dict(metrics_dictionary, write_precision=self.metrics_time_precision)
-                self.client.write(point)
+                self.metrics_client.write(point)
             else:
-                _LOGGER.warning(f"Could not write {origin} metrics to InfluxDB (self.client is None).")
+                _LOGGER.warning(
+                    f"Could not write {origin} metrics to the time series database (self.metrics_client is None)."
+                )
         except NewConnectionError:
             _LOGGER.warning(
-                f"No connection to InfluxDB could be established to propagate {origin} metrics.  Check "
+                f"No connection to the time series database could be established to propagate {origin} metrics.  Check "
                 f"whether this service is (still) running."
             )
 

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/src/egse/dummy.py

@@ -52,7 +52,6 @@ from egse.listener import Event
 from egse.listener import EventInterface
 from egse.protocol import CommandProtocol
 from egse.proxy import Proxy
-from egse.response import Failure
 from egse.settings import Settings
 from egse.system import SignalCatcher
 from egse.system import attrdict
@@ -648,6 +647,10 @@ def start_dev():
                             if not data:
                                 _LOGGER.info("Connection closed by peer, waiting for connection..")
                                 break  # connection closed by peer
+                            if data.decode().strip() == "QUIT":
+                                _LOGGER.info("QUIT command received, terminating...")
+                                quit_request = True
+                                break
                             if (response := process_command(data.decode().rstrip())) is not None:
                                 response = f"{response}\r\n".encode()
                                 conn.sendall(response)
@@ -667,6 +670,18 @@ def start_dev():
     _LOGGER.info("Dummy Device terminated.")
 
 
+@app.command()
+def stop_dev():
+    multiprocessing.current_process().name = "dummy_dev"
+
+    _LOGGER.info("Stopping the Dummy Device simulator")
+
+    dev = DummyDeviceEthernetInterface(DEV_HOST, DEV_PORT)
+    dev.connect()
+    dev.write("QUIT\n")
+    dev.disconnect()
+
+
 COMMAND_ACTIONS_RESPONSES = {"info": (None, f"Dummy Device {__version__}"), "get_value": (None, random.random)}
 
 

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/src/egse/metrics.py

@@ -1,14 +1,31 @@
+__all__ = [
+    "DataPoint",
+    "TimeSeriesRepository",
+    "define_metrics",
+    "get_metrics_repo",
+]
+
 import logging
+from dataclasses import dataclass
+from typing import Any
 from typing import Optional
+from typing import Protocol
+from typing import Self
 
 import numpy as np
 from prometheus_client import Gauge
 
 from egse.hk import TmDictionaryColumns
+from egse.plugin import load_plugins_fn
 from egse.settings import Settings
-from egse.setup import SetupError, load_setup, Setup
+from egse.setup import Setup
+from egse.setup import SetupError
+from egse.setup import load_setup
+from egse.system import format_datetime
+from egse.system import str_to_datetime
+
+logger = logging.getLogger("egse.metrics")
 
-LOGGER = logging.getLogger(__name__)
 SITE_ID = Settings.load("SITE").ID
 
 
@@ -64,7 +81,7 @@ def define_metrics(origin: str, dashboard: str = None, use_site: bool = False, s
             try:
                 metrics[syn_name] = Gauge(syn_name, description)
             except ValueError:
-                LOGGER.warning(f"ValueError for {syn_name}")
+                logger.warning(f"ValueError for {syn_name}")
 
         return metrics
 
@@ -100,4 +117,105 @@ def update_metrics(metrics: dict, updates: dict):
             else:
                 metrics[metric_name].set(float(value))
         except KeyError:
-            LOGGER.warning(f"Unknown metric name: {metric_name=}")
+            logger.warning(f"Unknown metric name: {metric_name=}")
+
+
+class PointLike(Protocol):
+    @staticmethod
+    def measurement(measurement_name: str) -> "PointLike": ...
+
+    def tag(self, key, value) -> Self: ...
+
+    def field(self, field, value) -> Self: ...
+
+    def time(self, time) -> Self: ...
+
+    def as_dict(self) -> dict: ...
+
+
+class DataPoint(PointLike):
+    def __init__(self, measurement_name: str):
+        self.measurement: str = measurement_name
+        self.tags: dict[str, str] = {}
+        self.fields: dict[str, Any] = {}
+        self.timestamp: int | str = format_datetime()
+
+    def as_dict(self):
+        if isinstance(self.timestamp, str):
+            timestamp = str_to_datetime(self.timestamp).timestamp()
+        else:
+            timestamp = self.timestamp
+
+        return {
+            "measurement": self.measurement,
+            "tags": self.tags,
+            "fields": self.fields,
+            "time": timestamp,
+        }
+
+    @staticmethod
+    def measurement(measurement_name):
+        p = DataPoint(measurement_name)
+        return p
+
+    def tag(self, key, value):
+        self.tags[key] = value
+        return self
+
+    def field(self, field, value):
+        self.fields[field] = value
+        return self
+
+    def time(self, time):
+        self.timestamp = time
+        return self
+
+
+class TimeSeriesRepository(Protocol):
+    def connect(self) -> None: ...
+
+    def write(self, points: PointLike | dict | list[PointLike | dict]) -> None: ...
+
+    def query(self, query_str: str, mode: str) -> Any: ...
+
+    def get_table_names(self) -> list[str]: ...
+
+    def get_column_names(self, table_name: str) -> list[str]: ...
+
+    def get_values_last_hours(self, table_name: str, column_name: str, hours: int, mode: str) -> Any: ...
+
+    def get_values_in_range(
+        self, table_name: str, column_name: str, start_time: str, end_time: str, mode: str
+    ) -> Any: ...
+
+    def close(self) -> None: ...
+
+
+def get_metrics_repo(plugin_name: str, config: dict[str, Any]) -> TimeSeriesRepository:
+    """
+    Create a TimeSeriesRepository instance from a plugin.
+
+    Args:
+        plugin_name: Name of the plugin (without .py extension)
+        config: Configuration parameters for the repository
+
+    Returns:
+        Configured TimeSeriesRepository instance.
+
+    Raises:
+        ModuleNotFoundError: If plugin not found
+        NotImplementedError: If plugin is missing get_repository_class()
+    """
+
+    package_name = "egse.plugins"
+    plugins = load_plugins_fn(f"{plugin_name}.py", package_name)
+
+    if plugin_name in plugins:
+        plugin = plugins[plugin_name]
+        if hasattr(plugin, "get_repository_class"):
+            repo_class = plugin.get_repository_class()
+        else:
+            raise NotImplementedError(f"Missing 'get_repository_class()` function in metrics plugin {plugin_name}.")
+        return repo_class(**config)
+    else:
+        raise ModuleNotFoundError(f"No plugin found for {plugin_name} in {package_name}.")

{cgse-0.12.1 → cgse-0.13.0}/libs/cgse-common/src/egse/plugin.py

@@ -3,18 +3,25 @@ This module provides function to load plugins and settings from entry-points.
 """
 
 __all__ = [
-    "load_plugins",
+    "load_plugins_ep",
+    "load_plugins_fn",
     "get_file_infos",
     "entry_points",
     "HierarchicalEntryPoints",
 ]
+
+import importlib.util
 import logging
 import os
 import sys
 import textwrap
 import traceback
+import types
+from functools import lru_cache
 from pathlib import Path
 
+from egse.system import type_name
+
 if sys.version_info >= (3, 12):  # Use the standard library version (Python 3.10+)
     from importlib.metadata import entry_points as lib_entry_points
     from importlib.metadata import EntryPoint
@@ -26,7 +33,8 @@ else:
 import click
 import rich
 
-_LOGGER = logging.getLogger(__name__)
+logger = logging.getLogger(__name__)
+HERE = Path(__file__).parent
 
 
 def entry_points(group: str) -> set[EntryPoint]:
@@ -43,7 +51,7 @@ def entry_points(group: str) -> set[EntryPoint]:
         return set()
 
 
-def load_plugins(entry_point: str) -> dict:
+def load_plugins_ep(entry_point: str) -> dict:
     """
     Returns a dictionary with plugins loaded. The keys are the names of the entry-points,
     the values are the loaded modules or objects.
@@ -58,11 +66,90 @@ def load_plugins(entry_point: str) -> dict:
             eps[ep.name] = ep.load()
         except Exception as exc:
             eps[ep.name] = None
-            _LOGGER.error(f"Couldn't load entry point: {exc}")
+            logger.error(f"Couldn't load entry point {entry_point}: {type_name(exc)} – {exc}")
 
     return eps
 
 
+@lru_cache
+def load_plugins_fn(pattern: str, package_name: str = None) -> dict[str, types.ModuleType]:
+    """
+    Returns a dictionary with plugins loaded for the filenames that match the given pattern.
+    The keys are the names of the modules, the values are the loaded modules.
+
+    If no package_name is provided, the pattern is relative to the location of this module
+    (which is in the top-level module `egse`).
+
+    If the pattern results in two or more modules with the same name, a warning will be logged
+    and only the last imported module will be returned in the dictionary.
+
+    Plugins are usually located in the `egse.plugins` module. Remember that `egse.plugins`
+    is a namespace and external packages can also deliver plugin modules in that location.
+
+    Note:
+        When a plugin cannot be loaded, an error is logged.
+
+        This function uses an LRU cache to avoid reloading modules. If you need to reload,
+        use `load_plugins_fn.cache_clear()` to reset the cache.
+
+    Raises:
+        ImportError when the given package_name cannot be imported as a module.
+
+    Examples:
+
+        # Loading the InfluxDB plugin for time series metrics
+        >>> influxdb = load_plugins_fn("influxdb.py", "egse.plugins.metrics")
+
+        # Loading the HDF5 storage plugin for PLATO
+        >>> hdf5 = load_plugins_fn("hdf5.py", "egse.plugins.storage")
+
+        # Loading all plugins
+        >>> x = load_plugins_fn("**/*.py", "egse.plugins")
+
+    """
+    loaded_modules = {}
+    failed_modules = {}  # we keep failed modules for housekeeping only
+
+    if package_name is None:
+        package_path = [HERE]
+    else:
+        try:
+            package = importlib.import_module(package_name)
+        except ImportError as exc:
+            raise ImportError(f"Cannot import package '{package_name}': {exc}")
+
+        if hasattr(package, "__path__"):
+            package_path = package.__path__
+        else:
+            package_path = [package.__file__.replace("__init__.py", "")]
+
+    # rich.print(package_path)
+
+    for path_entry in map(Path, package_path):
+        # rich.print(path_entry)
+        for plugin_file in path_entry.rglob(pattern):
+            # rich.print("    ", plugin_file)
+            module_name = plugin_file.stem
+            try:
+                spec = importlib.util.spec_from_file_location(module_name, plugin_file)
+                module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(module)
+                if module_name in loaded_modules:
+                    logger.warning(
+                        f"Overwriting module '{module_name}' from {loaded_modules[module_name].__file__} "
+                        f"with {plugin_file}."
+                    )
+                loaded_modules[module_name] = module
+            except Exception as exc:
+                error_msg = f"Couldn't load module '{module_name}' from {plugin_file}: {type_name(exc)} – {exc}"
+                logger.error(error_msg)
+                if module_name in failed_modules:
+                    logger.warning(f"Overwriting previously unloaded module '{module_name}' with {plugin_file}.")
+                failed_modules[module_name] = error_msg
+
+    return loaded_modules
+
+
 def get_file_infos(entry_point: str) -> dict[str, tuple[Path, str]]:
     """
     Returns a dictionary with location and filename of all the entries found for
@@ -93,7 +180,7 @@ def get_file_infos(entry_point: str) -> dict[str, tuple[Path, str]]:
             path = get_module_location(ep.module)
 
             if path is None:
-                _LOGGER.error(
+                logger.error(
                     f"The entry-point '{ep.name}' is ill defined. The module part doesn't exist or is a "
                     f"namespace. No settings are loaded for this entry-point."
                 )
@@ -101,7 +188,7 @@ def get_file_infos(entry_point: str) -> dict[str, tuple[Path, str]]:
                 eps[ep.name] = (path, ep.attr)
 
         except Exception as exc:
-            _LOGGER.error(f"The entry point '{ep.name}' is ill defined: {exc}")
+            logger.error(f"The entry point '{ep.name}' is ill defined: {exc}")
 
     return eps