cgse 0.12.1__tar.gz → 0.13.1__tar.gz

{cgse-0.12.1 → cgse-0.13.1}/.gitignore

@@ -18,6 +18,7 @@ dist
 
 .pytest_cache
 .coverage
+.coverage.*
 .nox
 htmlcov
 

{cgse-0.12.1 → cgse-0.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse
-Version: 0.12.1
+Version: 0.13.1
 Summary: Generic Common-EGSE: Commanding and monitoring lab equipment
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse-0.12.1 → cgse-0.13.1}/docs/libs/cgse-core/registry.md

@@ -44,11 +44,12 @@ A microservice can register as follows:
 from egse.registry.client import AsyncRegistryClient
 
 registry_client = AsyncRegistryClient()
+registry_client.connect()
 
 service_id = await registry_client.register(
     name=service_name,
-    host=get_host_ip(),
-    port=get_port(),
+    host=hostname,
+    port=port_number,
     service_type=service_type,
     metadata={},
     ttl=30,

{cgse-0.12.1 → cgse-0.13.1}/docs/roadmap.md

@@ -31,20 +31,20 @@ Provide a `cgse` command that is extensible with new commands and command groups
 
 - [ ] Reference Frames and coordinate transformations -> Graphs
 - [ ] Metrics for all devices will be handled using InfluxDB
-- [ ] Use of Grafana to visualize the metrics
+- [x] Use of Grafana to visualize the metrics
 
 ## Core Services
 
 - [ ] Process Manager needs to be added with optimised design and performance
 - [ ] Synoptic Manager
 - [x] Distributed Service Registry
-- [ ] dynamic port assignment for all services
+- [x] dynamic port assignment for all services -> service registry
 
 ## Devices
 
 - [x] The Symétrie Hexapods: PUNA, ZONDA, JORAN
 - [x] The Keithley Data Acquisition Multimeter
-- [ ] The Lakeshore temperature controller
+- [x] The Lakeshore temperature controller
 
 ## Projects
 
@@ -67,12 +67,14 @@ Provide a `cgse` command that is extensible with new commands and command groups
   use of command YAML files, replace `dynamic_interface` with `dynamic_command`.
 - [ ] Introduce asyncio into the commanding protocol, e.g. `get_status()` 
   and `get_housekeeping()` shall be handled asynchronously.
-- [ ] GlobalState Setup needs some redesign, especially `GlobalState.setup` which should not consult
+- [x] GlobalState Setup needs some redesign, especially `GlobalState.setup` 
+  which should not consult
   the configuration manager by default.
 
 ## Removals
 
-- [ ] The `get_common_egse_root()` is of no use anymore and needs to be removed or replaced in some
+- [x] The `get_common_egse_root()` is of no use anymore and needs to be 
+  removed or replaced in some
   cases.
 
 ## Testing
@@ -80,4 +82,6 @@ Provide a `cgse` command that is extensible with new commands and command groups
 - [x] Add unit testing with `nox` running tests for Python 3.9, 3.10, 3.11, and 3.12
 - [ ] Add proper unit tests for all packages – using `pytest`
 - [ ] Add a CI test suite
-- [ ] Use GitHub Actions for running tests before merging
+- [ ] Add GitHub Action to check proper formatting of all the code in a pull 
+  request
+- [ ] Add GitHub Actions for running tests before merging

{cgse-0.12.1 → cgse-0.13.1}/justfile

@@ -25,7 +25,7 @@ hello arg="":
 system-info:
   @echo "This is an {{arch()}} machine, running {{os()}}".
 
-default_python := "3.9.20"
+default_python := "3.10"
 
 # Start an IPython session
 ipython version=default_python:

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.12.1"
+version = "0.13.1"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}
@@ -19,7 +19,7 @@ keywords = [
     "software framework"
 ]
 dependencies = [
-    "click>=8.1.8",
+    "click>=8.2.1",
     "deepdiff>=8.1.1",
     "distro>=1.9.0",
     "gitpython>=3.1.44",
@@ -27,9 +27,8 @@ dependencies = [
     "psutil>=6.1.1",
     "pyyaml>=6.0.2",
     "rich>=13.9.4",
-    "typer>=0.15.1",
-    "navdict>=0.2.4",
-
+    "typer>=0.16.0",
+    "navdict>=0.5.9",
     # Python 3.9 specific dependencies
     "numpy==1.26.4; python_version == '3.9'",
     "pandas==1.5.1; python_version == '3.9'",
@@ -38,10 +37,12 @@ dependencies = [
     "numpy>=2.1; python_version >= '3.10'",
     "pandas>=2.2.0; python_version >= '3.10'",
     "pyzmq>=25.1.0; python_version >= '3.10'",
-
     # Python 3.10+ entrypoints interface changed
     "importlib_metadata>=4.6.0; python_version <= '3.11'",
     "influxdb3-python",
+    "duckdb>=1.3.1",
+    "requests>=2.32.4",
+    "psycopg>=3.2.9",
 ]
 
 [project.scripts]
@@ -57,7 +58,7 @@ cgse-common = "cgse_common:settings.yaml"
 [tool.pytest.ini_options]
 pythonpath = "src"
 testpaths = ["tests"]
-addopts = "-rA --cov --cov-branch --cov-report html"
+addopts = "-ra --cov --cov-branch --cov-report html"
 log_cli = true
 log_cli_level = "INFO"
 filterwarnings = [

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/src/cgse_common/cgse.py

@@ -85,7 +85,7 @@ class SortedCommandGroup(TyperGroup):
         return sorted(commands, key=get_command_priority)
 
 
-app = typer.Typer(add_completion=True, cls=SortedCommandGroup, rich_markup_mode="markdown")
+app = typer.Typer(add_completion=True, cls=SortedCommandGroup)
 
 
 @app.command()
@@ -121,22 +121,33 @@ def version(ctx: typer.Context):
             )
 
 
+def subcommand_callback(ctx: typer.Context):
+    """Normal callback."""
+    if ctx.invoked_subcommand is None:
+        typer.echo("No command specified:")
+        typer.echo(ctx.get_help())
+        raise typer.Exit()
+
+
 def build_app():
     global app
 
+    # rich.print("Available command groups:", entry_points("cgse.command"))
+
     for ep in entry_points("cgse.command"):
         try:
             obj = ep.load()
             if isinstance(obj, typer.Typer):
-                app.add_typer(ep.load(), name=ep.name)
+                obj.callback(invoke_without_command=True)(subcommand_callback)
+                app.add_typer(obj, name=ep.name)
             else:
-                app.command()(ep.load())
+                app.command()(obj)
         except Exception as exc:
             app.command()(broken_command(ep.name, ep.module, exc))
 
     cgse_eps = HierarchicalEntryPoints("cgse.service")
 
-    # rich.print("Available groups:", cgse_eps.get_all_groups())
+    # rich.print("Available services groups:", cgse_eps.get_all_groups())
 
     for group in cgse_eps.get_all_groups():
         for ep in entry_points(group):
@@ -145,12 +156,14 @@ def build_app():
                     app.add_typer(ep.load(), name=ep.name)
                 else:
                     command_group = snake_to_title(group.split(".")[-1])
-                    app.add_typer(ep.load(), name=ep.name, rich_help_panel=command_group)
+                    plugin_app: typer.Typer = ep.load()
+                    plugin_app.callback(invoke_without_command=True)(subcommand_callback)
+                    app.add_typer(plugin_app, name=ep.name, rich_help_panel=command_group)
             except Exception as exc:
                 app.command()(broken_command(ep.name, ep.module, exc))
 
 
-@app.callback(no_args_is_help=True, invoke_without_command=True)
+@app.callback(invoke_without_command=True)
 def main(ctx: typer.Context, verbose: bool = False):
     """
     The `cgse` command is used to:
@@ -167,6 +180,9 @@ def main(ctx: typer.Context, verbose: bool = False):
 
     ctx.obj = AppState(verbose=verbose)
 
+    if ctx.invoked_subcommand is None:
+        # print("Try 'cgse --help' for a list of commands.")
+        typer.echo(ctx.get_help())
 
 build_app()
 

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/src/egse/command.py

@@ -145,46 +145,6 @@ def stringify_function_call(function_info: dict) -> str:
     return result
 
 
-def dry_run(func: Callable) -> Callable:
-    """This decorator prepares the function to handle a dry run.
-
-    A dry run is used to check the logic of an instrument commanding script without
-    actually executing the instrument commands. The commands are instead added to the
-    command sequence in the global state.
-
-    Args:
-        func: the function that needs to be executed
-
-    Returns:
-        A wrapper around the given function.
-    """
-
-    @functools.wraps(func)
-    def func_wrapper(self, *args, **kwargs):
-        from egse.state import GlobalState  # prevent circular import
-
-        if GlobalState.dry_run:
-            if callable(func) and func.__name__ == "client_call":
-                # This client_call function takes an additional argument which is the Proxy.
-                # the Proxy is not part of the CommandExecution signature and shall be removed
-                # FIXME: do we introduce a memory leak here by adding 'self' to this GlobalState?
-                args = args[1:]
-                try:
-                    self.validate_arguments(*args, **kwargs)
-                except CommandError as e_ce:
-                    GlobalState.add_command(InvalidCommandExecution(e_ce, self, *args, **kwargs))
-                else:
-                    GlobalState.add_command(CommandExecution(self, *args, **kwargs))
-            else:
-                FunctionExecution = namedtuple("FunctionExecution", ["name", "args", "kwargs"])
-                GlobalState.add_command(FunctionExecution(func.__name__, args, kwargs))
-            return Success("Command execution appended to command sequence, function not executed in dry_run.")
-        else:
-            return func(self, *args, **kwargs)
-
-    return func_wrapper
-
-
 def parse_format_string(fstring):
     """
     Parse and decode the format string.
@@ -479,7 +439,6 @@ class Command:
 
 
 class ClientServerCommand(Command):
-    @dry_run
     def client_call(self, other: type, *args: tuple, **kwargs: dict) -> Any:
         """
         This method is called at the client side. It is used by the Proxy

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/src/egse/control.py

@@ -22,12 +22,15 @@ from urllib3.exceptions import NewConnectionError
 from egse.decorators import retry
 from egse.decorators import retry_with_exponential_backoff
 from egse.listener import Listeners
+from egse.metrics import get_metrics_repo
 from egse.registry.client import RegistryClient
 from egse.signal import FileBasedSignaling
 from egse.system import SignalCatcher
 from egse.system import camel_to_kebab
 from egse.system import camel_to_snake
+from egse.system import str_to_datetime
 from egse.system import time_in_ms
+from egse.system import type_name
 from egse.zmq_ser import get_port_number
 
 try:
@@ -47,9 +50,6 @@ from egse.system import get_average_execution_times
 from egse.system import get_full_classname
 from egse.system import get_host_ip
 from egse.system import save_average_execution_time
-from influxdb_client_3 import InfluxDBClient3
-from influxdb_client_3.write_client.domain.write_precision import WritePrecision
-from influxdb_client_3 import Point
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -179,12 +179,15 @@ class ControlServer(metaclass=abc.ABCMeta):
 
         token = os.getenv("INFLUXDB3_AUTH_TOKEN")
         project = os.getenv("PROJECT")
-        self.metrics_time_precision = WritePrecision.MS
 
         if project and token:
-            self.client = InfluxDBClient3(database=project, host="http://localhost:8181", token=token)
+            self.metrics_client = get_metrics_repo(
+                "influxdb", {"host": "http://localhost:8181", "database": project, "token": token}
+            )
+            # self.metrics_client = get_metrics_repo("duckdb", {"db_path": "duckdb_metrics.db", "table_name": "cs_timeseries"})
+            self.metrics_client.connect()
         else:
-            self.client = None
+            self.metrics_client = None
             _LOGGER.warning(
                 "INFLUXDB3_AUTH_TOKEN and/or PROJECT environment variable is not set. "
                 "Metrics will not be propagated to InfluxDB."
@@ -545,7 +548,7 @@ class ControlServer(metaclass=abc.ABCMeta):
                 except Exception as exc:
                     _LOGGER.error(
                         textwrap.dedent(
-                            f"""\
+                            f"""{type_name(exc)}
                             An Exception occurred while collecting housekeeping from the device to be stored in {self.get_storage_mnemonic()}.
                             This might be a temporary problem, still needs to be looked into:
         
@@ -659,20 +662,21 @@ class ControlServer(metaclass=abc.ABCMeta):
             return
 
         try:
-            if self.client:
-                metrics_dictionary = {
-                    "measurement": origin.lower(),  # Table name
-                    "tags": {"site_id": SITE_ID, "origin": origin},  # Site ID, Origin
-                    "fields": dict((hk_name.lower(), hk[hk_name]) for hk_name in hk if hk_name != "timestamp"),
-                    "time": hk["timestamp"],
+            if self.metrics_client:
+                point = {
+                    "measurement": origin.lower(),
+                    "tags": {"site_id": SITE_ID, "origin": origin},
+                    "fields": {hk_name.lower(): hk[hk_name] for hk_name in hk if hk_name != "timestamp"},
+                    "time": str_to_datetime(hk["timestamp"]),
                 }
-                point = Point.from_dict(metrics_dictionary, write_precision=self.metrics_time_precision)
-                self.client.write(point)
+                self.metrics_client.write(point)
             else:
-                _LOGGER.warning(f"Could not write {origin} metrics to InfluxDB (self.client is None).")
+                _LOGGER.warning(
+                    f"Could not write {origin} metrics to the time series database (self.metrics_client is None)."
+                )
         except NewConnectionError:
             _LOGGER.warning(
-                f"No connection to InfluxDB could be established to propagate {origin} metrics.  Check "
+                f"No connection to the time series database could be established to propagate {origin} metrics.  Check "
                 f"whether this service is (still) running."
             )
 

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/src/egse/dummy.py

@@ -52,7 +52,6 @@ from egse.listener import Event
 from egse.listener import EventInterface
 from egse.protocol import CommandProtocol
 from egse.proxy import Proxy
-from egse.response import Failure
 from egse.settings import Settings
 from egse.system import SignalCatcher
 from egse.system import attrdict
@@ -648,6 +647,10 @@ def start_dev():
                             if not data:
                                 _LOGGER.info("Connection closed by peer, waiting for connection..")
                                 break  # connection closed by peer
+                            if data.decode().strip() == "QUIT":
+                                _LOGGER.info("QUIT command received, terminating...")
+                                quit_request = True
+                                break
                             if (response := process_command(data.decode().rstrip())) is not None:
                                 response = f"{response}\r\n".encode()
                                 conn.sendall(response)
@@ -667,6 +670,18 @@ def start_dev():
     _LOGGER.info("Dummy Device terminated.")
 
 
+@app.command()
+def stop_dev():
+    multiprocessing.current_process().name = "dummy_dev"
+
+    _LOGGER.info("Stopping the Dummy Device simulator")
+
+    dev = DummyDeviceEthernetInterface(DEV_HOST, DEV_PORT)
+    dev.connect()
+    dev.write("QUIT\n")
+    dev.disconnect()
+
+
 COMMAND_ACTIONS_RESPONSES = {"info": (None, f"Dummy Device {__version__}"), "get_value": (None, random.random)}
 
 

{cgse-0.12.1 → cgse-0.13.1}/libs/cgse-common/src/egse/metrics.py

@@ -1,14 +1,31 @@
+__all__ = [
+    "DataPoint",
+    "TimeSeriesRepository",
+    "define_metrics",
+    "get_metrics_repo",
+]
+
 import logging
+from dataclasses import dataclass
+from typing import Any
 from typing import Optional
+from typing import Protocol
+from typing import Self
 
 import numpy as np
 from prometheus_client import Gauge
 
 from egse.hk import TmDictionaryColumns
+from egse.plugin import load_plugins_fn
 from egse.settings import Settings
-from egse.setup import SetupError, load_setup, Setup
+from egse.setup import Setup
+from egse.setup import SetupError
+from egse.setup import load_setup
+from egse.system import format_datetime
+from egse.system import str_to_datetime
+
+logger = logging.getLogger("egse.metrics")
 
-LOGGER = logging.getLogger(__name__)
 SITE_ID = Settings.load("SITE").ID
 
 
@@ -64,7 +81,7 @@ def define_metrics(origin: str, dashboard: str = None, use_site: bool = False, s
             try:
                 metrics[syn_name] = Gauge(syn_name, description)
             except ValueError:
-                LOGGER.warning(f"ValueError for {syn_name}")
+                logger.warning(f"ValueError for {syn_name}")
 
         return metrics
 
@@ -100,4 +117,105 @@ def update_metrics(metrics: dict, updates: dict):
             else:
                 metrics[metric_name].set(float(value))
         except KeyError:
-            LOGGER.warning(f"Unknown metric name: {metric_name=}")
+            logger.warning(f"Unknown metric name: {metric_name=}")
+
+
+class PointLike(Protocol):
+    @staticmethod
+    def measurement(measurement_name: str) -> "PointLike": ...
+
+    def tag(self, key, value) -> Self: ...
+
+    def field(self, field, value) -> Self: ...
+
+    def time(self, time) -> Self: ...
+
+    def as_dict(self) -> dict: ...
+
+
+class DataPoint(PointLike):
+    def __init__(self, measurement_name: str):
+        self.measurement: str = measurement_name
+        self.tags: dict[str, str] = {}
+        self.fields: dict[str, Any] = {}
+        self.timestamp: int | str = format_datetime()
+
+    def as_dict(self):
+        if isinstance(self.timestamp, str):
+            timestamp = str_to_datetime(self.timestamp).timestamp()
+        else:
+            timestamp = self.timestamp
+
+        return {
+            "measurement": self.measurement,
+            "tags": self.tags,
+            "fields": self.fields,
+            "time": timestamp,
+        }
+
+    @staticmethod
+    def measurement(measurement_name):
+        p = DataPoint(measurement_name)
+        return p
+
+    def tag(self, key, value):
+        self.tags[key] = value
+        return self
+
+    def field(self, field, value):
+        self.fields[field] = value
+        return self
+
+    def time(self, time):
+        self.timestamp = time
+        return self
+
+
+class TimeSeriesRepository(Protocol):
+    def connect(self) -> None: ...
+
+    def write(self, points: PointLike | dict | list[PointLike | dict]) -> None: ...
+
+    def query(self, query_str: str, mode: str) -> Any: ...
+
+    def get_table_names(self) -> list[str]: ...
+
+    def get_column_names(self, table_name: str) -> list[str]: ...
+
+    def get_values_last_hours(self, table_name: str, column_name: str, hours: int, mode: str) -> Any: ...
+
+    def get_values_in_range(
+        self, table_name: str, column_name: str, start_time: str, end_time: str, mode: str
+    ) -> Any: ...
+
+    def close(self) -> None: ...
+
+
+def get_metrics_repo(plugin_name: str, config: dict[str, Any]) -> TimeSeriesRepository:
+    """
+    Create a TimeSeriesRepository instance from a plugin.
+
+    Args:
+        plugin_name: Name of the plugin (without .py extension)
+        config: Configuration parameters for the repository
+
+    Returns:
+        Configured TimeSeriesRepository instance.
+
+    Raises:
+        ModuleNotFoundError: If plugin not found
+        NotImplementedError: If plugin is missing get_repository_class()
+    """
+
+    package_name = "egse.plugins"
+    plugins = load_plugins_fn(f"{plugin_name}.py", package_name)
+
+    if plugin_name in plugins:
+        plugin = plugins[plugin_name]
+        if hasattr(plugin, "get_repository_class"):
+            repo_class = plugin.get_repository_class()
+        else:
+            raise NotImplementedError(f"Missing 'get_repository_class()` function in metrics plugin {plugin_name}.")
+        return repo_class(**config)
+    else:
+        raise ModuleNotFoundError(f"No plugin found for {plugin_name} in {package_name}.")