cgse 0.11.4__tar.gz → 0.12.0__tar.gz

cgse-0.12.0/.github/workflows/ruff-format-check.yaml

@@ -0,0 +1,19 @@
+name: Ruff Formatting Check
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  ruff-format:
+    name: Run ruff format for the whole project
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Ruff Format Check
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: format --check --diff .

{cgse-0.11.4 → cgse-0.12.0}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: cgse
-Version: 0.11.4
+Version: 0.12.0
 Summary: Generic Common-EGSE: Commanding and monitoring lab equipment
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>
 License: MIT
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: cgse-common
 Requires-Dist: cgse-core
 Requires-Dist: cgse-tools

cgse-0.12.0/advanced-process-monitoring.py

@@ -0,0 +1,111 @@
+import psutil
+import time
+from dataclasses import dataclass
+from typing import List, Dict
+import json
+
+
+@dataclass
+class ProcessInfo:
+    pid: int
+    name: str
+    username: str
+    cpu_percent: float
+    memory_percent: float
+    memory_mb: float
+    status: str
+    create_time: float
+    command: str
+
+    def to_dict(self):
+        return {
+            "pid": self.pid,
+            "name": self.name,
+            "username": self.username,
+            "cpu_percent": self.cpu_percent,
+            "memory_percent": self.memory_percent,
+            "memory_mb": self.memory_mb,
+            "status": self.status,
+            "create_time": self.create_time,
+            "command": self.command,
+        }
+
+
+class ProcessMonitor:
+    def __init__(self):
+        self.previous_cpu = {}
+
+    def get_processes(self, filter_func=None) -> List[ProcessInfo]:
+        """Get current processes with optional filtering"""
+        processes = []
+
+        for proc in psutil.process_iter():
+            try:
+                with proc.oneshot():  # Efficient way to get multiple attributes
+                    info = ProcessInfo(
+                        pid=proc.pid,
+                        name=proc.name(),
+                        username=proc.username(),
+                        cpu_percent=proc.cpu_percent(),
+                        memory_percent=proc.memory_percent(),
+                        memory_mb=proc.memory_info().rss / 1024 / 1024,
+                        status=proc.status(),
+                        create_time=proc.create_time(),
+                        command=" ".join(proc.cmdline()) if proc.cmdline() else proc.name(),
+                    )
+
+                    if filter_func is None or filter_func(info):
+                        processes.append(info)
+
+            except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+                continue
+
+        return processes
+
+    def find_by_name(self, name: str) -> List[ProcessInfo]:
+        """Find processes by name"""
+        return self.get_processes(lambda p: name.lower() in p.name.lower())
+
+    def find_high_cpu(self, threshold: float = 5.0) -> List[ProcessInfo]:
+        """Find processes using high CPU"""
+        return self.get_processes(lambda p: p.cpu_percent > threshold)
+
+    def find_high_memory(self, threshold_mb: float = 100.0) -> List[ProcessInfo]:
+        """Find processes using high memory"""
+        return self.get_processes(lambda p: p.memory_mb > threshold_mb)
+
+    def kill_process(self, pid: int, force: bool = False):
+        """Safely kill a process"""
+        try:
+            proc = psutil.Process(pid)
+            if force:
+                proc.kill()
+            else:
+                proc.terminate()
+            return True
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return False
+
+    def export_to_json(self, filename: str):
+        """Export current process list to JSON"""
+        processes = [p.to_dict() for p in self.get_processes()]
+        with open(filename, "w") as f:
+            json.dump(processes, f, indent=2)
+
+
+# Usage examples
+monitor = ProcessMonitor()
+
+# Find all Python processes
+python_procs = monitor.find_by_name("python")
+print(f"Found {len(python_procs)} Python processes")
+for proc in python_procs:
+    print(f"{proc.name} ({proc.pid} – {proc.cpu_percent}%)")
+
+# Find high CPU usage
+high_cpu = monitor.find_high_cpu(10.0)
+for proc in high_cpu:
+    print(f"High CPU: {proc.name} (PID: {proc.pid}) - {proc.cpu_percent}%")
+
+# Export current state
+monitor.export_to_json("processes.json")

{cgse-0.11.4 → cgse-0.12.0}/bump.py

@@ -16,7 +16,7 @@ Note:
 """
 
 # /// script
-# requires-python = ">=3.9"
+# requires-python = ">=3.10"
 # dependencies = [
 #   "tomlkit",
 #   "rich",
@@ -33,17 +33,17 @@ import tomlkit.exceptions
 import typer
 
 
-def bump_version(version, part='patch'):
-    major, minor, patch = map(int, version.split('.'))
+def bump_version(version, part="patch"):
+    major, minor, patch = map(int, version.split("."))
 
-    if part == 'major':
+    if part == "major":
         major += 1
         minor = 0
         patch = 0
-    elif part == 'minor':
+    elif part == "minor":
         minor += 1
         patch = 0
-    elif part == 'patch':
+    elif part == "patch":
         patch += 1
     else:
         raise ValueError("Part must be 'major', 'minor', or 'patch'")
@@ -77,7 +77,7 @@ def update_project_version(project_dir, new_version):
             tomlkit.dump(data, file)
 
     except tomlkit.exceptions.NonExistentKey:
-        rich.print(f"[red]\[project.version] is not defined in pyproject.toml in {project_dir}[/]")
+        rich.print(rf"[red]\[project.version] is not defined in pyproject.toml in {project_dir}[/]")
 
 
 def update_all_projects_in_monorepo(root_dir: pathlib.Path, part: str, dry_run: bool = False, verbose: bool = True):
@@ -117,7 +117,6 @@ app = typer.Typer()
 
 @app.command()
 def main(part: str, dry_run: bool = False, verbose: bool = True):
-
     monorepo_root = pathlib.Path(__file__).parent.resolve()
 
     cwd = os.getcwd()

cgse-0.12.0/docs/dev_guide/monitoring.md

@@ -0,0 +1,261 @@
+# Monitoring with InfluxDB and Grafana
+
+The CGSE provides mechanisms to monitor metrics from different processes and devices.  These are 
+collected by InfluxDB and are visualised by means of Grafana dashboards.
+
+In this section, we explain how to install both InfluxDB and Grafana on the Server, how to set up and populate a database 
+in InfluxDB, and how to visualise the metrics in Grafana dashboards.
+
+
+---
+
+## Installation
+
+### InfluxDB3 Core
+
+To install InfluxDB3 Core, execute this in a Terminal on the Server:
+
+```
+curl -s https://repos.influxdata.com/influxdata-archive_compat.key > influxdata-archive_compat.key
+
+echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
+echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
+
+sudo apt-get update
+
+sudo apt-get install influxdb3-core
+```
+
+For database management, you will need a token to interact with InfluxDB.  This can be generated as follows (in a 
+Terminal on the Server):
+
+```
+influxdb3 create token --admin
+```
+
+This will print out a token, starting with `apiv3_`.  Export this value as environment variable `INFLUXDB3_AUTH_TOKEN`.
+
+
+### Grafana
+
+To install Grafana, execute this in a Terminal on the Server:
+
+```
+sudo apt-get install -y apt-transport-https software-properties-common wget
+
+sudo mkdir -p /etc/apt/keyrings/
+wget -q -O - https://apt.grafana.com/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/grafana.gpg > /dev/null
+
+echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
+sudo apt-get update
+
+sudo apt-get install grafana
+
+sudo /bin/systemctl daemon-reload
+sudo /bin/systemctl enable grafana-server
+```
+
+---
+
+## Database Structure
+
+### Database Name
+
+In the current implementation, the name of the database in which all CGSE metrics are stored, is taken 
+from the `PROJECT` environment variable.  At a later stage, we could change this by means of a dedicated 
+environment variable or by specifying it in the local settings file.
+
+To inspect the names of the available databases, execute this in a Terminal on the Server:
+
+```
+influxdb3 show databases
+```
+
+The command to create a database manually (from a Terminal on the Server) is:
+
+```
+influxdb3 create database <database name>
+```
+
+### Table Names
+
+For each of the processes, we have a dedicated table in the database, the name of which is the same as 
+the storage mnemonic of the process.
+
+To get an overview of the available tables, execute this in a Terminal on the Server:
+
+```
+influxdb3 query --database <database name> "SHOW TABLES"
+```
+
+### Content of the Tables
+
+For each table, the column names are the names of the corresponding housekeeping/metrics parameters.  The name of the column with the timestamp is `TIME`. 
+
+To get an overview of the columns for a given table in a given database, execute this in a Terminal on the Server:
+
+```
+influxdb3 query --database <database name> "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '<table name>'"
+```
+
+To get an overview of the data stored in a given table (sorted by ascending order of timestamp), execute this in a 
+Terminal on the Server:
+
+```
+influxdb3 query --database <database name> "SELECT * FROM <table name> ORDER BY TIME"
+```
+
+---
+
+## Propagating Metrics to InfluxDB via Python
+
+To populate the metrics in InfluxDB, we use the Python package `influxdb3-python`. This has been added to the `pyproject.toml` 
+file of the `cgse-common` module.
+
+When implementing a new device, nothing has to be done specifically to propagate the metrics to InfluxDB.  This is 
+automatically being taken care of in the `serve` method of the `ControlServer` class.  In the section where the 
+housekeeping information is written to a dedicated CSV file, the `propagate_metrics` method of the `ControlServer` 
+class is called and this takes care of everything.
+
+
+
+[//]: # (### Establishing a Connection)
+
+[//]: # ()
+[//]: # (In the initialisation a the `CommandProtocol`, an `influxdb_client_3.InfluxDBClient3` is created &#40;and stored in )
+
+[//]: # (`self.client`&#41;, which establishes a connection to InfluxDB.  This is also where the write precision for the )
+
+[//]: # (timestamp is specified.  Note that - to establish this connection - both the `PROJECT` and `INFLUXDB3_AUTH_TOKEN` )
+
+[//]: # (environment variables have to be set.  If this is not the case, `self.client` will be `None`. )
+
+[//]: # ()
+[//]: # (Note that when the InfluxDB process is &#40;re-&#41;started, the individual processes pick up on this automatically &#40;so you )
+
+[//]: # (don't have to do anything extra for the metrics to start appearing in the database again&#41;.)
+
+[//]: # ()
+[//]: # (### Device Protocols)
+
+[//]: # ()
+[//]: # (In the implementing protocols, in the `get_housekeeping` method, we have to propagate the metrics to InfluxDB.  Usually )
+
+[//]: # (we have an `hk` dictionary &#40;or something alike&#41; with the housekeeping information that will be stored in a dedicated CSV file.  This dictionary )
+
+[//]: # (also contains the timestamp for all parameters.  This is the return value of the `get_housekeeping` method.)
+
+[//]: # ()
+[//]: # (To propagate these housekeeping parameters as metrics to InfluxDB, you have to construct a dictionary &#40;see further for the required content&#41;, )
+
+[//]: # (say `metrics_dictionary`, which we will write to InfluxDB, as follows &#40;in the `get_housekeeping` method&#41;:)
+
+[//]: # ()
+[//]: # (``` python )
+
+[//]: # (try:)
+
+[//]: # (    if self.client:)
+
+[//]: # (        metrics_dictionary = {...})
+
+[//]: # (        point = Point.from_dict&#40;metrics_dictionary, write_precision=self.metrics_time_precision&#41;)
+
+[//]: # (        self.client.write&#40;point&#41;)
+
+[//]: # ( except NewConnectionError:)
+
+[//]: # (    pass)
+
+[//]: # (```)
+
+[//]: # ()
+[//]: # (This requires the following imports:)
+
+[//]: # ()
+[//]: # (``` python)
+
+[//]: # (from influxdb_client_3 import Point)
+
+[//]: # (from urllib3.exceptions import NewConnectionError)
+
+[//]: # (```)
+
+[//]: # ()
+[//]: # (#### Metrics Dictionary )
+
+[//]: # ()
+[//]: # (The `metrics_dictionary` has the following structure:)
+
+[//]: # ()
+[//]: # (- `measurement`: Storage mnemonic of the process, which will be used as table name in the database;)
+
+[//]: # (- `tags`: Dictionary with tags for the table in the database.  At this point, its function is not entirely clear yet. We are using the following tags:)
+
+[//]: # (  - `site_id`: Site identifier, as taken from the settings/setup &#40;e.g. "KUL", "CSL",...&#41;;)
+
+[//]: # (  - `origin`: Storage mnemonic of the process;)
+
+[//]: # (- `fields`: Dictionary with metrics.  This is the same as the `hk` dictionary but without the timestamp;)
+
+[//]: # (- `time`: Timestamp as taken from the `hk` dictionary.)
+
+[//]: # ()
+[//]: # (### Closing the Connection)
+
+[//]: # ()
+[//]: # (In the `quit` method of the device protocol, do not forget to close the connection to InfluxDB, by adding this line:)
+
+[//]: # ()
+[//]: # (``` python)
+
+[//]: # (if self.client:)
+
+[//]: # (    self.client.close&#40;&#41;)
+
+[//]: # (```)
+
+---
+
+## Visualisation of the Metrics
+
+The Metrics are visualised in Grafana dashboards, but before we can start building those, we have to instruct Grafana to 
+interpret InfluxDB as its data source.
+
+### Adding InfluxDB as Data Source
+
+After the InfluxDB and Grafana processes have been started, you can access Grafana in your browser via http://localhost:3000. 
+
+On the left-hand side, select (under "Connections") "Add new connection" and search for "Influx DB".  After clicking on 
+the search result, a Settings tab will open, in which you have to enter the following information:
+
+- `Name`: Arbitrary name for the data source (it's a good idea to make this reflect the database you are settings up, e.g. `influxdb3-ariel`);
+- `Query language`: Select "SQL";
+- `HTTP` > `URL`: `http://127.0.0.1:8181`;
+- `Auth`: Only enable "Basic auth";
+- `InfluxDB Details` > `Database`: Name of the database for which you want to create Grafana dashboards;
+- `InfluxDB Details` > `Token`: InfluxDB token you have created earlier (see above; starting with `apiv3_`);
+- `InfluxDB Details`: Enable "Insecure Connection".
+
+After having configured all of this, press "Save & test".
+
+### Creating Grafana Dashboards
+
+Creating a Grafana dashboard can be done by clicking "Dashboards" on the left-hand side.  In the "Queries" tab of such a 
+dashboard panel, you have to specify the following information:
+
+- `Data source`: Name of the data source you have just created (selectable via a drop-down menu);
+- `Table`: Name of the table for which you want to visualise metrics.  Remember that this is the same as the storage
+           mnemonic of the process generating the metrics (selectable via a drop-down menu);
+- `Data operations`: Besides the name of the metric (selectable via a drop-down menu), you also have to add `time` under 
+                     `Data Operations`, to make sure Grafana recognises and is able to show the timestamp in its panels.
+                     Have a look at the screenshot below for an example.
+  ![grafana-queries.png](../images/grafana-queries.png)
+---
+
+## Open Questions
+
+- How can we specify the data retention period?
+- Is it ok to use the `PROJECT` environment variable as name of the database or should this be configurable (e.g. via 
+  the local settings, an extra environment variable)?
+- Is ms the desired write precision of the timestamp?  Should this be configurable (via the local settings)?

{cgse-0.11.4 → cgse-0.12.0}/docs/libs/cgse-core/registry.md

@@ -44,12 +44,11 @@ A microservice can register as follows:
 from egse.registry.client import AsyncRegistryClient
 
 registry_client = AsyncRegistryClient()
-registry_client.connect()
 
 service_id = await registry_client.register(
     name=service_name,
-    host=hostname,
-    port=port_number,
+    host=get_host_ip(),
+    port=get_port(),
     service_type=service_type,
     metadata={},
     ttl=30,

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/noxfile.py

@@ -1,6 +1,6 @@
 import nox
 
-python_versions = ['3.9', '3.10', '3.11', '3.12', '3.13']
+python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"]
 
 
 @nox.session(python=python_versions)
@@ -20,7 +20,7 @@ def uv_tests(session: nox.Session):
     session.run_install("uv", "venv", *py_version)
 
     session.run_install(
-        *uv_sync_cmd, # "--all-packages",
+        *uv_sync_cmd,  # "--all-packages",
         env={"UV_PROJECT_ENVIRONMENT": session.virtualenv.location},
     )
 

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.11.4"
+version = "0.12.0"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}
@@ -10,7 +10,7 @@ maintainers = [
     {name = "Sara Regibo", email = "sara.regibo@kuleuven.be"}
 ]
 readme = {"file" = "README.md", "content-type" = "text/markdown"}
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = "MIT"
 keywords = [
     "CGSE",

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/src/cgse_common/__init__.py

@@ -4,4 +4,5 @@ from dataclasses import dataclass
 @dataclass
 class AppState:
     """Global Typer app parameters."""
+
     verbose: bool = False

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/src/cgse_common/cgse.py

@@ -19,6 +19,7 @@ the group name 'cgse.command.plugins'.
 Commands can be added as single commands or as a group containing further sub-commands. To add a group,
 the entry point shall contain 'group' in its extras.
 """
+
 import textwrap
 
 import rich
@@ -57,6 +58,7 @@ def broken_command(name: str, module: str, exc: Exception):
 
 # Load the known plugins for the `cgse` command. Plugins are added as commands to the `cgse`.
 
+
 class SortedCommandGroup(TyperGroup):
     """This class sorts the commands based on the following criteria:
 
@@ -64,6 +66,7 @@ class SortedCommandGroup(TyperGroup):
     - the rest of the commands are sorted alphabetically
 
     """
+
     def list_commands(self, ctx):
         # Get list of all commands
         commands = super().list_commands(ctx)
@@ -141,17 +144,14 @@ def build_app():
                 if group == "cgse.service":
                     app.add_typer(ep.load(), name=ep.name)
                 else:
-                    command_group = snake_to_title(group.split('.')[-1])
+                    command_group = snake_to_title(group.split(".")[-1])
                     app.add_typer(ep.load(), name=ep.name, rich_help_panel=command_group)
             except Exception as exc:
                 app.command()(broken_command(ep.name, ep.module, exc))
 
 
 @app.callback(no_args_is_help=True, invoke_without_command=True)
-def main(
-        ctx: typer.Context,
-        verbose: bool = False
-):
+def main(ctx: typer.Context, verbose: bool = False):
     """
     The `cgse` command is used to:
 

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/src/egse/async_control_claude.py

@@ -28,9 +28,11 @@ try:
     # This function is only available when the cgse-core package is installed
     from egse.logger import close_all_zmq_handlers
 except ImportError:
+
     def close_all_zmq_handlers():  # noqa
         pass
 
+
 from egse.process import ProcessStatus
 from egse.settings import Settings
 from egse.system import get_average_execution_time
@@ -390,7 +392,7 @@ class ControlServer(abc.ABC):
             if condition and not condition():
                 self.logger.debug(
                     f"Task {task_name} rescheduled in {self.scheduled_task_delay}s, condition not met...."
-                    )
+                )
                 self.logger.info(f"Task {task_name} rescheduled in {self.scheduled_task_delay}s")
                 current_time = datetime.datetime.now(tz=datetime.timezone.utc)
                 scheduled_time = current_time + datetime.timedelta(seconds=self.scheduled_task_delay)
@@ -621,7 +623,7 @@ class ControlServer(abc.ABC):
                 if not self.device_protocol.is_alive():
                     self.logger.error(
                         "Some Thread or sub-process that was started by Protocol has died, terminating..."
-                        )
+                    )
                     self.quit()
                     break
             except asyncio.CancelledError:
@@ -698,7 +700,7 @@ class ControlServer(abc.ABC):
             asyncio.create_task(self.process_scheduled_tasks()),
             asyncio.create_task(self.update_metrics()),
             asyncio.create_task(self.check_device_protocol_alive()),
-            asyncio.create_task(self.process_sequential_queue())  # Add sequential queue processor
+            asyncio.create_task(self.process_sequential_queue()),  # Add sequential queue processor
         ]
 
         # Add housekeeping task if storage manager is active

{cgse-0.11.4 → cgse-0.12.0}/libs/cgse-common/src/egse/command.py

@@ -90,6 +90,7 @@ Do we need additional hooks into this commanding?
 * provide an execute method for the CommandExecution that executes the command
   with the saved parameters
 """
+
 from __future__ import annotations
 
 import functools
@@ -618,8 +619,7 @@ def get_function(parent_class: type, method_name: str) -> Callable:
 
 
 def load_commands(
-        protocol_class: type[CommandProtocol], command_settings: dict, command_class: type[Command],
-        device_class: type
+    protocol_class: type[CommandProtocol], command_settings: dict, command_class: type[Command], device_class: type
 ) -> dict[str, Command]:
     """
     Loads the command definitions from the given ``command_settings`` and builds an internal