cgse-common 0.16.14__tar.gz → 0.17.1__tar.gz

{cgse_common-0.16.14 → cgse_common-0.17.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse-common
-Version: 0.16.14
+Version: 0.17.1
 Summary: Software framework to support hardware testing
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse_common-0.16.14 → cgse_common-0.17.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.16.14"
+version = "0.17.1"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}
@@ -48,7 +48,6 @@ dependencies = [
 
 [project.scripts]
 cgse = 'cgse_common.cgse:app'
-monitor = "egse.monitoring:app"
 
 [project.entry-points."cgse.version"]
 cgse-common = 'egse.version:get_version_installed'
@@ -61,10 +60,13 @@ pythonpath = "src"
 testpaths = ["tests"]
 addopts = "-ra --cov --cov-branch --cov-report html"
 log_cli = true
-log_cli_level = "INFO"
+log_cli_level = "DEBUG"
 filterwarnings = [
     "ignore::DeprecationWarning"
 ]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')"
+]
 
 [tool.coverage.run]
 omit = [

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/cgse_common/cgse.py

@@ -72,7 +72,7 @@ class SortedCommandGroup(TyperGroup):
         commands = super().list_commands(ctx)
 
         # Define priority commands in specific order
-        priority_commands = ["init", "version", "show", "top", "core", "reg", "not", "log", "cm", "sm", "pm"]
+        priority_commands = ["init", "version", "show", "top", "core", "reg", "log", "nh", "cm", "sm", "pm"]
 
         # Custom sort:
         # First the priority commands in the given order (their index)
@@ -175,6 +175,10 @@ def main(ctx: typer.Context, verbose: bool = False):
     - inspect, configure, monitor the core services and device control servers.
     """
 
+    from egse.env import setup_env
+
+    setup_env()
+
     # This is more of a show-case for using application wide optional arguments and how to pass
     # them into (sub-)commands.
 

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/config.py

@@ -154,13 +154,14 @@ def find_files(pattern: str, root: Path | str, in_dir: str = None) -> Generator[
     Returns:
         Paths of files matching pattern, from root.
     """
-    root = Path(root).resolve()
+    root = Path(root).expanduser().resolve()
     if not root.is_dir():
         root = root.parent
     if not root.exists():
         raise ValueError(f"The root argument didn't resolve into a valid directory: {root}.")
 
-    exclude_dirs = ("venv", "venv38", ".venv", ".nox", ".git", ".idea", ".DS_Store")
+    # FIXME: at some point we might want to make this configurable, through env?
+    exclude_dirs = ("venv", "venv38", ".venv", ".nox", ".git", ".idea", ".DS_Store", "__pycache__")
 
     for path, folders, files in os.walk(root):
         folders[:] = list(filter(lambda x: x not in exclude_dirs, folders))

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/decorators.py

@@ -15,7 +15,6 @@ from typing import Optional
 
 import rich
 
-from egse.settings import Settings
 from egse.system import get_caller_info
 from egse.log import logger
 
@@ -390,6 +389,8 @@ def profile(func):
     if not hasattr(profile, "counter"):
         profile.counter = 0
 
+    from egse.settings import Settings
+
     @functools.wraps(func)
     def wrapper_profile(*args, **kwargs):
         if Settings.profiling():

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/device.py

@@ -199,7 +199,7 @@ class DeviceConnectionInterface(DeviceConnectionObservable):
 
 
 class DeviceInterface(DeviceConnectionInterface):
-    """Generic interface for all device classes."""
+    """A generic interface for all device classes."""
 
     @dynamic_interface
     def is_simulator(self) -> bool:
@@ -237,6 +237,7 @@ class DeviceTransport:
         raise NotImplementedError
 
     def read_string(self, encoding="utf-8") -> str:
+        """Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
         return self.read().decode(encoding).strip()
 
     def trans(self, command: str) -> bytes:
@@ -297,6 +298,11 @@ class AsyncDeviceTransport:
 
         raise NotImplementedError
 
+    async def read_string(self, encoding="utf-8") -> str:
+        """Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
+        b = await self.read()
+        return b.decode(encoding).strip()
+
     async def trans(self, command: str) -> bytes:
         """
         Send a single command to the device controller and block until a response from the
@@ -389,7 +395,7 @@ class AsyncDeviceConnectionInterface(DeviceConnectionObservable):
 
 
 class AsyncDeviceInterface(AsyncDeviceConnectionInterface):
-    """Generic interface for all device classes."""
+    """A generic interface for all device classes."""
 
     def is_simulator(self) -> bool:
         """Checks whether the device is a simulator rather than a real hardware controller.

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/env.py

@@ -43,6 +43,10 @@ from __future__ import annotations
 
 __all__ = [
     "bool_env",
+    "int_env",
+    "str_env",
+    "load_dotenv",
+    "setup_env",
     "env_var",
     "get_conf_data_location",
     "get_conf_data_location_env_name",
@@ -68,14 +72,16 @@ import os
 import warnings
 from pathlib import Path
 
+from dotenv import load_dotenv as _load_dotenv
+from dotenv import find_dotenv
+from rich.console import Console
+
+from egse.decorators import static_vars
 from egse.log import logger
 from egse.system import all_logging_disabled
 from egse.system import get_caller_info
 from egse.system import ignore_m_warning
-
-from rich.console import Console
-
-console = Console(width=100)
+from egse.system import type_name
 
 # Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
 # create the other environment variables that are specific to the project.
@@ -99,7 +105,57 @@ KNOWN_PROJECT_ENVIRONMENT_VARIABLES = [
 ]
 
 
-def initialize():
+def int_env(var_name: str, default: int, *, minimum: int | None = 1) -> int:
+    """Return an integer environment override, falling back to `default` on errors."""
+    raw = os.getenv(var_name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        logger.warning(f"Ignoring invalid integer for {var_name}: {raw!r}, returning {default=}")
+        return default
+    if minimum is not None and value < minimum:
+        logger.warning(f"Ignoring {var_name} because value {value} < {minimum}, returning {default=}")
+        return default
+    return value
+
+
+def bool_env(var_name: str, default: bool = False) -> bool:
+    """Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
+
+    if value := os.getenv(var_name):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+
+    return default
+
+
+def str_env(var_name: str, default: str | None = None) -> str:
+    """Return the value of the environment variable or default if variable is not defined."""
+    return os.getenv(var_name, default)
+
+
+VERBOSE_DEBUG = bool_env("VERBOSE_DEBUG")
+
+
+def load_dotenv():
+    """Overwrites the load_dotenv function for CGSE specifics.
+
+    - set `CGSE_DOTENV_DISABLED=true` to disable loading the `.env` file
+    - the `.env` file is searched for relative to the current working directory,
+      unlike the default, which searches from the script location.
+
+    NOTE:
+    - don't use the `load_dotenv` function from the `dotenv` package
+    - don't use `load_dotenv` in any modules, only in entrypoints and apps.
+    """
+    if not bool_env("CGSE_DOTENV_DISABLED") and (dotenv_location := find_dotenv(usecwd=True)):
+        logger.debug(f"Loading environment variables from {dotenv_location}.")
+        _load_dotenv(dotenv_path=dotenv_location)
+
+
+@static_vars(is_initialized=False)
+def setup_env():
     """
     Initialize the environment variables that are required for the CGSE to function properly.
     This function will print a warning if any of the mandatory environment variables is not set.
@@ -110,6 +166,14 @@ def initialize():
 
     global _env
 
+    if setup_env.is_initialized:
+        return
+
+    if VERBOSE_DEBUG:
+        logger.debug(f"Initialising the environment...")
+
+    load_dotenv()
+
     for name in MANDATORY_ENVIRONMENT_VARIABLES:
         try:
             _env.set(name, os.environ[name])
@@ -123,8 +187,10 @@ def initialize():
     project = _env.get("PROJECT")
 
     for gen_var_name in KNOWN_PROJECT_ENVIRONMENT_VARIABLES:
-        env_var = f"{project}_{gen_var_name}"
-        _env.set(gen_var_name, os.environ.get(env_var, NoValue()))
+        _env_var = f"{project}_{gen_var_name}"
+        _env.set(gen_var_name, os.environ.get(_env_var, NoValue()))
+
+    setup_env.is_initialized = True
 
 
 class _Env:
@@ -136,8 +202,12 @@ class _Env:
     def set(self, key, value):
         if value is None:
             if key in self._env:
+                if VERBOSE_DEBUG:
+                    logger.debug(f"Unsetting environment variable {key}")
                 del self._env[key]
         else:
+            if VERBOSE_DEBUG:
+                logger.debug(f"Setting environment variable {key}={value}")
             self._env[key] = value
 
     def get(self, key) -> str:
@@ -166,11 +236,11 @@ class NoValue:
         return False
 
     def __repr__(self):
-        return f"{self.__class__.__name__}"
+        return f"{type_name(self)}"
 
 
 # The module needs to be initialized before it can be used.
-initialize()
+setup_env()
 
 
 def _check_no_value(var_name, value):
@@ -244,7 +314,7 @@ def set_data_storage_location(location: str | Path | None):
         _env.set("DATA_STORAGE_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = str(location)
@@ -312,7 +382,7 @@ def set_conf_data_location(location: str | Path | None):
         _env.set("CONF_DATA_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -378,7 +448,7 @@ def set_log_file_location(location: str | Path | None):
         _env.set("LOG_FILE_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -423,7 +493,7 @@ def get_log_file_location(site_id: str = None, check_exists: bool = False) -> st
         log_data_root = f"{data_root}/log"
 
     if check_exists:
-        if not Path(log_data_root).is_dir():
+        if not Path(log_data_root).expanduser().is_dir():
             raise ValueError(f"The location that was constructed doesn't exist: {log_data_root}")
 
     return log_data_root
@@ -453,8 +523,11 @@ def set_local_settings(path: str | Path | None):
         _env.set("LOCAL_SETTINGS", None)
         return
 
-    if not Path(path).exists():
-        warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {path}.")
+    if not Path(path).expanduser().is_file():
+        warnings.warn(
+            f"The location you provided for the environment variable {env_name} doesn't exist or is not a "
+            f"regular file: {path}."
+        )
 
     os.environ[env_name] = path
     _env.set("LOCAL_SETTINGS", path)
@@ -481,9 +554,9 @@ def get_local_settings_path() -> str | None:
         )
         return None
 
-    if not Path(local_settings).exists():
+    if not Path(local_settings).expanduser().is_file():
         warnings.warn(
-            f"The local settings path '{local_settings}' doesn't exist. As a result, "
+            f"The local settings path '{local_settings}' doesn't exist or is not a regular file. As a result, "
             f"the local settings for your project will not be loaded."
         )
 
@@ -519,7 +592,7 @@ def get_conf_repo_location() -> str | None:
         )
         return None
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location of the configuration data repository doesn't exist: {location}.")
         return None
 
@@ -545,7 +618,7 @@ def set_conf_repo_location(location: str | Path | None):
         _env.set("CONF_REPO_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -574,20 +647,11 @@ def print_env():
         console.print(f"  {get_local_settings_env_name():{col_width}s}: {get_local_settings_path()}")
 
 
-def bool_env(var_name: str) -> bool:
-    """Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
-
-    if value := os.getenv(var_name):
-        return value.strip().lower() in {"1", "true", "yes", "on"}
-
-    return False
-
-
 @contextlib.contextmanager
 def env_var(**kwargs: str | int | float | bool | None):
     """
-    Context manager to run some code that need alternate settings for environment variables.
-    This will automatically initialize the CGSE environment upon entry and re-initialize upon exit.
+    Context manager to run some code that need alternate settings for environment variables. This will automatically
+    initialize the CGSE environment upon entry and re-initialize when the context manager exits.
 
     Note:
         This context manager is different from the one in `egse.system` because of the CGSE environment changes.
@@ -614,7 +678,8 @@ def env_var(**kwargs: str | int | float | bool | None):
         else:
             os.environ[k] = v
 
-    initialize()
+    setup_env.is_initialized = False
+    setup_env()
 
     yield
 
@@ -625,7 +690,8 @@ def env_var(**kwargs: str | int | float | bool | None):
         else:
             os.environ[k] = v
 
-    initialize()
+    setup_env.is_initialized = False
+    setup_env()
 
 
 def main(args: list | None = None):  # pragma: no cover
@@ -650,32 +716,34 @@ def main(args: list | None = None):  # pragma: no cover
         "--mkdir",
         default=False,
         action="store_true",
-        help="Create directory that doesn't exist.",
+        help="Create any directory that doesn't exist.",
     )
 
     args = parser.parse_args(args or [])
 
-    def check_env_dir(env_var: str):
-        value = _env.get(env_var)
+    setup_env()
+
+    def check_env_dir(_env_var: str):
+        value = _env.get(_env_var)
 
         if value == NoValue():
             value = "[bold red]not set"
-        elif not value.startswith("/"):
+        elif not Path(value).expanduser().is_absolute():
             value = f"[default]{value} [bold orange3](this is a relative path!)"
-        elif not os.path.exists(value):
+        elif not Path(value).expanduser().exists():
             value = f"[default]{value} [bold red](location doesn't exist!)"
-        elif not os.path.isdir(value):
+        elif not Path(value).expanduser().is_dir():
             value = f"[default]{value} [bold red](location is not a directory!)"
         else:
             value = f"[default]{value}"
         return value
 
-    def check_env_file(env_var: str):
-        value = _env.get(env_var)
+    def check_env_file(_env_var: str):
+        value = _env.get(_env_var)
 
         if not value:
             value = "[bold red]not set"
-        elif not os.path.exists(value):
+        elif not Path(value).expanduser().exists():
             value = f"[default]{value} [bold red](location doesn't exist!)"
         else:
             value = f"[default]{value}"
@@ -701,7 +769,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_data_storage_location() = }", flush=True, end="")
             location = get_data_storage_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating data storage location: {location} (+ daily + obs)[/]")
                     Path(location).mkdir(parents=True)
@@ -717,7 +785,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_conf_data_location() = }", flush=True, end="")
             location = get_conf_data_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating configuration data location: {location}[/]")
                     Path(location).mkdir(parents=True)
@@ -731,7 +799,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_conf_repo_location() = }", flush=True, end="")
             location = get_conf_repo_location()
-            if location is None or not Path(location).exists():
+            if location is None or not Path(location).expanduser().exists():
                 rich.print("  [red]⟶ ERROR: The configuration repository location doesn't exist![/]")
             else:
                 rich.print()
@@ -741,7 +809,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_log_file_location() = }", flush=True, end="")
             location = get_log_file_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating log files location: {location}[/]")
                     Path(location).mkdir(parents=True)
@@ -755,7 +823,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_local_settings_path() = }", flush=True, end="")
             location = get_local_settings_path()
-            if location is None or not Path(location).exists():
+            if location is None or not Path(location).expanduser().exists():
                 rich.print("  [red]⟶ ERROR: The local settings file is not defined or doesn't exist![/]")
             else:
                 rich.print()
@@ -782,8 +850,8 @@ def main(args: list | None = None):  # pragma: no cover
     of the name. By default, this directory is located in the overall data storage folder.
 
 [bold]PROJECT_CONF_REPO_LOCATION[/bold]:
-    This variable is the root of the working copy of the 'plato-cgse-conf' project.
-    The value is usually set to `~/git/plato-cgse-conf`.
+    This variable is the root of the working copy of the repo for configuration files.
+    The value is usually set to `~/git/{project}-conf`.
 
 [bold]PROJECT_DATA_STORAGE_LOCATION[/bold]:
     This directory contains all the data files from the control servers and other
@@ -805,7 +873,7 @@ def main(args: list | None = None):  # pragma: no cover
 [bold]PROJECT_LOCAL_SETTINGS[/bold]:
     This file is used for local site-specific settings. When the environment
     variable is not set, no local settings will be loaded. By default, this variable
-    is assumed to be '/cgse/local_settings.yaml'.
+    assumes to be '/cgse/local_settings.yaml'.
 """
 
     if args.doc:

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/heartbeat.py

@@ -80,7 +80,7 @@ if __name__ == "__main__":
     broadcaster.start()
 
     # You will see that the timestamp of this custom message is drifting because
-    # we do not take the execution time of the `set_message()` etc. intyo accout.
+    # we do not take the execution time of the `set_message()` etc. into account.
     # In the actual HeartbeatBroadcaster above, we do take that into account.
 
     try:

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/log.py

@@ -135,20 +135,27 @@ for handler in root_logger.handlers:
 logger = egse_logger
 
 if __name__ == "__main__":
+    from egse.env import str_env
+
     root_logger = logging.getLogger()
 
-    rich.print(
+    print(
         textwrap.dedent(
             """
+            Environment variables:
+              - LOG_LEVEL=debug|info|warning|critical
+              - LOG_FORMAT=full|clean
+            
             Example logging statements
               - logging level set to INFO
+              - logging format set to full
               - fields are separated by a colon ':'
               - fields: date & time: process name : level : logger name : lineno : filename : message
             """
         )
     )
 
-    if os.getenv("LOG_FORMAT_FULL") == "true":
+    if str_env("LOG_FORMAT", "clean").strip().lower() == "full":
         rich.print(
             f"[b]{'Date & Time':^23s} : {'Process Name':20s} : {'Level':8s} : {'Logger Name':^25s} : {' Line '} : "
             f"{'Filename':20s} : {'Message'}[/]"

{cgse_common-0.16.14 → cgse_common-0.17.1}/src/egse/plugins/metrics/influxdb.py

@@ -37,11 +37,14 @@ import pandas
 import pyarrow
 from influxdb_client_3 import InfluxDBClient3
 from influxdb_client_3 import Point
-from influxdb_client_3 import SYNCHRONOUS
+from influxdb_client_3 import WriteType
 from influxdb_client_3 import write_client_options
 from influxdb_client_3.exceptions import InfluxDB3ClientError
 from influxdb_client_3.write_client.domain.write_precision import WritePrecision
 
+from egse.env import bool_env
+from egse.env import int_env
+from egse.env import str_env
 from egse.metrics import TimeSeriesRepository
 from egse.system import type_name
 
@@ -63,9 +66,38 @@ class InfluxDBRepository(TimeSeriesRepository):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    def _load_client_options(self):
+        self._batch_size = int_env("CGSE_INFLUX_BATCH_SIZE", 1_000)
+        self._flush_interval = int_env("CGSE_INFLUX_FLUSH_MS", 1_000)
+        self._retry_interval = int_env("CGSE_INFLUX_RETRY_MS", 3_000)
+        self._max_retry_delay = int_env("CGSE_INFLUX_RETRY_MAX_DELAY_MS", 3_000)
+        self._max_retry_time = int_env("CGSE_INFLUX_RETRY_MAX_TIME_MS", 6_000, minimum=0)
+        self._max_retries = int_env("CGSE_INFLUX_MAX_RETRIES", 5, minimum=0)
+        self._no_sync = bool_env("CGSE_INFLUX_NO_SYNC", default=True)
+        self._write_type_env = str_env("CGSE_INFLUX_WRITE_TYPE")
+        self._write_type = WriteType.asynchronous
+        if self._write_type_env:
+            match self._write_type_env.strip().lower():
+                case "batch":
+                    self._write_type = WriteType.batching
+                case "async":
+                    self._write_type = WriteType.asynchronous
+                case "sync":
+                    self._write_type = WriteType.synchronous
+
     def connect(self):
+        self._load_client_options()
+
         wco = write_client_options(
-            write_options=SYNCHRONOUS, write_precision=self.metrics_time_precision, flush_interval=200
+            write_type=self._write_type,
+            batch_size=self._batch_size,
+            flush_interval=self._flush_interval,
+            retry_interval=self._retry_interval,
+            max_retries=self._max_retries,
+            max_retry_delay=self._max_retry_delay,
+            max_retry_time=self._max_retry_time,
+            no_sync=self._no_sync,
+            write_precision=self.metrics_time_precision,
         )
         self.client = InfluxDBClient3(host=self.host, database=self.database, token=self.token, write_options=wco)