cgse-common 0.16.13__py3-none-any.whl → 0.17.0__py3-none-any.whl

cgse_common/cgse.py

@@ -72,7 +72,7 @@ class SortedCommandGroup(TyperGroup):
         commands = super().list_commands(ctx)
 
         # Define priority commands in specific order
-        priority_commands = ["init", "version", "show", "top", "core", "reg", "not", "log", "cm", "sm", "pm"]
+        priority_commands = ["init", "version", "show", "top", "core", "reg", "log", "nh", "cm", "sm", "pm"]
 
         # Custom sort:
         # First the priority commands in the given order (their index)
@@ -175,6 +175,10 @@ def main(ctx: typer.Context, verbose: bool = False):
     - inspect, configure, monitor the core services and device control servers.
     """
 
+    from egse.env import setup_env
+
+    setup_env()
+
     # This is more of a show-case for using application wide optional arguments and how to pass
     # them into (sub-)commands.
 

{cgse_common-0.16.13.dist-info → cgse_common-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse-common
-Version: 0.16.13
+Version: 0.17.0
 Summary: Software framework to support hardware testing
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse_common-0.16.13.dist-info → cgse_common-0.17.0.dist-info}/RECORD

@@ -1,18 +1,18 @@
 cgse_common/__init__.py,sha256=wTYOpVomEeDFFuqt4Ss9ROSAIa48UUnYCSafdEOx-CU,129
-cgse_common/cgse.py,sha256=4ZCm5UfaXld0RGI-Mjnm6dSAqfWCPX1qz1pew46J-1s,6590
+cgse_common/cgse.py,sha256=S8MxwJcSzqYNR0wirEscQw5CTg89AKORa8-NHRdrxJM,6642
 cgse_common/settings.yaml,sha256=PS8HOoxbhwVoQ7zzDtZyhx25RZB6SGq3uXNuJTgtRIw,443
 egse/bits.py,sha256=cg6diLN2IwNtnrlZyLgXPfDf9ttcHzS2A7Tjwlwc-lQ,12498
 egse/calibration.py,sha256=a5JDaXTC6fMwQ1M-qrwNO31Ass-yYSXxDQUK_PPsZg4,8818
 egse/config.py,sha256=qNW3uvwuEV9VSjiaoQfM_svBYfz4Kwxd1-jv1eTGqyw,9530
 egse/counter.py,sha256=7UwBeTAu213xdNdGAOYpUWNQ4jD4yVM1bOG10Ax4UFs,5097
-egse/decorators.py,sha256=B-zRa1WdLO71zqS5M27JBglcThYPho7seYfa4HOGj5c,27171
-egse/device.py,sha256=nn2HkN1KIHAmo37WZcqig-p2mQz1LgqpIfj1wPrUTLc,13240
+egse/decorators.py,sha256=RT_-TmcHLN3oqha-1xn-YbqT1tEtKA84zr0Qu1KFyKo,27176
+egse/device.py,sha256=Ga5MNE96ERuPLDzmb0HF7UfjrTqfp0f_r3NDVZY9l28,13595
 egse/dicts.py,sha256=dUAq7PTPvs73OrZb2Fh3loxvYv4ifUiK6bBcgrFU77Y,3972
-egse/env.py,sha256=0z57ESbD9qtJSQiK_7CcpEZm5sxwrKkOMQaJGANmTho,29090
+egse/env.py,sha256=iJV4gKCd1wFWDMsJbdjtgZJSE4Gr8hBSD4YLTN8xKfM,31630
 egse/exceptions.py,sha256=QB3MZRJizecWOj1cPbvG0UcIqFn7NRJ6rw1xtdNSFxw,1225
-egse/heartbeat.py,sha256=xt5mePu9Zr9fLAhN1MLq1Z7aCOKtNIhRVCAmWhtNwP8,3039
+egse/heartbeat.py,sha256=2SeZzX3tWFog1rgYThX-iaZPwHYq8TVma2ll7r624Eg,3039
 egse/hk.py,sha256=AumSpB8SYXes75CB2iiKXfLkMK5IkVDHITFKrf8IT6g,32010
-egse/log.py,sha256=EJSlv_hueSnN1acy12Js_ywnARG_b49j97keiQnF3MY,5021
+egse/log.py,sha256=Q_TviwuyMam6CwRklaJk1lZM5WIZK_L9GS7wWDy_naM,5251
 egse/metrics.py,sha256=2hHtJXG0Rn782l2bfmLNBbw6ucC5nf7jPnNzqbhP_Zs,7012
 egse/observer.py,sha256=xQ7F7NVHqdRZ6IIsBM5M0kMuullMghoR98dwAsjgh0s,1287
 egse/obsid.py,sha256=y87AYX5mtNEBqEtpEFEec2MhEmo1Hej3Wwi5od84wR8,5848
@@ -25,21 +25,21 @@ egse/ratelimit.py,sha256=JdJxD6UIi9LYngKEsG9zh8bTE9r_56D4EZCnp_fkrI0,9161
 egse/reload.py,sha256=PzOE0m1tmcNcQPVFH8orMe_cMoQIIiH9Gw2anpQTC40,4717
 egse/resource.py,sha256=kzNI6kJOE6Jd5QKJs2MkVAycUpwpOTLi1qydh3NSRng,15345
 egse/response.py,sha256=F04uqOYv1ClpHgDLYZlKTuOCSldHs5TezI_4x6zf2Fw,2717
-egse/scpi.py,sha256=WJ73EaLgRUV6ah1V41l0L7AXI-Dc6Jct7hPHlbbCIcg,15461
-egse/settings.py,sha256=YrRsMUn_IpOVnhTqUGREQUjMw8-AQ6aUBulQiij9MwY,15486
+egse/scpi.py,sha256=geIHxoolItjPO1LUFVNzL1c9QxpmxcyVwEV9BIi2ftI,15722
+egse/settings.py,sha256=s89xqvS31MOBq7p1dTOj41ODnZnBNB35nHqfknJpqcc,15466
 egse/settings.yaml,sha256=mz9O2QqmiptezsMvxJRLhnC1ROwIHENX0nbnhMaXUpE,190
-egse/setup.py,sha256=iWFe3zx4bIkzaUabpbxedteG8qEc3JYHgpu2HjU7s4E,33973
+egse/setup.py,sha256=ezPYA3n1P3navdPR3qDxh0qJvZCzGl2bIREEP9n2w3Y,34116
 egse/signal.py,sha256=f5pyOiNW9iTSIxV_ce5stIfG0ub9MRbaekE85kQOVzs,7992
-egse/socketdevice.py,sha256=R8XwYHTH3lFhFngfsGbi_L7bTnTLHxMTEKIF7gmm5rc,7465
+egse/socketdevice.py,sha256=rsLJRm4AQhfO3VcM16Q9qwtPbKI-iynVlZqKZ3tc3Fk,14636
 egse/state.py,sha256=HdU2MFOlYRbawYRZmizV6Y8MgnZrUF0bx4fXaYU-M_s,3023
 egse/system.py,sha256=3aycoK15UvNdk2yxQ8hx4N-s2Ag4BqTTPQ8eFuJa3Ko,76111
 egse/task.py,sha256=ODSLE05f31CgWsSVcVFFq1WYUZrJMb1LioPTx6VM824,2804
 egse/version.py,sha256=e9GvelUZ9mfCDlRju4MWEJeMHJW9kUzK6SKzJpyj91s,6156
 egse/zmq_ser.py,sha256=YJFupsxuvhI8TJMeS2Hem9oMMcVmSBx0rZv93gvN-hA,3263
 egse/plugins/metrics/duckdb.py,sha256=E2eeNo3I7ajRuByodaYiPNvC0Zwyc7hsIlhr1W_eXdo,16148
-egse/plugins/metrics/influxdb.py,sha256=ecxjA_csYwf8RW3sXjiQxZHREfyrfStH1HA_rAs1AA8,6690
+egse/plugins/metrics/influxdb.py,sha256=WnAqTWRkAyMSd7W2ASwUAIEwFborrv55iX-umceevFA,8162
 egse/plugins/metrics/timescaledb.py,sha256=Ug0NWDV1Ky2VeFY6tDZL9xg6AFgnAEh2F_llVPnlRBA,21191
-cgse_common-0.16.13.dist-info/METADATA,sha256=FEUo16l14WCirVck9cdGRQb3EqHlPJYDkoMjInflkNA,3069
-cgse_common-0.16.13.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cgse_common-0.16.13.dist-info/entry_points.txt,sha256=erQovXd1bGzsngB0_sfY7IYRNwHIhwq3K8fmQvGS12o,198
-cgse_common-0.16.13.dist-info/RECORD,,
+cgse_common-0.17.0.dist-info/METADATA,sha256=YIF0aWp-_8lG-Js0EVRZHUqVVVqG5uqspV2Nfsgwsew,3068
+cgse_common-0.17.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cgse_common-0.17.0.dist-info/entry_points.txt,sha256=xJsPRIDjtADVgd_oEDHVW10wS5LG30Ox_3brVKeyCGw,168
+cgse_common-0.17.0.dist-info/RECORD,,

{cgse_common-0.16.13.dist-info → cgse_common-0.17.0.dist-info}/entry_points.txt

@@ -1,6 +1,5 @@
 [console_scripts]
 cgse = cgse_common.cgse:app
-monitor = egse.monitoring:app
 
 [cgse.settings]
 cgse-common = cgse_common:settings.yaml

egse/decorators.py

@@ -15,7 +15,6 @@ from typing import Optional
 
 import rich
 
-from egse.settings import Settings
 from egse.system import get_caller_info
 from egse.log import logger
 
@@ -390,6 +389,8 @@ def profile(func):
     if not hasattr(profile, "counter"):
         profile.counter = 0
 
+    from egse.settings import Settings
+
     @functools.wraps(func)
     def wrapper_profile(*args, **kwargs):
         if Settings.profiling():

egse/device.py

@@ -199,7 +199,7 @@ class DeviceConnectionInterface(DeviceConnectionObservable):
 
 
 class DeviceInterface(DeviceConnectionInterface):
-    """Generic interface for all device classes."""
+    """A generic interface for all device classes."""
 
     @dynamic_interface
     def is_simulator(self) -> bool:
@@ -237,6 +237,7 @@ class DeviceTransport:
         raise NotImplementedError
 
     def read_string(self, encoding="utf-8") -> str:
+        """Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
         return self.read().decode(encoding).strip()
 
     def trans(self, command: str) -> bytes:
@@ -297,6 +298,11 @@ class AsyncDeviceTransport:
 
         raise NotImplementedError
 
+    async def read_string(self, encoding="utf-8") -> str:
+        """Reads a bytes object from the instrument and returns it converted into a stripped UTF-8 string."""
+        b = await self.read()
+        return b.decode(encoding).strip()
+
     async def trans(self, command: str) -> bytes:
         """
         Send a single command to the device controller and block until a response from the
@@ -389,7 +395,7 @@ class AsyncDeviceConnectionInterface(DeviceConnectionObservable):
 
 
 class AsyncDeviceInterface(AsyncDeviceConnectionInterface):
-    """Generic interface for all device classes."""
+    """A generic interface for all device classes."""
 
     def is_simulator(self) -> bool:
         """Checks whether the device is a simulator rather than a real hardware controller.

egse/env.py

@@ -43,6 +43,10 @@ from __future__ import annotations
 
 __all__ = [
     "bool_env",
+    "int_env",
+    "str_env",
+    "load_dotenv",
+    "setup_env",
     "env_var",
     "get_conf_data_location",
     "get_conf_data_location_env_name",
@@ -68,14 +72,16 @@ import os
 import warnings
 from pathlib import Path
 
+from dotenv import load_dotenv as _load_dotenv
+from dotenv import find_dotenv
+from rich.console import Console
+
+from egse.decorators import static_vars
 from egse.log import logger
 from egse.system import all_logging_disabled
 from egse.system import get_caller_info
 from egse.system import ignore_m_warning
-
-from rich.console import Console
-
-console = Console(width=100)
+from egse.system import type_name
 
 # Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
 # create the other environment variables that are specific to the project.
@@ -99,7 +105,57 @@ KNOWN_PROJECT_ENVIRONMENT_VARIABLES = [
 ]
 
 
-def initialize():
+def int_env(var_name: str, default: int, *, minimum: int | None = 1) -> int:
+    """Return an integer environment override, falling back to `default` on errors."""
+    raw = os.getenv(var_name)
+    if raw is None:
+        return default
+    try:
+        value = int(raw)
+    except ValueError:
+        logger.warning(f"Ignoring invalid integer for {var_name}: {raw!r}, returning {default=}")
+        return default
+    if minimum is not None and value < minimum:
+        logger.warning(f"Ignoring {var_name} because value {value} < {minimum}, returning {default=}")
+        return default
+    return value
+
+
+def bool_env(var_name: str, default: bool = False) -> bool:
+    """Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
+
+    if value := os.getenv(var_name):
+        return value.strip().lower() in {"1", "true", "yes", "on"}
+
+    return default
+
+
+def str_env(var_name: str, default: str | None = None) -> str:
+    """Return the value of the environment variable or default if variable is not defined."""
+    return os.getenv(var_name, default)
+
+
+VERBOSE_DEBUG = bool_env("VERBOSE_DEBUG")
+
+
+def load_dotenv():
+    """Overwrites the load_dotenv function for CGSE specifics.
+
+    - set `CGSE_DOTENV_DISABLED=true` to disable loading the `.env` file
+    - the `.env` file is searched for relative to the current working directory,
+      unlike the default, which searches from the script location.
+
+    NOTE:
+    - don't use the `load_dotenv` function from the `dotenv` package
+    - don't use `load_dotenv` in any modules, only in entrypoints and apps.
+    """
+    if not bool_env("CGSE_DOTENV_DISABLED") and (dotenv_location := find_dotenv(usecwd=True)):
+        logger.debug(f"Loading environment variables from {dotenv_location}.")
+        _load_dotenv(dotenv_path=dotenv_location)
+
+
+@static_vars(is_initialized=False)
+def setup_env():
     """
     Initialize the environment variables that are required for the CGSE to function properly.
     This function will print a warning if any of the mandatory environment variables is not set.
@@ -110,6 +166,14 @@ def initialize():
 
     global _env
 
+    if setup_env.is_initialized:
+        return
+
+    if VERBOSE_DEBUG:
+        logger.debug(f"Initialising the environment...")
+
+    load_dotenv()
+
     for name in MANDATORY_ENVIRONMENT_VARIABLES:
         try:
             _env.set(name, os.environ[name])
@@ -123,8 +187,10 @@ def initialize():
     project = _env.get("PROJECT")
 
     for gen_var_name in KNOWN_PROJECT_ENVIRONMENT_VARIABLES:
-        env_var = f"{project}_{gen_var_name}"
-        _env.set(gen_var_name, os.environ.get(env_var, NoValue()))
+        _env_var = f"{project}_{gen_var_name}"
+        _env.set(gen_var_name, os.environ.get(_env_var, NoValue()))
+
+    setup_env.is_initialized = True
 
 
 class _Env:
@@ -136,8 +202,12 @@ class _Env:
     def set(self, key, value):
         if value is None:
             if key in self._env:
+                if VERBOSE_DEBUG:
+                    logger.debug(f"Unsetting environment variable {key}")
                 del self._env[key]
         else:
+            if VERBOSE_DEBUG:
+                logger.debug(f"Setting environment variable {key}={value}")
             self._env[key] = value
 
     def get(self, key) -> str:
@@ -166,11 +236,11 @@ class NoValue:
         return False
 
     def __repr__(self):
-        return f"{self.__class__.__name__}"
+        return f"{type_name(self)}"
 
 
 # The module needs to be initialized before it can be used.
-initialize()
+setup_env()
 
 
 def _check_no_value(var_name, value):
@@ -244,7 +314,7 @@ def set_data_storage_location(location: str | Path | None):
         _env.set("DATA_STORAGE_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = str(location)
@@ -312,7 +382,7 @@ def set_conf_data_location(location: str | Path | None):
         _env.set("CONF_DATA_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -378,7 +448,7 @@ def set_log_file_location(location: str | Path | None):
         _env.set("LOG_FILE_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -423,7 +493,7 @@ def get_log_file_location(site_id: str = None, check_exists: bool = False) -> st
         log_data_root = f"{data_root}/log"
 
     if check_exists:
-        if not Path(log_data_root).is_dir():
+        if not Path(log_data_root).expanduser().is_dir():
             raise ValueError(f"The location that was constructed doesn't exist: {log_data_root}")
 
     return log_data_root
@@ -453,8 +523,11 @@ def set_local_settings(path: str | Path | None):
         _env.set("LOCAL_SETTINGS", None)
         return
 
-    if not Path(path).exists():
-        warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {path}.")
+    if not Path(path).expanduser().is_file():
+        warnings.warn(
+            f"The location you provided for the environment variable {env_name} doesn't exist or is not a "
+            f"regular file: {path}."
+        )
 
     os.environ[env_name] = path
     _env.set("LOCAL_SETTINGS", path)
@@ -481,9 +554,9 @@ def get_local_settings_path() -> str | None:
         )
         return None
 
-    if not Path(local_settings).exists():
+    if not Path(local_settings).expanduser().is_file():
         warnings.warn(
-            f"The local settings path '{local_settings}' doesn't exist. As a result, "
+            f"The local settings path '{local_settings}' doesn't exist or is not a regular file. As a result, "
             f"the local settings for your project will not be loaded."
         )
 
@@ -519,7 +592,7 @@ def get_conf_repo_location() -> str | None:
         )
         return None
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location of the configuration data repository doesn't exist: {location}.")
         return None
 
@@ -545,7 +618,7 @@ def set_conf_repo_location(location: str | Path | None):
         _env.set("CONF_REPO_LOCATION", None)
         return
 
-    if not Path(location).exists():
+    if not Path(location).expanduser().exists():
         warnings.warn(f"The location you provided for the environment variable {env_name} doesn't exist: {location}.")
 
     os.environ[env_name] = location
@@ -574,20 +647,11 @@ def print_env():
         console.print(f"  {get_local_settings_env_name():{col_width}s}: {get_local_settings_path()}")
 
 
-def bool_env(var_name: str) -> bool:
-    """Return True if the environment variable is set to 1, true, yes, or on. All case-insensitive."""
-
-    if value := os.getenv(var_name):
-        return value.strip().lower() in {"1", "true", "yes", "on"}
-
-    return False
-
-
 @contextlib.contextmanager
 def env_var(**kwargs: str | int | float | bool | None):
     """
-    Context manager to run some code that need alternate settings for environment variables.
-    This will automatically initialize the CGSE environment upon entry and re-initialize upon exit.
+    Context manager to run some code that need alternate settings for environment variables. This will automatically
+    initialize the CGSE environment upon entry and re-initialize when the context manager exits.
 
     Note:
         This context manager is different from the one in `egse.system` because of the CGSE environment changes.
@@ -614,7 +678,7 @@ def env_var(**kwargs: str | int | float | bool | None):
         else:
             os.environ[k] = v
 
-    initialize()
+    setup_env()
 
     yield
 
@@ -625,7 +689,7 @@ def env_var(**kwargs: str | int | float | bool | None):
         else:
             os.environ[k] = v
 
-    initialize()
+    setup_env()
 
 
 def main(args: list | None = None):  # pragma: no cover
@@ -650,32 +714,34 @@ def main(args: list | None = None):  # pragma: no cover
         "--mkdir",
         default=False,
         action="store_true",
-        help="Create directory that doesn't exist.",
+        help="Create any directory that doesn't exist.",
     )
 
     args = parser.parse_args(args or [])
 
-    def check_env_dir(env_var: str):
-        value = _env.get(env_var)
+    setup_env()
+
+    def check_env_dir(_env_var: str):
+        value = _env.get(_env_var)
 
         if value == NoValue():
             value = "[bold red]not set"
-        elif not value.startswith("/"):
+        elif not Path(value).expanduser().is_absolute():
             value = f"[default]{value} [bold orange3](this is a relative path!)"
-        elif not os.path.exists(value):
+        elif not Path(value).expanduser().exists():
             value = f"[default]{value} [bold red](location doesn't exist!)"
-        elif not os.path.isdir(value):
+        elif not Path(value).expanduser().is_dir():
             value = f"[default]{value} [bold red](location is not a directory!)"
         else:
             value = f"[default]{value}"
         return value
 
-    def check_env_file(env_var: str):
-        value = _env.get(env_var)
+    def check_env_file(_env_var: str):
+        value = _env.get(_env_var)
 
         if not value:
             value = "[bold red]not set"
-        elif not os.path.exists(value):
+        elif not Path(value).expanduser().exists():
             value = f"[default]{value} [bold red](location doesn't exist!)"
         else:
             value = f"[default]{value}"
@@ -701,7 +767,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_data_storage_location() = }", flush=True, end="")
             location = get_data_storage_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating data storage location: {location} (+ daily + obs)[/]")
                     Path(location).mkdir(parents=True)
@@ -717,7 +783,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_conf_data_location() = }", flush=True, end="")
             location = get_conf_data_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating configuration data location: {location}[/]")
                     Path(location).mkdir(parents=True)
@@ -731,7 +797,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_conf_repo_location() = }", flush=True, end="")
             location = get_conf_repo_location()
-            if location is None or not Path(location).exists():
+            if location is None or not Path(location).expanduser().exists():
                 rich.print("  [red]⟶ ERROR: The configuration repository location doesn't exist![/]")
             else:
                 rich.print()
@@ -741,7 +807,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_log_file_location() = }", flush=True, end="")
             location = get_log_file_location()
-            if not Path(location).exists():
+            if not Path(location).expanduser().exists():
                 if args.mkdir:
                     rich.print(f"  [green]⟶ Creating log files location: {location}[/]")
                     Path(location).mkdir(parents=True)
@@ -755,7 +821,7 @@ def main(args: list | None = None):  # pragma: no cover
         try:
             rich.print(f"    {get_local_settings_path() = }", flush=True, end="")
             location = get_local_settings_path()
-            if location is None or not Path(location).exists():
+            if location is None or not Path(location).expanduser().exists():
                 rich.print("  [red]⟶ ERROR: The local settings file is not defined or doesn't exist![/]")
             else:
                 rich.print()
@@ -782,8 +848,8 @@ def main(args: list | None = None):  # pragma: no cover
     of the name. By default, this directory is located in the overall data storage folder.
 
 [bold]PROJECT_CONF_REPO_LOCATION[/bold]:
-    This variable is the root of the working copy of the 'plato-cgse-conf' project.
-    The value is usually set to `~/git/plato-cgse-conf`.
+    This variable is the root of the working copy of the repo for configuration files.
+    The value is usually set to `~/git/{project}-conf`.
 
 [bold]PROJECT_DATA_STORAGE_LOCATION[/bold]:
     This directory contains all the data files from the control servers and other
@@ -805,7 +871,7 @@ def main(args: list | None = None):  # pragma: no cover
 [bold]PROJECT_LOCAL_SETTINGS[/bold]:
     This file is used for local site-specific settings. When the environment
     variable is not set, no local settings will be loaded. By default, this variable
-    is assumed to be '/cgse/local_settings.yaml'.
+    assumes to be '/cgse/local_settings.yaml'.
 """
 
     if args.doc:

egse/heartbeat.py

@@ -80,7 +80,7 @@ if __name__ == "__main__":
     broadcaster.start()
 
     # You will see that the timestamp of this custom message is drifting because
-    # we do not take the execution time of the `set_message()` etc. intyo accout.
+    # we do not take the execution time of the `set_message()` etc. into account.
     # In the actual HeartbeatBroadcaster above, we do take that into account.
 
     try:

egse/log.py

@@ -135,20 +135,27 @@ for handler in root_logger.handlers:
 logger = egse_logger
 
 if __name__ == "__main__":
+    from egse.env import str_env
+
     root_logger = logging.getLogger()
 
-    rich.print(
+    print(
         textwrap.dedent(
             """
+            Environment variables:
+              - LOG_LEVEL=debug|info|warning|critical
+              - LOG_FORMAT=full|clean
+            
             Example logging statements
               - logging level set to INFO
+              - logging format set to full
               - fields are separated by a colon ':'
               - fields: date & time: process name : level : logger name : lineno : filename : message
             """
         )
     )
 
-    if os.getenv("LOG_FORMAT_FULL") == "true":
+    if str_env("LOG_FORMAT", "clean").strip().lower() == "full":
         rich.print(
             f"[b]{'Date & Time':^23s} : {'Process Name':20s} : {'Level':8s} : {'Logger Name':^25s} : {' Line '} : "
             f"{'Filename':20s} : {'Message'}[/]"

egse/plugins/metrics/influxdb.py

@@ -37,11 +37,14 @@ import pandas
 import pyarrow
 from influxdb_client_3 import InfluxDBClient3
 from influxdb_client_3 import Point
-from influxdb_client_3 import SYNCHRONOUS
+from influxdb_client_3 import WriteType
 from influxdb_client_3 import write_client_options
 from influxdb_client_3.exceptions import InfluxDB3ClientError
 from influxdb_client_3.write_client.domain.write_precision import WritePrecision
 
+from egse.env import bool_env
+from egse.env import int_env
+from egse.env import str_env
 from egse.metrics import TimeSeriesRepository
 from egse.system import type_name
 
@@ -63,9 +66,38 @@ class InfluxDBRepository(TimeSeriesRepository):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    def _load_client_options(self):
+        self._batch_size = int_env("CGSE_INFLUX_BATCH_SIZE", 1_000)
+        self._flush_interval = int_env("CGSE_INFLUX_FLUSH_MS", 1_000)
+        self._retry_interval = int_env("CGSE_INFLUX_RETRY_MS", 3_000)
+        self._max_retry_delay = int_env("CGSE_INFLUX_RETRY_MAX_DELAY_MS", 3_000)
+        self._max_retry_time = int_env("CGSE_INFLUX_RETRY_MAX_TIME_MS", 6_000, minimum=0)
+        self._max_retries = int_env("CGSE_INFLUX_MAX_RETRIES", 5, minimum=0)
+        self._no_sync = bool_env("CGSE_INFLUX_NO_SYNC", default=True)
+        self._write_type_env = str_env("CGSE_INFLUX_WRITE_TYPE")
+        self._write_type = WriteType.asynchronous
+        if self._write_type_env:
+            match self._write_type_env.strip().lower():
+                case "batch":
+                    self._write_type = WriteType.batching
+                case "async":
+                    self._write_type = WriteType.asynchronous
+                case "sync":
+                    self._write_type = WriteType.synchronous
+
     def connect(self):
+        self._load_client_options()
+
         wco = write_client_options(
-            write_options=SYNCHRONOUS, write_precision=self.metrics_time_precision, flush_interval=200
+            write_type=self._write_type,
+            batch_size=self._batch_size,
+            flush_interval=self._flush_interval,
+            retry_interval=self._retry_interval,
+            max_retries=self._max_retries,
+            max_retry_delay=self._max_retry_delay,
+            max_retry_time=self._max_retry_time,
+            no_sync=self._no_sync,
+            write_precision=self.metrics_time_precision,
         )
         self.client = InfluxDBClient3(host=self.host, database=self.database, token=self.token, write_options=wco)
 

egse/scpi.py

@@ -77,7 +77,7 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
     def is_simulator(self) -> bool:
         return False
 
-    async def initialize(self, commands: list[tuple[str, bool]] = None, reset_device: bool = False):
+    async def initialize(self, commands: list[tuple[str, bool]] = None, reset_device: bool = False) -> list[str | None]:
         """Initialize the device with optional reset and command sequence.
 
         Performs device initialization by optionally resetting the device and then
@@ -92,21 +92,25 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
                 the command sequence. Defaults to False.
 
         Returns:
-            None
+           Response for each of the commands, or None when no response was expected.
 
         Raises:
             Any exceptions raised by the underlying write() or trans() methods,
             typically communication errors or device timeouts.
 
         Example:
-            >>> await device.initialize([
-            ...     ("*IDN?", True),           # Query device ID, expect response
-            ...     ("SYST:ERR?", True),       # Check for errors, expect response
-            ...     ("OUTP ON", False)         # Enable output, no response expected
-            ... ], reset_device=True)
+            await device.initialize(
+                [
+                    ("*IDN?", True),           # Query device ID, expect response
+                    ("SYST:ERR?", True),       # Check for errors, expect response
+                    ("OUTP ON", False),        # Enable output, no response expected
+                ],
+                reset_device=True
+            )
         """
 
         commands = commands or []
+        responses = []
 
         if reset_device:
             logger.info(f"Resetting the {self.device_name}...")
@@ -116,10 +120,14 @@ class AsyncSCPIInterface(AsyncDeviceInterface, AsyncDeviceTransport):
             if expects_response:
                 logger.debug(f"Sending {cmd}...")
                 response = (await self.trans(cmd)).decode().strip()
+                responses.append(response)
                 logger.debug(f"{response = }")
             else:
                 logger.debug(f"Sending {cmd}...")
                 await self.write(cmd)
+                responses.append(None)
+
+        return responses
 
     async def connect(self) -> None:
         """Connect to the device asynchronously.

egse/settings.py

@@ -66,9 +66,6 @@ The above code will read the YAML file from the given location and not from the
 
 """
 
-from __future__ import annotations
-
-import logging
 import re
 from pathlib import Path
 from typing import Any
@@ -197,7 +194,7 @@ def load_local_settings(force: bool = False) -> attrdict:
     local_settings_path = get_local_settings_path()
 
     if local_settings_path:
-        path = Path(local_settings_path)
+        path = Path(local_settings_path).expanduser()
         local_settings = load_settings_file(path.parent, path.name, force)
 
     return local_settings
@@ -386,7 +383,9 @@ def main(args: list | None = None):  # pragma: no cover
     #
     # Use the '--help' option to see what your choices are.
 
-    logging.basicConfig(level=20)
+    from egse.env import setup_env
+
+    setup_env()
 
     import argparse
 

egse/setup.py

@@ -644,7 +644,7 @@ def _check_conditions_for_get_path_of_setup_file(site_id: str) -> Path:
 
     print_env()
 
-    repo_location = Path(repo_location)
+    repo_location = Path(repo_location).expanduser()
     setup_location = repo_location / "data" / site_id / "conf"
 
     if not repo_location.is_dir():
@@ -689,7 +689,7 @@ def get_path_of_setup_file(setup_id: int, site_id: str) -> Path:
     """
 
     if not has_conf_repo_location():
-        setup_location = Path(get_conf_data_location(site_id))
+        setup_location = Path(get_conf_data_location(site_id)).expanduser()
     else:
         setup_location = _check_conditions_for_get_path_of_setup_file(site_id)
 
@@ -823,10 +823,11 @@ def submit_setup(setup: Setup, description: str, **kwargs) -> str | None:
 
 def main(args: list = None):  # pragma: no cover
     import argparse
-
     from rich import print
-
     from egse.config import find_files
+    from egse.env import setup_env
+
+    setup_env()
 
     site_id = get_site_id()
     location = get_conf_data_location()
@@ -882,7 +883,7 @@ def main(args: list = None):  # pragma: no cover
 
 
 class SetupManager:
-    """Unified manager that routes Setup access to appropriate providers.
+    """A unified manager that routes Setup access to appropriate providers.
 
     Providers are loaded from the `cgse.extension.setup_providers` entrypoints.
     Providers serve different purposes, the default provider accesses Setups from
@@ -958,6 +959,11 @@ _setup_manager = SetupManager()
 
 
 if __name__ == "__main__":
+    from egse.env import setup_env
+
+    setup_env()
+
+    # import sys
     # main(sys.argv[1:])
-    #
+
     rich.print(load_setup(site_id=get_site_id()))

egse/socketdevice.py

@@ -2,29 +2,50 @@
 This module defines base classes and generic functions to work with sockets.
 """
 
+import asyncio
+import select
 import socket
+import time
+from typing import Optional
 
 from egse.device import DeviceConnectionError
 from egse.device import DeviceConnectionInterface
 from egse.device import DeviceTimeoutError
 from egse.device import DeviceTransport
 from egse.log import logger
+from egse.system import type_name
 
 
 class SocketDevice(DeviceConnectionInterface, DeviceTransport):
     """Base class that implements the socket interface."""
 
-    def __init__(self, hostname: str, port: int):
+    # We set a default connect timeout of 3.0 sec before connecting and reset
+    # to None (=blocking) after connecting. The reason for this is that when no
+    # device is available, e.g. during testing, the timeout will take about
+    # two minutes which is way too long. It needs to be evaluated if this
+    # approach is acceptable and not causing problems during production.
+
+    def __init__(
+        self,
+        hostname: str,
+        port: int,
+        connect_timeout: float = 3.0,
+        read_timeout: float | None = 1.0,
+        separator: str = b"\x03",
+    ):
         super().__init__()
         self.is_connection_open = False
         self.hostname = hostname
         self.port = port
+        self.connect_timeout = connect_timeout
+        self.read_timeout = read_timeout
+        self.separator = separator
         self.socket = None
 
     @property
     def device_name(self):
         """The name of the device that this interface connects to."""
-        raise NotImplementedError
+        return f"SocketDevice({self.hostname}:{self.port})"
 
     def connect(self):
         """
@@ -53,18 +74,12 @@ class SocketDevice(DeviceConnectionInterface, DeviceTransport):
 
         try:
             self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        except socket.error as e_socket:
-            raise ConnectionError(f"{self.device_name}: Failed to create socket.") from e_socket
-
-        # We set a timeout of 3 sec before connecting and reset to None
-        # (=blocking) after the connect. The reason for this is because when no
-        # device is available, e.g during testing, the timeout will take about
-        # two minutes which is way too long. It needs to be evaluated if this
-        # approach is acceptable and not causing problems during production.
+        except socket.error as exc:
+            raise ConnectionError(f"{self.device_name}: Failed to create socket.") from exc
 
         try:
             logger.debug(f'Connecting a socket to host "{self.hostname}" using port {self.port}')
-            self.socket.settimeout(3)
+            self.socket.settimeout(self.connect_timeout)
             self.socket.connect((self.hostname, self.port))
             self.socket.settimeout(None)
         except ConnectionRefusedError as exc:
@@ -120,32 +135,64 @@ class SocketDevice(DeviceConnectionInterface, DeviceTransport):
 
     def read(self) -> bytes:
         """
-
-        Returns:
-            A bytes object containing the received telemetry.
+        Read until ETX (b'\x03') or until `self.read_timeout` elapses.
+        Uses `select` to avoid blocking indefinitely when no data is available.
+        If `self.read_timeout` was set to None in the constructor, this will block anyway.
         """
-        idx, n_total = 0, 0
+        if not self.socket:
+            raise DeviceConnectionError(self.device_name, "Not connected")
+
         buf_size = 1024 * 4
-        response = bytes()
+        response = bytearray()
+
+        # If read_timeout is None we preserve blocking behaviour; otherwise enforce overall timeout.
+        if self.read_timeout is None:
+            end_time = None
+        else:
+            end_time = time.monotonic() + self.read_timeout
 
         try:
-            for idx in range(100):
-                # time.sleep(0.1)  # Give the device time to fill the buffer
-                data = self.socket.recv(buf_size)
-                n = len(data)
-                n_total += n
-                response += data
-                # if n < buf_size:
-                #     break  # there is not more data in the buffer
-                if b"\x03" in response:
+            while True:
+                # compute remaining timeout for select, this is needed because we read in different parts
+                # until ETX is received, and we want to receive the complete messages including ETX within
+                # the read timeout.
+                if end_time is None:
+                    timeout = None
+                else:
+                    remaining = end_time - time.monotonic()
+                    if remaining <= 0.0:
+                        raise DeviceTimeoutError(self.device_name, "Socket read timed out")
+                    timeout = remaining
+
+                ready, _, _ = select.select([self.socket], [], [], timeout)
+
+                if not ready:
+                    # no socket ready within timeout
+                    raise DeviceTimeoutError(self.device_name, "Socket read timed out")
+
+                try:
+                    data = self.socket.recv(buf_size)
+                except OSError as exc:
+                    raise DeviceConnectionError(self.device_name, f"Caught {type_name(exc)}: {exc}") from exc
+
+                if not data:
+                    # remote closed connection (EOF)
+                    raise DeviceConnectionError(self.device_name, "Connection closed by peer")
+
+                response.extend(data)
+
+                if self.separator in response:
                     break
-        except socket.timeout as e_timeout:
-            logger.warning(f"Socket timeout error from {e_timeout}")
-            raise DeviceTimeoutError(self.device_name, "Socket timeout error") from e_timeout
 
-        # logger.debug(f"Total number of bytes received is {n_total}, idx={idx}")
+        except DeviceTimeoutError:
+            raise
+        except DeviceConnectionError:
+            raise
+        except Exception as exc:
+            # unexpected errors - translate to DeviceConnectionError
+            raise DeviceConnectionError(self.device_name, "Socket read error") from exc
 
-        return response
+        return bytes(response)
 
     def write(self, command: str):
         """
@@ -163,11 +210,11 @@ class SocketDevice(DeviceConnectionInterface, DeviceTransport):
 
         try:
             self.socket.sendall(command.encode())
-        except socket.timeout as e_timeout:
-            raise DeviceTimeoutError(self.device_name, "Socket timeout error") from e_timeout
-        except socket.error as e_socket:
+        except socket.timeout as exc:
+            raise DeviceTimeoutError(self.device_name, "Socket timeout error") from exc
+        except socket.error as exc:
             # Interpret any socket-related error as an I/O error
-            raise DeviceConnectionError(self.device_name, "Socket communication error.") from e_socket
+            raise DeviceConnectionError(self.device_name, "Socket communication error.") from exc
 
     def trans(self, command: str) -> bytes:
         """
@@ -198,8 +245,133 @@ class SocketDevice(DeviceConnectionInterface, DeviceTransport):
 
             return return_string
 
-        except socket.timeout as e_timeout:
-            raise DeviceTimeoutError(self.device_name, "Socket timeout error") from e_timeout
-        except socket.error as e_socket:
+        except socket.timeout as exc:
+            raise DeviceTimeoutError(self.device_name, "Socket timeout error") from exc
+        except socket.error as exc:
             # Interpret any socket-related error as an I/O error
-            raise DeviceConnectionError(self.device_name, "Socket communication error.") from e_socket
+            raise DeviceConnectionError(self.device_name, "Socket communication error.") from exc
+
+
+class AsyncSocketDevice(DeviceConnectionInterface, DeviceTransport):
+    """
+    Async socket-backed device using asyncio streams.
+
+    - async connect() / disconnect()
+    - async read() reads until ETX (b'\\x03') or timeout
+    - async write() and async trans()
+    """
+
+    def __init__(
+        self,
+        hostname: str,
+        port: int,
+        connect_timeout: float = 3.0,
+        read_timeout: float | None = 1.0,
+        separator: str = b"\x03",
+    ):
+        super().__init__()
+        self.hostname = hostname
+        self.port = port
+        self.connect_timeout = connect_timeout
+        self.read_timeout = read_timeout
+        self.separator = separator
+        self.reader: Optional[asyncio.StreamReader] = None
+        self.writer: Optional[asyncio.StreamWriter] = None
+        self.is_connection_open = False
+
+    @property
+    def device_name(self) -> str:
+        # Override this property for a decent name
+        return f"AsyncSocketDevice({self.hostname}:{self.port})"
+
+    async def connect(self) -> None:
+        if self.is_connection_open:
+            logger.debug(f"{self.device_name}: already connected")
+            return
+
+        if not self.hostname:
+            raise ValueError(f"{self.device_name}: hostname is not initialized.")
+        if not self.port:
+            raise ValueError(f"{self.device_name}: port is not initialized.")
+
+        try:
+            logger.debug(f"{self.device_name}: connect() called; is_connection_open={self.is_connection_open}")
+            coro = asyncio.open_connection(self.hostname, self.port)
+            self.reader, self.writer = await asyncio.wait_for(coro, timeout=self.connect_timeout)
+            self.is_connection_open = True
+            logger.debug(f"{self.device_name}: connected -> peer={self.writer.get_extra_info('peername')}")
+
+        except asyncio.TimeoutError as exc:
+            await self._cleanup()
+            logger.warning(f"{self.device_name}: connect timed out")
+            raise DeviceTimeoutError(self.device_name, f"Connection to {self.hostname}:{self.port} timed out.") from exc
+        except Exception as exc:
+            await self._cleanup()
+            logger.warning(f"{self.device_name}: connect failed: {type_name(exc)} – {exc}")
+            raise DeviceConnectionError(self.device_name, f"Failed to connect to {self.hostname}:{self.port}") from exc
+
+    async def disconnect(self) -> None:
+        logger.debug(f"{self.device_name}: disconnect() called; writer_exists={bool(self.writer)}")
+        peer = None
+        try:
+            if self.writer and not self.writer.is_closing():
+                peer = self.writer.get_extra_info("peername")
+                self.writer.close()
+                # wait for close, but don't hang forever
+                try:
+                    await asyncio.wait_for(self.writer.wait_closed(), timeout=1.0)
+                except asyncio.TimeoutError:
+                    logger.debug(f"{self.device_name}: wait_closed() timed out for peer={peer}")
+
+        finally:
+            await self._cleanup()
+            logger.debug(f"{self.device_name}: disconnected ({peer=})")
+
+    def is_connected(self) -> bool:
+        return bool(self.is_connection_open and self.writer and not self.writer.is_closing())
+
+    async def _cleanup(self) -> None:
+        self.reader = None
+        self.writer = None
+        self.is_connection_open = False
+
+    async def read(self) -> bytes:
+        if not self.reader:
+            raise DeviceConnectionError(self.device_name, "Not connected")
+        try:
+            # readuntil includes the separator; we keep it for parity with existing code
+            data = await asyncio.wait_for(self.reader.readuntil(separator=self.separator), timeout=self.read_timeout)
+            return data
+        except asyncio.IncompleteReadError as exc:
+            # EOF before separator
+            await self._cleanup()
+            raise DeviceConnectionError(self.device_name, "Connection closed while reading") from exc
+        except asyncio.TimeoutError as exc:
+            raise DeviceTimeoutError(self.device_name, "Socket read timed out") from exc
+        except Exception as exc:
+            await self._cleanup()
+            raise DeviceConnectionError(self.device_name, "Socket read error") from exc
+
+    async def write(self, command: str) -> None:
+        if not self.writer:
+            raise DeviceConnectionError(self.device_name, "Not connected")
+        try:
+            self.writer.write(command.encode())
+            await asyncio.wait_for(self.writer.drain(), timeout=self.read_timeout)
+        except asyncio.TimeoutError as exc:
+            raise DeviceTimeoutError(self.device_name, "Socket write timed out") from exc
+        except Exception as exc:
+            await self._cleanup()
+            raise DeviceConnectionError(self.device_name, "Socket write error") from exc
+
+    async def trans(self, command: str) -> bytes:
+        if not self.writer or not self.reader:
+            raise DeviceConnectionError(self.device_name, "Not connected")
+        try:
+            await self.write(command)
+            return await self.read()
+        except (DeviceTimeoutError, DeviceConnectionError):
+            raise
+        except Exception as exc:
+            await self._cleanup()
+            raise DeviceConnectionError(self.device_name, "Socket trans error") from exc