cgse-common 2025.0.4__py3-none-any.whl → 2025.0.5__py3-none-any.whl

{cgse_common-2025.0.4.dist-info → cgse_common-2025.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse-common
-Version: 2025.0.4
+Version: 2025.0.5
 Summary: Software framework to support hardware testing
 Author: IVS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse_common-2025.0.4.dist-info → cgse_common-2025.0.5.dist-info}/RECORD

@@ -2,10 +2,10 @@ egse/bits.py,sha256=1DPcC5R-8w-KLfbr2WLx-4Ezfwm6jpRMz-E5QfQ_YKw,11138
 egse/calibration.py,sha256=DZ-LgEwamTWDukMqTr2JLCPt3M1lw6W0SfpHQfG7cXg,8989
 egse/command.py,sha256=su35PU_0Bru1Es4up1lJFps9FTQpJ7xcxl9Y4w8suxA,22880
 egse/config.py,sha256=Ib1Ddt0wFwRrioBvMs7Md-wgEEPdK3eaXCt0H7HJEow,14888
-egse/control.py,sha256=gM9II-BgtgJjG7NqxyY8nBtMiRD0b9azGYFeD9_WU6s,12816
+egse/control.py,sha256=cH0EA5mqaS3nQc35x1Sa0o2iM2Q4_hsrak6GOdcRMMU,12793
 egse/decorators.py,sha256=M0PavrPcTwhWS6yLlEo-mOS20lcy90Mym7HUVzPNmhs,13404
 egse/device.py,sha256=SIMROwB9YdNdFowTcZp1HW_el8LWVWDJ5tDUxs58YuY,8500
-egse/env.py,sha256=GdoUDq5trQhSNXwMDjQlCb7Rqs7fgd_eF9ge6yDYZpw,22136
+egse/env.py,sha256=RnqUb_LF_9pjrt2Vqf7b-Pa66XstiKyqgzA_HFlp7Mc,27777
 egse/exceptions.py,sha256=Tc1xqUrWxV6SXxc9xB9viK_O-GVa_SpzqZUVEhZkwzA,1801
 egse/hk.py,sha256=5WMOGlx043XmEPj7ML1TkDzA8ZaTYLD-i5OcNUxE268,31163
 egse/metrics.py,sha256=cZKMEe3OTT2uomj7vXjEl54JD0CStfEC4nCgS6U5YSM,3794
@@ -16,21 +16,21 @@ egse/obsid.py,sha256=-HPuHApZrr3Nj1J2-qqnIiE814C-gm4FSHdM2afKdRY,5883
 egse/persistence.py,sha256=Lx6LMJ1-dh8N43XF7tTM6RwD0sSETiGQ9DNqug-G-zQ,2160
 egse/plugin.py,sha256=Fd_QnABm33_dLjaj1zDgEZr3RKy-N88U5Hz5OZm9Jj0,2684
 egse/process.py,sha256=mQ2ojeL_9oE_QkMJlQDPd1290z0j2mOrGXrlrWtOtzI,16615
-egse/protocol.py,sha256=Psy0iOLPTgARn1VqeKtPCSKepHr_S2KW58UYwjOA6J0,23827
+egse/protocol.py,sha256=G1HTU7xfS4xE1Oi1diDOTWQUTkPLyuooawWCOY6p-b0,23828
 egse/proxy.py,sha256=pMKdnF62SXm0quLoKfgvK9GFkH2mLMB5fWNrZenfqQQ,18100
 egse/reload.py,sha256=rDT0bC6RFeRhW38wSgFcxr30h8FvaKkoGp_OE-AwBB4,4388
 egse/resource.py,sha256=VoB7BVrQULT_SJ1XioDzB59-uH47nUcN-KNVLvFxiFE,15163
-egse/response.py,sha256=WdUUgYe1ABhPAy1HL8lVCM3Gu0Vgy3niaRi_twZuLgo,2700
-egse/services.py,sha256=ZgkF0Rx_PykOVHAOV1cKduJdUhuY6A4DgwjPJWRGj3U,7642
+egse/response.py,sha256=WFtWftovrmmn92NW4mhJjibMtCWteZmAzNuPLVsV-lg,2716
+egse/services.py,sha256=rGS_5mSUFDw8wISEU_nuS9P9_XNtHvpViD9oSH9FqKo,7709
 egse/services.yaml,sha256=p8QBF56zLI21iJ9skt65VlNz4rIqRoFfBTZxOIUZCZ4,1853
-egse/settings.py,sha256=iu17w0xvLhZPXImOt_C3duy2XamClQpKuP-OkCEIZVo,13481
+egse/settings.py,sha256=E8i9nYyz6n931XP0BQu8xmtP3gHICBaSP4VMKL47IXs,15160
 egse/settings.yaml,sha256=M6WMtTbNqNuXsdEf-OzKz0dAxkFTRabpugmIJ2HKyZk,290
-egse/setup.py,sha256=KbByR_6J-deIiY53zBKEaoR3jLwCfdG-TyUo6rqCudo,42902
+egse/setup.py,sha256=wFHqRQtDdkdbJeXH6hF28xUYnYhv-TYPIV-PunUoEow,43789
 egse/state.py,sha256=ekcCZu_DZKkKYn-5iWG7ij7Aif2WYMNVs5h3cia-cVc,5352
-egse/system.py,sha256=t2NrtTAPXbMgBxsqicGjti-b8oEmiHtQu2Ydm77q4lI,48941
+egse/system.py,sha256=KCXz8eH3PNwLaB7ww3OCkK_tQoaTYhh2FEwX1mvN8RQ,48996
 egse/version.py,sha256=-EMuiSn2eEp8QJX6csmBEu1m2yGqlXHJj2Hj49w6a2k,6217
 egse/zmq_ser.py,sha256=2-nwVUBWZ3vvosKNmlWobHJrIJA2HlM3V5a63Gz2JY0,1819
-cgse_common-2025.0.4.dist-info/METADATA,sha256=EFTHkcjwC9ISJ4nL2xY_4Ma57lPDJS9UovA5rtm8rpI,2574
-cgse_common-2025.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cgse_common-2025.0.4.dist-info/entry_points.txt,sha256=E-KaQ9NGWAP1XvLHncNxq5oa22EAf9sOpBZWphXcxiE,34
-cgse_common-2025.0.4.dist-info/RECORD,,
+cgse_common-2025.0.5.dist-info/METADATA,sha256=RBKcR3sDE90ugMy7tQy4c28Qfqae2f6vzEkCWwQ7kjA,2574
+cgse_common-2025.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cgse_common-2025.0.5.dist-info/entry_points.txt,sha256=E-KaQ9NGWAP1XvLHncNxq5oa22EAf9sOpBZWphXcxiE,34
+cgse_common-2025.0.5.dist-info/RECORD,,

egse/control.py

@@ -5,7 +5,6 @@ import abc
 import logging
 import pickle
 import threading
-from typing import Any
 
 import zmq
 

egse/env.py

@@ -40,32 +40,38 @@ WARNING:
 from __future__ import annotations
 
 __all__ = [
-    "get_project_name",
-    "get_site_id",
-    "get_data_storage_location",
-    "set_data_storage_location",
-    "get_data_storage_location_env_name",
+    "env_var",
     "get_conf_data_location",
-    "set_conf_data_location",
     "get_conf_data_location_env_name",
     "get_conf_repo_location",
-    "set_conf_repo_location",
     "get_conf_repo_location_env_name",
+    "get_data_storage_location",
+    "get_data_storage_location_env_name",
+    "get_local_settings",
+    "get_local_settings_env_name",
     "get_log_file_location",
-    "set_log_file_location",
     "get_log_file_location_env_name",
-    "get_local_settings",
+    "get_project_name",
+    "get_site_id",
+    "set_conf_data_location",
+    "set_conf_repo_location",
+    "set_data_storage_location",
     "set_local_settings",
-    "get_local_settings_env_name",
+    "set_log_file_location",
 ]
 
+import contextlib
 import os
 import warnings
 from pathlib import Path
 
 from egse.system import all_logging_disabled
+from egse.system import get_caller_info
 from egse.system import ignore_m_warning
 
+from rich.console import Console
+console = Console(width=100)
+
 # Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
 # create the other environment variables that are specific to the project.
 
@@ -123,11 +129,18 @@ class _Env:
         self._env = {}
 
     def set(self, key, value):
-        self._env[key] = value
+        if value is None:
+            if key in self._env:
+                del self._env[key]
+        else:
+            self._env[key] = value
 
     def get(self, key) -> str:
         return self._env.get(key, NoValue())
 
+    def __rich__(self):
+        return self._env
+
 
 _env = _Env()
 
@@ -159,28 +172,60 @@ def _check_no_value(var_name, value):
     """Raise a ValueError when the value for the variable is NoValue."""
     if value == NoValue():
         project = _env.get("PROJECT")
-        env_name = var_name if var_name == "SITE_ID" else f"{project}_{var_name}"
+        env_name = var_name if var_name in ("PROJECT", "SITE_ID") else f"{project}_{var_name}"
         raise ValueError(
             f"The environment variable {env_name} is not set. "
             f"Please set the environment variable before proceeding."
         )
 
 
+def set_default_environment(
+        project: str,
+        site_id: str,
+        data_storage_location: str | Path,
+        conf_data_location: str | Path | None = None,
+        conf_repo_location: str | Path | None = None,
+        log_file_location: str | Path | None = None,
+        local_settings: str | Path | None = None,
+):
+    set_project_name(project)
+    set_site_id(site_id)
+    set_data_storage_location(data_storage_location)
+    set_conf_data_location(conf_data_location)
+    set_conf_repo_location(conf_repo_location)
+    set_log_file_location(log_file_location)
+    set_local_settings(local_settings)
+
+
 def get_project_name():
+    """Get the PROJECT name. Return None when the PROJECT is not set."""
     return _env.get("PROJECT") or None
 
 
+def set_project_name(name: str):
+    """Set the environment variable PROJECT and its internal representation."""
+    os.environ["PROJECT"] = name
+    _env.set("PROJECT", name)
+
+
 def get_site_id():
+    """Get the SITE_ID. Return None if the SITE_ID is not set."""
     return _env.get("SITE_ID") or None
 
 
+def set_site_id(name: str):
+    """Set the environment variable SITE_ID and its internal representation."""
+    os.environ["SITE_ID"] = name
+    _env.set("SITE_ID", name)
+
+
 def get_data_storage_location_env_name() -> str:
     """Returns the name of the environment variable for the project."""
     project = _env.get("PROJECT")
     return f"{project}_DATA_STORAGE_LOCATION"
 
 
-def set_data_storage_location(location: str | Path):
+def set_data_storage_location(location: str | Path | None):
     """
     Sets the environment variable and the internal representation to the given value.
 
@@ -189,14 +234,19 @@ def set_data_storage_location(location: str | Path):
     """
     env_name = get_data_storage_location_env_name()
 
-    # Check if location exists and is readable
+    if location is None:
+        if env_name in os.environ:
+            del os.environ[env_name]
+        _env.set("DATA_STORAGE_LOCATION", None)
+        return
+
     if not Path(location).exists():
         warnings.warn(
             f"The location you provided for the environment variable {env_name} doesn't exist: {location}."
         )
 
-    os.environ[env_name] = location
-    _env.set('DATA_STORAGE_LOCATION', location)
+    os.environ[env_name] = str(location)
+    _env.set('DATA_STORAGE_LOCATION', str(location))
 
 
 def get_data_storage_location(site_id: str = None) -> str:
@@ -223,6 +273,9 @@ def get_data_storage_location(site_id: str = None) -> str:
     """
     global _env
 
+    project = _env.get("PROJECT")
+    _check_no_value("PROJECT", project)
+
     site_id = site_id or _env.get("SITE_ID")
     _check_no_value("SITE_ID", site_id)
 
@@ -240,7 +293,7 @@ def get_conf_data_location_env_name() -> str:
     return f"{project}_CONF_DATA_LOCATION"
 
 
-def set_conf_data_location(location: str | Path):
+def set_conf_data_location(location: str | Path | None):
     """
     Sets the environment variable and the internal representation to the given value.
 
@@ -250,7 +303,12 @@ def set_conf_data_location(location: str | Path):
 
     env_name = get_conf_data_location_env_name()
 
-    # Check if location exists and is readable
+    if location is None:
+        if env_name in os.environ:
+            del os.environ[env_name]
+        _env.set("CONF_DATA_LOCATION", None)
+        return
+
     if not Path(location).exists():
         warnings.warn(
             f"The location you provided for the environment variable {env_name} doesn't exist: {location}."
@@ -303,7 +361,7 @@ def get_log_file_location_env_name():
     return f"{project}_LOG_FILE_LOCATION"
 
 
-def set_log_file_location(location: str | Path):
+def set_log_file_location(location: str | Path | None):
     """
     Sets the environment variable and the internal representation to the given value.
 
@@ -313,7 +371,12 @@ def set_log_file_location(location: str | Path):
 
     env_name = get_log_file_location_env_name()
 
-    # Check if location exists and is readable
+    if location is None:
+        if env_name in os.environ:
+            del os.environ[env_name]
+        _env.set("LOG_FILE_LOCATION", None)
+        return
+
     if not Path(location).exists():
         warnings.warn(
             f"The location you provided for the environment variable {env_name} doesn't exist: {location}."
@@ -365,17 +428,24 @@ def get_local_settings_env_name() -> str:
     return f"{project}_LOCAL_SETTINGS"
 
 
-def set_local_settings(path: str | Path):
+def set_local_settings(path: str | Path | None):
     """
     Sets the environment variable and the internal representation to the given value.
 
+    When the path is set to None, the environment variable will be unset.
+
     Warnings:
         Issues a warning when the given path doesn't exist.
     """
 
     env_name = get_local_settings_env_name()
 
-    # Check if location exists and is readable
+    if path is None:
+        if env_name in os.environ:
+            del os.environ[env_name]
+        _env.set('LOCAL_SETTINGS', None)
+        return
+
     if not Path(path).exists():
         warnings.warn(
             f"The location you provided for the environment variable {env_name} doesn't exist: {path}."
@@ -399,34 +469,61 @@ def get_local_settings() -> str:
     return local_settings or None
 
 
+def has_conf_repo_location() -> bool:
+    location = _env.get("CONF_REPO_LOCATION")
+    return True if location else False
+
+
 def get_conf_repo_location_env_name() -> str:
     """Returns the name of the environment variable for the project."""
     project = _env.get("PROJECT")
     return f"{project}_CONF_REPO_LOCATION"
 
 
-def get_conf_repo_location() -> str:
-    """Returns the fully qualified name of the location of the repository with configuration and calibration data."""
+def get_conf_repo_location() -> str | None:
+    """
+    Returns the fully qualified name of the location of the repository with
+    configuration and calibration data.
+
+    Returns None if no environment variable was defined or if the location doesn't exist.
+    In both cases a Warning is issued.
+    """
 
     location = _env.get("CONF_REPO_LOCATION")
 
-    if location and not Path(location).exists():
+    if location in (None, NoValue()):
+        warnings.warn(
+            f"The environment variable for the configuration data repository is "
+            f"not defined ({get_conf_repo_location_env_name()})."
+        )
+        return None
+
+    if not Path(location).exists():
         warnings.warn(f"The location of the configuration data repository doesn't exist: {location}.")
+        return None
 
-    return location or None
+    return location
 
 
-def set_conf_repo_location(location: str):
+def set_conf_repo_location(location: str | Path | None):
     """
     Sets the environment variable and the internal representation to the given value.
 
+    When the location is None, the environment variable will be unset and its internal
+    representation will be NoValue().
+
     Warnings:
         Issues a warning when the given location doesn't exist.
     """
 
     env_name = get_conf_repo_location_env_name()
 
-    # Check if location exists and is readable
+    if location is None:
+        if env_name in os.environ:
+            del os.environ[env_name]
+        _env.set("CONF_REPO_LOCATION", None)
+        return
+
     if not Path(location).exists():
         warnings.warn(
             f"The location you provided for the environment variable {env_name} doesn't exist: {location}."
@@ -436,7 +533,69 @@ def set_conf_repo_location(location: str):
     _env.set('CONF_REPO_LOCATION', location)
 
 
-ignore_m_warning('egse.env')
+def print_env():
+    """
+    Prints out the mandatory and known environment variables at the time of the
+    function call. The function and lineno is also printed for information.
+    """
+    col_width = 30
+
+    console = Console(width=200)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        caller_info = get_caller_info(level=2)
+        console.print(f"[b]Environment as in {caller_info.filename}:{caller_info.lineno}[/]")
+        console.print(f"  {'PROJECT':{col_width}s}: {get_project_name()}")
+        console.print(f"  {'SITE_ID':{col_width}s}: {get_site_id()}")
+        console.print(f"  {get_data_storage_location_env_name():{col_width}s}: {get_data_storage_location()}")
+        console.print(f"  {get_log_file_location_env_name():{col_width}s}: {get_log_file_location()}")
+        console.print(f"  {get_conf_data_location_env_name():{col_width}s}: {get_conf_data_location()}")
+        console.print(f"  {get_conf_repo_location_env_name():{col_width}s}: {get_conf_repo_location()}")
+        console.print(f"  {get_local_settings_env_name():{col_width}s}: {get_local_settings()}")
+
+
+@contextlib.contextmanager
+def env_var(**kwargs):
+    """
+    Context manager to run some code that need alternate settings for environment variables.
+    This will automatically initialize the CGSE environment upon entry and re-initialize upon exit.
+
+    NOTE: This context manager is different from the one from `egse.system` because of the CGSE environment changes.
+
+    Args:
+        **kwargs: dictionary with environment variables that are needed
+
+    Examples:
+
+        >>> from egse.env import env_var
+        >>> with env_var(PLATO_DATA_STORAGE_LOCATION="/Users/rik/data"):
+        ...    # do stuff that needs these alternate setting
+        ...    pass
+
+    """
+    saved_env = {}
+
+    for k, v in kwargs.items():
+        saved_env[k] = os.environ.get(k)
+        if v is None:
+            if k in os.environ:
+                del os.environ[k]
+        else:
+            os.environ[k] = v
+
+    initialize()
+
+    yield
+
+    for k, v in saved_env.items():
+        if v is None:
+            if k in os.environ:
+                del os.environ[k]
+        else:
+            os.environ[k] = v
+
+    initialize()
 
 
 def main(args: list | None = None):  # pragma: no cover
@@ -517,8 +676,10 @@ def main(args: list | None = None):  # pragma: no cover
             location = get_data_storage_location()
             if not Path(location).exists():
                 if args.mkdir:
-                    rich.print(f"  [green]⟶ Creating data storage location: {location}[/]")
+                    rich.print(f"  [green]⟶ Creating data storage location: {location} (+ daily + obs)[/]")
                     Path(location).mkdir(parents=True)
+                    (Path(location) / "daily").mkdir()
+                    (Path(location) / "obs").mkdir()
                 else:
                     rich.print("  [red]⟶ ERROR: The data storage location doesn't exist![/]")
             else:
@@ -540,6 +701,16 @@ def main(args: list | None = None):  # pragma: no cover
         except ValueError as exc:
             rich.print(f"    get_conf_data_location() = [red]{exc}[/]")
 
+        try:
+            rich.print(f"    {get_conf_repo_location() = }", flush=True, end="")
+            location = get_conf_repo_location()
+            if location is None or not Path(location).exists():
+                rich.print("  [red]⟶ ERROR: The configuration repository location doesn't exist![/]")
+            else:
+                rich.print()
+        except ValueError as exc:
+            rich.print(f"    get_conf_repo_location() = [red]{exc}[/]")
+
         try:
             rich.print(f"    {get_log_file_location() = }", flush=True, end="")
             location = get_log_file_location()
@@ -623,6 +794,9 @@ def main(args: list | None = None):  # pragma: no cover
     # PLATO_COMMON_EGSE_PATH - YES
 
 
+ignore_m_warning('egse.env')
+
+
 if __name__ == "__main__":
     import sys
     main(sys.argv[1:])

egse/protocol.py

@@ -20,7 +20,7 @@ from prometheus_client import Summary
 from egse.command import Command
 from egse.command import CommandExecution
 from egse.control import ControlServer
-from egse.control import Failure
+from egse.response import Failure
 from egse.device import DeviceConnectionObserver
 from egse.system import format_datetime
 from egse.zmq_ser import bind_address

egse/response.py

@@ -2,9 +2,10 @@
 This module provides functionality to handle responses from the control servers.
 """
 __all__ = [
-    "Success",
     "Failure",
     "Message",
+    "Response",
+    "Success",
 ]
 
 from typing import Any

egse/services.py

@@ -10,6 +10,7 @@ logging levels, or to quit the control server in a controlled way.
 
 import inspect
 import logging
+from pathlib import Path
 
 from egse.command import ClientServerCommand
 from egse.control import ControlServer
@@ -22,7 +23,9 @@ from egse.zmq_ser import connect_address
 
 LOGGER = logging.getLogger(__name__)
 
-SERVICE_SETTINGS = Settings.load(filename="services.yaml")
+HERE = Path(__file__).parent
+
+SERVICE_SETTINGS = Settings.load(filename=str(HERE / "services.yaml"))
 
 
 class ServiceCommand(ClientServerCommand):

egse/settings.py

@@ -8,7 +8,7 @@ settings are loaded from a file called ``settings.yaml``. The default yaml confi
 located in the same directory as this module.
 
 The YAML file is read and the configuration parameters for the given group are
-made available as instance variables of the returned class.
+available as instance variables of the returned class.
 
 The intended use is as follows:
 
@@ -54,24 +54,27 @@ If that file is located at a specific location, also use the ``location=`` keywo
 The above code will read the complete YAML file, i.e. all the groups into a dictionary.
 
 """
+from __future__ import annotations
 
 import logging
-import os
-import pathlib
 import re
+from pathlib import Path
+from typing import Any
 
 import yaml  # This module is provided by the pip package PyYaml - pip install pyyaml
 
 from egse.env import get_local_settings
 from egse.env import get_local_settings_env_name
 from egse.exceptions import FileIsEmptyError
-from egse.system import AttributeDict
-from egse.system import get_caller_info
+from egse.system import attrdict
+from egse.system import get_package_location
 from egse.system import ignore_m_warning
 from egse.system import recursive_dict_update
 
 _LOGGER = logging.getLogger(__name__)
 
+_HERE = Path(__file__).resolve().parent
+
 
 class SettingsError(Exception):
     pass
@@ -109,6 +112,107 @@ SAFE_LOADER.add_implicit_resolver(
     list(u'-+0123456789.'))
 
 
+def get_settings_locations(location: str | Path = None, filename: str = "settings.yaml") -> list[Path]:
+
+    yaml_locations: set[Path] = set()
+
+    if location is None:
+        package_locations = get_package_location("egse")  # `egse` is a namespace
+
+        for package_location in package_locations:
+            if (package_location / filename).exists():
+                yaml_locations.add(package_location)
+
+        yaml_locations.add(_HERE)
+        _LOGGER.debug(f"yaml_locations in Settings.load():  {yaml_locations}")
+
+    else:
+
+        package_location = Path(location).resolve()
+        if (package_location / filename).exists():
+            yaml_locations.add(package_location)
+        else:
+            _LOGGER.warning(f"No '{filename}' file found at {package_location}.")
+
+    return list(yaml_locations)
+
+
+def load_global_settings(locations, filename: str = "settings.yaml", force: bool = False) -> attrdict:
+
+    global_settings = attrdict()
+
+    for yaml_location in locations:
+        try:
+            yaml_document = read_configuration_file(str(yaml_location / filename), force=force)
+            recursive_dict_update(global_settings, yaml_document)
+        except FileNotFoundError as exc:
+            raise SettingsError(
+                f"Filename {filename} not found at location {locations}."
+            ) from exc
+
+    return global_settings
+
+
+def load_local_settings(force: bool = False):
+
+    local_settings = {}
+    try:
+        local_settings_location = get_local_settings()
+
+        if local_settings_location:
+            _LOGGER.debug(f"Using {local_settings_location} to update global settings.")
+            try:
+                yaml_document_local = read_configuration_file(local_settings_location, force=force)
+                if yaml_document_local is None:
+                    raise FileIsEmptyError()
+                local_settings = attrdict(
+                    {name: value for name, value in yaml_document_local.items()}
+                )
+            except FileNotFoundError as exc:
+                raise SettingsError(
+                    f"Local settings YAML file '{local_settings_location}' not found. "
+                    f"Check your environment variable {get_local_settings_env_name()}."
+                ) from exc
+            except FileIsEmptyError:
+                _LOGGER.warning(
+                    f"Local settings YAML file '{local_settings_location}' is empty. "
+                    f"No local settings were loaded.")
+
+    except KeyError as exc:
+        _LOGGER.warning(f"The environment variable {get_local_settings_env_name()} is not defined. (from "
+                        f"{exc.__class__.__name__}: {exc})")
+
+    return local_settings
+
+
+def read_configuration_file(filename: str, *, force=False):
+    """
+    Read the YAML input configuration file. The configuration file is only read
+    once and memoized as load optimization.
+
+    Args:
+        filename (str): the fully qualified filename of the YAML file
+        force (bool): force reloading the file
+
+    Returns:
+        a dictionary containing all the configuration settings from the YAML file.
+    """
+    if force or not Settings.is_memoized(filename):
+
+        _LOGGER.debug(f"Parsing YAML configuration file {filename}.")
+
+        with open(filename, "r") as stream:
+            try:
+                yaml_document = yaml.load(stream, Loader=SAFE_LOADER)
+            except yaml.YAMLError as exc:
+                _LOGGER.error(exc)
+                raise SettingsError(f"Error loading YAML document {filename}") from exc
+
+        Settings.add_memoized(filename, yaml_document)
+
+    return Settings.get_memoized(filename) or {}
+
+
 class Settings:
     """
     The Settings class provides a load() method that loads configuration settings for a group
@@ -116,9 +220,7 @@ class Settings:
     """
 
     __memoized_yaml = {}  # Memoized settings yaml files
-
     __profile = False  # Used for profiling methods and functions
-    __simulation = False  # Use simulation mode where applicable and possible
 
     LOG_FORMAT_DEFAULT = "%(levelname)s:%(module)s:%(lineno)d:%(message)s"
     LOG_FORMAT_FULL = "%(asctime)23s:%(levelname)8s:%(lineno)5d:%(name)-20s: %(message)s"
@@ -132,40 +234,35 @@ class Settings:
     LOG_FORMAT_DATE = "%d/%m/%Y %H:%M:%S"
 
     @classmethod
-    def read_configuration_file(cls, filename: str, *, force=False):
-        """
-        Read the YAML input configuration file. The configuration file is only read
-        once and memoized as load optimization.
-
-        Args:
-            filename (str): the fully qualified filename of the YAML file
-            force (bool): force reloading the file
+    def get_memoized_locations(cls) -> list:
+        return list(cls.__memoized_yaml.keys())
 
-        Returns:
-            a dictionary containing all the configuration settings from the YAML file.
-        """
-        if force or filename not in cls.__memoized_yaml:
+    @classmethod
+    def is_memoized(cls, filename: str) -> bool:
+        return filename in cls.__memoized_yaml
 
-            _LOGGER.debug(f"Parsing YAML configuration file {filename}.")
+    @classmethod
+    def add_memoized(cls, filename: str, yaml_document: Any):
+        cls.__memoized_yaml[filename] = yaml_document
 
-            with open(filename, "r") as stream:
-                try:
-                    yaml_document = yaml.load(stream, Loader=SAFE_LOADER)
-                except yaml.YAMLError as exc:
-                    _LOGGER.error(exc)
-                    raise SettingsError(f"Error loading YAML document {filename}") from exc
+    @classmethod
+    def get_memoized(cls, filename: str):
+        return cls.__memoized_yaml.get(filename)
 
-            cls.__memoized_yaml[filename] = yaml_document
+    @classmethod
+    def clear_memoized(cls):
+        cls.__memoized_yaml.clear()
 
-        return cls.__memoized_yaml[filename]
+    @classmethod
+    def set_profiling(cls, flag):
+        cls.__profile = flag
 
     @classmethod
-    def get_memoized_locations(cls):
-        return cls.__memoized_yaml.keys()
+    def profiling(cls):
+        return cls.__profile
 
     @classmethod
-    def load(cls, group_name=None, filename="settings.yaml", location=None, *, force=False,
-             add_local_settings=True):
+    def load(cls, group_name=None, filename="settings.yaml", location=None, *, force=False, add_local_settings=True):
         """
         Load the settings for the given group from YAML configuration file.
         When no group is provided, the complete configuration is returned.
@@ -186,7 +283,7 @@ class Settings:
         Args:
             group_name (str): the name of one of the main groups from the YAML file
             filename (str): the name of the YAML file to read
-            location (str): the path to the location of the YAML file
+            location (str, Path): the path to the location of the YAML file
             force (bool): force reloading the file
             add_local_settings (bool): update the Settings with site specific local settings
 
@@ -197,99 +294,58 @@ class Settings:
             a SettingsError when the group is not defined in the YAML file.
         """
 
-        _THIS_FILE_LOCATION = pathlib.Path(__file__).resolve().parent
-
-        if location is None:
-
-            # Check if the yaml file is located at the location of the caller,
-            # if not, use the file that is located where the Settings module is located.
-
-            caller_dir = get_caller_info(level=2).filename
-            caller_dir = pathlib.Path(caller_dir).resolve().parent
-
-            if (caller_dir / filename).is_file():
-                yaml_location = caller_dir
-            else:
-                yaml_location = _THIS_FILE_LOCATION
-        else:
-
-            # The location was given as an argument
-
-            yaml_location = pathlib.Path(location).resolve()
-
-        _LOGGER.debug(f"yaml_location in Settings.load(location={location}) is {yaml_location}")
+        # Load all detected YAML documents, these are considered global settings
 
-        # Load the YAML global document
+        yaml_locations = get_settings_locations(location, filename)
+        global_settings = load_global_settings(yaml_locations, filename, force)
 
-        try:
-            yaml_document_global = cls.read_configuration_file(
-                str(yaml_location / filename), force=force
-            )
-        except FileNotFoundError as exc:
-            raise SettingsError(
-                f"Filename {filename} not found at location {yaml_location}."
-            ) from exc
-
-        # Check if there were any groups defined in the YAML document
-
-        if not yaml_document_global:
-            raise SettingsError(f"Empty YAML document {filename} at {yaml_location}.")
+        if not global_settings:
+            raise SettingsError(f"There were no global settings defined for {filename} at {yaml_locations}.")
 
         # Load the LOCAL settings YAML file
 
-        local_settings = {}
-
         if add_local_settings:
-            try:
-                local_settings_location = get_local_settings()
-                if local_settings_location:
-                    _LOGGER.debug(f"Using {local_settings_location} to update global settings.")
-                    try:
-                        yaml_document_local = cls.read_configuration_file(
-                            local_settings_location, force=force
-                        )
-                        if yaml_document_local is None:
-                            raise FileIsEmptyError()
-                        local_settings = AttributeDict(
-                            {name: value for name, value in yaml_document_local.items()}
-                        )
-                    except FileNotFoundError as exc:
-                        raise SettingsError(
-                            f"Local settings YAML file '{local_settings_location}' not found. "
-                            f"Check your environment variable {get_local_settings_env_name()}."
-                        ) from exc
-                    except FileIsEmptyError:
-                        _LOGGER.warning(f"Local settings YAML file '{local_settings_location}' is empty. "
-                                       f"No local settings were loaded.")
-            except KeyError:
-                _LOGGER.debug(f"The environment variable {get_local_settings_env_name()} is not defined.")
+            local_settings = load_local_settings(force)
+        else:
+            local_settings = {}
 
         if group_name in (None, ""):
-            global_settings = AttributeDict(
-                {name: value for name, value in yaml_document_global.items()}
+            global_settings = attrdict(
+                {name: value for name, value in global_settings.items()},
+                label="Settings"
             )
             if add_local_settings:
                 recursive_dict_update(global_settings, local_settings)
             return global_settings
 
-        # Check if the requested group is defined in the YAML document
+        if group_name in global_settings:
+            include_global_settings = True
+        else:
+            include_global_settings = False
+        if group_name in local_settings:
+            include_local_settings = True
+        else:
+            include_local_settings = False
 
-        if group_name not in yaml_document_global:
+        if not include_global_settings and not include_local_settings:
             raise SettingsError(
-                f"Group name '{group_name}' is not defined in the YAML "
-                f"document '{filename}' at '{yaml_location}."
+                f"Group name '{group_name}' is not defined in the global nor in the local settings."
             )
 
         # Check if the group has any settings
 
-        if not yaml_document_global[group_name]:
-            raise SettingsError(f"Empty group in YAML document {filename} at {yaml_location}.")
+        if include_global_settings and not global_settings[group_name]:
+            _LOGGER.warning(f"Empty group in YAML document {filename} for {group_name}.")
 
-        group_settings = AttributeDict(
-            {name: value for name, value in yaml_document_global[group_name].items()}
-        )
+        if include_global_settings:
+            group_settings = attrdict(
+                {name: value for name, value in global_settings[group_name].items()},
+                label=group_name
+            )
+        else:
+            group_settings = attrdict(label=group_name)
 
-        if add_local_settings and group_name in local_settings:
+        if add_local_settings and include_local_settings:
             recursive_dict_update(group_settings, local_settings[group_name])
 
         return group_settings
@@ -312,34 +368,15 @@ class Settings:
                     trunc += " ..."
                 msg += f"   {field}: {trunc}\n"
 
-        return msg
-
-    @classmethod
-    def set_profiling(cls, flag):
-        cls.__profile = flag
-
-    @classmethod
-    def profiling(cls):
-        return cls.__profile
+        return msg.rstrip()
 
-    @classmethod
-    def set_simulation_mode(cls, flag: bool):
-        cls.__simulation = flag
-
-    @classmethod
-    def simulation_mode(cls) -> bool:
-        return cls.__simulation
-
-
-ignore_m_warning('egse.settings')
-
-if __name__ == "__main__":
 
+def main(args: list | None = None):  # pragma: no cover
     # We provide convenience to inspect the settings by calling this module directly from Python.
     #
     # python -m egse.settings
     #
-    # Use the '--help' option to see the what your choices are.
+    # Use the '--help' option to see what your choices are.
 
     logging.basicConfig(level=20)
 
@@ -352,7 +389,10 @@ if __name__ == "__main__":
         ),
     )
     parser.add_argument("--local", action="store_true", help="print only the local settings.")
-    args = parser.parse_args()
+    parser.add_argument("--global", action="store_true",
+                        help="print only the global settings, don't include local settings.")
+    parser.add_argument("--group", help="print only settings for this group")
+    args = parser.parse_args(args or [])
 
     # The following import will activate the pretty printing of the AttributeDict
     # through the __rich__ method.
@@ -362,13 +402,20 @@ if __name__ == "__main__":
     if args.local:
         location = get_local_settings()
         if location:
+            location = str(Path(location).expanduser().resolve())
             settings = Settings.load(filename=location)
             print(settings)
             print(f"Loaded from [purple]{location}.")
         else:
             print("[red]No local settings defined.")
     else:
-        settings = Settings.load()
+        # if the global option is given we don't want to include local settings
+        add_local_settings = False if vars(args)["global"] else True
+
+        if args.group:
+            settings = Settings.load(args.group, add_local_settings=add_local_settings)
+        else:
+            settings = Settings.load(add_local_settings=add_local_settings)
         print(settings)
         print("[blue]Memoized locations:")
         locations = Settings.get_memoized_locations()
@@ -379,3 +426,9 @@ def get_site_id() -> str:
 
     site = Settings.load("SITE")
     return site.ID
+
+
+# ignore_m_warning('egse.settings')
+
+if __name__ == "__main__":
+    main()

egse/setup.py

@@ -133,6 +133,8 @@ from rich.tree import Tree
 from egse.env import get_conf_repo_location
 from egse.env import get_conf_repo_location_env_name
 from egse.env import get_data_storage_location
+from egse.env import has_conf_repo_location
+from egse.env import print_env
 from egse.response import Failure
 from egse.env import get_conf_data_location
 from egse.system import format_datetime
@@ -227,7 +229,11 @@ def _load_yaml(resource_name: str):
 
 
 def _load_pandas(resource_name: str, separator: str):
-    """ Find and return the content of the given files as a pandas DataFrame object.
+    """
+    Find and return the content of the given file as a pandas DataFrame object.
+
+    The file is loaded relative from the location of the configuration data
+    as defined by `get_conf_data_location()`.
 
     Args:
         - resource_name: Filename, preceded by "pandas//".
@@ -336,7 +342,7 @@ class NavigableDict(dict):
 
     """
 
-    def __init__(self, head: dict = None):
+    def __init__(self, head: dict = None, label: str = None):
         """
         Args:
             head (dict): the original dictionary
@@ -344,6 +350,7 @@ class NavigableDict(dict):
         head = head or {}
         super().__init__(head)
         self.__dict__["_memoized"] = {}
+        self.__dict__["_label"] = label
 
         # By agreement, we only want the keys to be set as attributes if all keys are strings.
         # That way we enforce that always all keys are navigable, or none.
@@ -572,7 +579,7 @@ class NavigableDict(dict):
         return msg
 
     def __rich__(self) -> Tree:
-        tree = Tree("NavigableDict", guide_style="dim")
+        tree = Tree(self.__dict__["_label"] or "NavigableDict", guide_style="dim")
         walk_dict_tree(self, tree, text_style="dark grey")
         return tree
 
@@ -636,8 +643,8 @@ class Setup(NavigableDict):
     """The Setup class represents a version of the configuration of the test facility, the
     test setup and the Camera Under Test (CUT)."""
 
-    def __init__(self, nav_dict: NavigableDict | dict = None):
-        super().__init__(nav_dict or {})
+    def __init__(self, nav_dict: NavigableDict | dict = None, label: str = None):
+        super().__init__(nav_dict or {}, label=label)
 
     @staticmethod
     def from_dict(my_dict):
@@ -651,7 +658,7 @@ class Setup(NavigableDict):
             >>> assert setup["ID"] == setup.ID == "my-setup-001"
 
         """
-        return Setup(my_dict)
+        return Setup(my_dict, label="Setup")
 
     @staticmethod
     def from_yaml_string(yaml_content: str = None):
@@ -674,15 +681,16 @@ class Setup(NavigableDict):
         if "Setup" in setup_dict:
             setup_dict = setup_dict["Setup"]
 
-        return Setup(setup_dict)
+        return Setup(setup_dict, label="Setup")
 
     @staticmethod
     @lru_cache
-    def from_yaml_file(filename: Union[str, Path] = None):
+    def from_yaml_file(filename: Union[str, Path] = None, add_local_settings: bool = True):
         """Loads a Setup from the given YAML file.
 
         Args:
             filename (str): the path of the YAML file to be loaded
+            add_local_settings (bool): if local settings shall be loaded and override the settings from the YAML file.
 
         Returns:
             a Setup that was loaded from the given location.
@@ -692,9 +700,11 @@ class Setup(NavigableDict):
         if not filename:
             raise ValueError("Invalid argument to function: No filename or None given.")
 
-        setup_dict = Settings.load("Setup", filename=filename, force=True)
+        # MODULE_LOGGER.info(f"Loading {filename}...")
 
-        setup = Setup(setup_dict)
+        setup_dict = Settings.load("Setup", filename=filename, force=True, add_local_settings=add_local_settings)
+
+        setup = Setup(setup_dict, label="Setup")
         setup.set_private_attribute("_filename", filename)
         if setup_id := _parse_filename_for_setup_id(str(filename)):
             setup.set_private_attribute("_setup_id", setup_id)
@@ -911,18 +921,15 @@ def get_setup(setup_id: int = None):
 
 def _check_conditions_for_get_path_of_setup_file(site_id: str) -> Path:
     """
-    Check some pre-conditions that need to be met before we try to determine the file path for
-    the requested Setup file.
+    Check some pre-conditions that need to be met before we try to determine the
+    file path for the requested Setup file.
 
     The following checks are performed:
 
-    * if the environment variable 'PLATO_CONF_REPO_LOCATION' is set
-
+    * if the environment variable '{PROJECT}_CONF_REPO_LOCATION' is set
     * if the directory specified in the env variable actually exists
-
     * if the folder with the Setups exists for the given site_id
 
-
     Args:
         site_id (str): the name of the test house
 
@@ -939,9 +946,12 @@ def _check_conditions_for_get_path_of_setup_file(site_id: str) -> Path:
 
     if not (repo_location := get_conf_repo_location()):
         raise LookupError(
-            f"Environment variable doesn't exist, please define {repo_location_env} and try again."
+            f"Environment variable doesn't exist or points to an invalid location, please (re-)define"
+            f" {repo_location_env} and try again."
         )
 
+    print_env()
+
     repo_location = Path(repo_location)
     setup_location = repo_location / 'data' / site_id / 'conf'
 
@@ -962,11 +972,13 @@ def _check_conditions_for_get_path_of_setup_file(site_id: str) -> Path:
 
 def get_path_of_setup_file(setup_id: int, site_id: str) -> Path:
     """
-    Returns the Path to the last Setup file for the given site_id. The last Setup file is the file
-    with the largest setup_id number.
+    Returns the Path to the last Setup file for the given site_id. The last Setup
+    file is the file with the largest setup_id number.
 
-    This function needs the environment variable <PROJECT>_CONF_REPO_LOCATION to be defined as the
-    location of the repository with configuration data on your disk.
+    This function needs the environment variable <PROJECT>_CONF_REPO_LOCATION to
+    be defined as the location of the repository with configuration data on your
+    disk. If the repo is not defined, the configuration data location will be used
+    instead.
 
     Args:
         setup_id (int): the identifier for the requested Setup
@@ -984,7 +996,10 @@ def get_path_of_setup_file(setup_id: int, site_id: str) -> Path:
 
     """
 
-    setup_location = _check_conditions_for_get_path_of_setup_file(site_id)
+    if not has_conf_repo_location():
+        setup_location = Path(get_conf_data_location(site_id))
+    else:
+        setup_location = _check_conditions_for_get_path_of_setup_file(site_id)
 
     if setup_id:
         files = list(setup_location.glob(f'SETUP_{site_id}_{setup_id:05d}_*.yaml'))

egse/system.py

@@ -28,6 +28,7 @@ import socket
 import subprocess  # For executing a shell command
 import sys
 import time
+import warnings
 from collections import namedtuple
 from contextlib import contextmanager
 from pathlib import Path
@@ -512,6 +513,9 @@ class AttributeDict(dict):
         return self.__class__.__name__ + f"({{{sub_msg}{', ...' if len(self) > count else ''}}})"
 
 
+attrdict = AttributeDict
+
+
 def walk_dict_tree(dictionary: dict, tree: Tree, text_style: str = "green"):
     for k, v in dictionary.items():
         if isinstance(v, dict):
@@ -931,22 +935,23 @@ def env_var(**kwargs):
 
     """
     saved_env = {}
-    try:
-        for k, v in kwargs.items():
-            saved_env[k] = os.environ.get(k)
-            if v is None:
-                if k in os.environ:
-                    del os.environ[k]
-            else:
-                os.environ[k] = v
-        yield
-    finally:
-        for k, v in saved_env.items():
-            if v is None:
-                if k in os.environ:
-                    del os.environ[k]
-            else:
-                os.environ[k] = v
+
+    for k, v in kwargs.items():
+        saved_env[k] = os.environ.get(k)
+        if v is None:
+            if k in os.environ:
+                del os.environ[k]
+        else:
+            os.environ[k] = v
+
+    yield
+
+    for k, v in saved_env.items():
+        if v is None:
+            if k in os.environ:
+                del os.environ[k]
+        else:
+            os.environ[k] = v
 
 
 def filter_by_attr(elements: Iterable, **attrs) -> List:
@@ -1120,7 +1125,7 @@ def is_namespace(module) -> bool:
         return False
 
 
-def get_package_location(module) -> List[Path]:
+def get_package_location(module: str) -> List[Path]:
     """
     Retrieves the file system locations associated with a Python package.
 
@@ -1153,6 +1158,7 @@ def get_package_location(module) -> List[Path]:
     try:
         module = importlib.import_module(module)
     except TypeError:
+        warnings.warn(f"The module is not found or is not valid: {module_name}.")
         return []
 
     if is_namespace(module):