cgse-common 2025.0.4__py3-none-any.whl → 2025.0.6__py3-none-any.whl

egse/protocol.py

@@ -20,7 +20,7 @@ from prometheus_client import Summary
 from egse.command import Command
 from egse.command import CommandExecution
 from egse.control import ControlServer
-from egse.control import Failure
+from egse.response import Failure
 from egse.device import DeviceConnectionObserver
 from egse.system import format_datetime
 from egse.zmq_ser import bind_address

egse/response.py

@@ -2,9 +2,10 @@
 This module provides functionality to handle responses from the control servers.
 """
 __all__ = [
-    "Success",
     "Failure",
     "Message",
+    "Response",
+    "Success",
 ]
 
 from typing import Any

egse/services.py

@@ -10,6 +10,7 @@ logging levels, or to quit the control server in a controlled way.
 
 import inspect
 import logging
+from pathlib import Path
 
 from egse.command import ClientServerCommand
 from egse.control import ControlServer
@@ -22,7 +23,9 @@ from egse.zmq_ser import connect_address
 
 LOGGER = logging.getLogger(__name__)
 
-SERVICE_SETTINGS = Settings.load(filename="services.yaml")
+HERE = Path(__file__).parent
+
+SERVICE_SETTINGS = Settings.load(filename=str(HERE / "services.yaml"))
 
 
 class ServiceCommand(ClientServerCommand):

egse/settings.py

@@ -3,12 +3,19 @@ The Settings class handles user and configuration settings that are provided in
 a [`YAML`](http://yaml.org) file.
 
 The idea is that settings are grouped by components or any arbitrary grouping that makes sense for
-the application or for the user. The Settings class can read from different YAML files. By default,
-settings are loaded from a file called ``settings.yaml``. The default yaml configuration file is
-located in the same directory as this module.
+the application or for the user. Settings are also modular and provided by each package by means
+of entry-points.The Settings class can read from different YAML files.
+
+By default, settings are loaded from a file called `settings.yaml`, but this can be changed in the entry-point
+definition.
+
+The yaml configuration files are provided as entry-points by the packages that provided an entry-point
+group 'cgse.settings'. The Settings dictionary (attrdict) is constructed from the configuration YAML
+files from each of the packages. Settings can be overwritten by the next package configuration file.
+So, make sure the group names in each package configuration file are unique.
 
 The YAML file is read and the configuration parameters for the given group are
-made available as instance variables of the returned class.
+available as instance variables of the returned class.
 
 The intended use is as follows:
 
@@ -23,11 +30,10 @@ The intended use is as follows:
     else:
         raise RMAPError("Attempt to access outside the RMAP memory map.")
 
-
-The above code reads the settings from the default YAML file for a group called ``DSI``.
+The above code reads the settings from the default YAML file for a group called `DSI`.
 The settings will then be available as variables of the returned class, in this case
-``dsi_settings``. The returned class is and behaves also like a dictionary, so you can check
-if a configuration parameter is defined like this:
+`dsi_settings`. The returned class is and behaves also like a dictionary, so you can
+check if a configuration parameter is defined like this:
 
     if "DSI_FEE_IP_ADDRESS" not in dsi_settings:
         # define the IP address of the DSI
@@ -46,53 +52,37 @@ The YAML section for the above code looks like this:
         RMAP_BASE_ADDRESS:     0x00000000   # The start of the RMAP memory map managed by the FEE
         RMAP_MEMORY_SIZE:            4096   # The size of the RMAP memory map managed by the FEE
 
-When you want to read settings from another YAML file, specify the ``filename=`` keyword.
-If that file is located at a specific location, also use the ``location=`` keyword.
+When you want to read settings from another YAML file, specify the `filename=` keyword.
+If that file is located at a specific location, also use the `location=` keyword.
 
     my_settings = Settings.load(filename="user.yaml", location="/Users/JohnDoe")
 
-The above code will read the complete YAML file, i.e. all the groups into a dictionary.
+The above code will read the YAML file from the given location and not from the entry-points.
 
 """
+from __future__ import annotations
 
 import logging
-import os
-import pathlib
 import re
+from pathlib import Path
+from typing import Any
 
 import yaml  # This module is provided by the pip package PyYaml - pip install pyyaml
 
-from egse.env import get_local_settings
 from egse.env import get_local_settings_env_name
-from egse.exceptions import FileIsEmptyError
-from egse.system import AttributeDict
-from egse.system import get_caller_info
-from egse.system import ignore_m_warning
+from egse.env import get_local_settings_path
+from egse.system import attrdict
 from egse.system import recursive_dict_update
 
 _LOGGER = logging.getLogger(__name__)
 
+_HERE = Path(__file__).resolve().parent
+
 
 class SettingsError(Exception):
     pass
 
 
-def is_defined(cls, name):
-    return hasattr(cls, name)
-
-
-def get_attr_value(cls, name, default=None):
-    try:
-        return getattr(cls, name)
-    except AttributeError:
-        return default
-
-
-def set_attr_value(cls, name, value):
-    if hasattr(cls, name):
-        raise KeyError(f"Overwriting setting {name} with {value}, was {hasattr(cls, name)}")
-
-
 # Fix the problem: YAML loads 5e-6 as string and not a number
 # https://stackoverflow.com/questions/30458977/yaml-loads-5e-6-as-string-and-not-a-number
 
@@ -109,6 +99,133 @@ SAFE_LOADER.add_implicit_resolver(
     list(u'-+0123456789.'))
 
 
+def load_settings_file(path: Path, filename: str, force: bool = False) -> attrdict:
+    """
+    Loads the YAML configuration file that is located at `path / filename`.
+
+    Args:
+        - path (PATH): the folder where the YAML file is located
+        - filename (str): the name of the YAML configuration file
+        - force (bool): force reloading, i.e. don't use the cached information
+
+    Raises:
+        A SettingsError when the configuration file doesn't exist or cannot be found.
+
+        A SettingsError when there was an error reading the configuration file.
+
+    Returns:
+         A dictionary (attrdict) with all the settings from the given file.
+
+         Note that, in case of an empty configuration file, and empty dictionary
+         is returned and a warning message is issued.
+    """
+    try:
+        yaml_document = read_configuration_file(path / filename, force=force)
+        settings = attrdict(
+            {name: value for name, value in yaml_document.items()}
+        )
+    except FileNotFoundError as exc:
+        raise SettingsError(
+            f"The Settings YAML file '{filename}' is not found at {path!s}. "
+        ) from exc
+
+    if not settings:
+        _LOGGER.warning(
+            f"The Settings YAML file '{filename}' at {path!s} is empty. "
+            f"No local settings were loaded, an empty dictionary is returned.")
+
+    return settings
+
+
+def load_global_settings(entry_point: str = 'cgse.settings', force: bool = False) -> attrdict:
+    """
+    Loads the settings that are defined by the given entry_point. The entry-points are defined in the
+    `pyproject.toml` files of the packages that export their global settings.
+
+    Args:
+         - entry_point (str): the name of the entry-point group [default: 'cgse.settings']
+         - force (bool): force reloading the settings, i.e. ignore the cache
+
+    Returns:
+        A dictionary (attrdict) containing a collection of all the settings exported by the packages
+        through the given entry-point.
+
+    """
+    from egse.plugin import get_file_infos
+
+    ep_settings = get_file_infos(entry_point)
+
+    global_settings = attrdict(label="Settings")
+
+    for ep_name, (path, filename) in ep_settings.items():
+        settings = load_settings_file(path, filename, force)
+        recursive_dict_update(global_settings, settings)
+
+    return global_settings
+
+
+def load_local_settings(force: bool = False) -> attrdict:
+    """
+    Loads the local settings file that is defined from the environment variable <PROJECT>_LOCAL_SETTINGS.
+
+    This function might return an empty dictionary when
+
+      - the local settings YAML file is empty
+      - the local settings environment variable is not defined.
+
+    in both cases a warning message is logged.
+
+    Returns:
+        A dictionary (attrdict) with all local settings.
+
+    Raises:
+        A SettingsError when the local settings YAML file is not found. Check the <PROJECT>_LOCAL_SETTINGS
+        environment variable.
+    """
+    local_settings = attrdict()
+
+    local_settings_path = get_local_settings_path()
+
+    if local_settings_path:
+        path = Path(local_settings_path)
+        local_settings = load_settings_file(path.parent, path.name, force)
+
+    return local_settings
+
+
+def read_configuration_file(filename: Path, *, force=False):
+    """
+    Read the YAML input configuration file. The configuration file is only read
+    once and memoized as load optimization.
+
+    Args:
+        filename (Path): the fully qualified filename of the YAML file
+        force (bool): force reloading the file, even when it was memoized
+
+    Raises:
+        A SettingsError when there was an error reading the YAML file.
+
+    Returns:
+        a dictionary containing all the configuration settings from the YAML file.
+    """
+    filename = str(filename)
+
+    if force or not Settings.is_memoized(filename):
+
+        _LOGGER.debug(f"Parsing YAML configuration file {filename}.")
+
+        with open(filename, "r") as stream:
+            try:
+                yaml_document = yaml.load(stream, Loader=SAFE_LOADER)
+            except yaml.YAMLError as exc:
+                _LOGGER.error(exc)
+                raise SettingsError(f"Error loading YAML document {filename}") from exc
+
+        Settings.add_memoized(filename, yaml_document)
+
+    return Settings.get_memoized(filename) or {}
+
+
 class Settings:
     """
     The Settings class provides a load() method that loads configuration settings for a group
@@ -116,9 +233,7 @@ class Settings:
     """
 
     __memoized_yaml = {}  # Memoized settings yaml files
-
     __profile = False  # Used for profiling methods and functions
-    __simulation = False  # Use simulation mode where applicable and possible
 
     LOG_FORMAT_DEFAULT = "%(levelname)s:%(module)s:%(lineno)d:%(message)s"
     LOG_FORMAT_FULL = "%(asctime)23s:%(levelname)8s:%(lineno)5d:%(name)-20s: %(message)s"
@@ -132,167 +247,122 @@ class Settings:
     LOG_FORMAT_DATE = "%d/%m/%Y %H:%M:%S"
 
     @classmethod
-    def read_configuration_file(cls, filename: str, *, force=False):
-        """
-        Read the YAML input configuration file. The configuration file is only read
-        once and memoized as load optimization.
-
-        Args:
-            filename (str): the fully qualified filename of the YAML file
-            force (bool): force reloading the file
-
-        Returns:
-            a dictionary containing all the configuration settings from the YAML file.
-        """
-        if force or filename not in cls.__memoized_yaml:
-
-            _LOGGER.debug(f"Parsing YAML configuration file {filename}.")
-
-            with open(filename, "r") as stream:
-                try:
-                    yaml_document = yaml.load(stream, Loader=SAFE_LOADER)
-                except yaml.YAMLError as exc:
-                    _LOGGER.error(exc)
-                    raise SettingsError(f"Error loading YAML document {filename}") from exc
-
-            cls.__memoized_yaml[filename] = yaml_document
-
-        return cls.__memoized_yaml[filename]
+    def get_memoized_locations(cls) -> list:
+        return list(cls.__memoized_yaml.keys())
 
     @classmethod
-    def get_memoized_locations(cls):
-        return cls.__memoized_yaml.keys()
+    def is_memoized(cls, filename: str) -> bool:
+        return filename in cls.__memoized_yaml
 
     @classmethod
-    def load(cls, group_name=None, filename="settings.yaml", location=None, *, force=False,
-             add_local_settings=True):
-        """
-        Load the settings for the given group from YAML configuration file.
-        When no group is provided, the complete configuration is returned.
+    def add_memoized(cls, filename: str, yaml_document: Any):
+        cls.__memoized_yaml[filename] = yaml_document
 
-        The default YAML file is 'settings.yaml' and is located in the same directory
-        as the settings module.
-
-        About the ``location`` keyword several options are available.
-
-        * when no location is given, i.e. ``location=None``, the YAML settings file is searched for
-          at the same location as the settings module.
-
-        * when a relative location is given, the YAML settings file is searched for relative to the
-          current working directory.
+    @classmethod
+    def get_memoized(cls, filename: str):
+        return cls.__memoized_yaml.get(filename)
 
-        * when an absolute location is given, that location is used 'as is'.
+    @classmethod
+    def clear_memoized(cls):
+        cls.__memoized_yaml.clear()
 
-        Args:
-            group_name (str): the name of one of the main groups from the YAML file
-            filename (str): the name of the YAML file to read
-            location (str): the path to the location of the YAML file
-            force (bool): force reloading the file
-            add_local_settings (bool): update the Settings with site specific local settings
+    @classmethod
+    def set_profiling(cls, flag):
+        cls.__profile = flag
 
-        Returns:
-            a dynamically created class with the configuration parameters as instance variables.
+    @classmethod
+    def profiling(cls):
+        return cls.__profile
 
-        Raises:
-            a SettingsError when the group is not defined in the YAML file.
+    @staticmethod
+    def _load_all(
+            entry_point: str = 'cgse.settings',
+            add_local_settings: bool = False,
+            force: bool = False
+    ) -> attrdict:
         """
+        Loads all settings from all package with the entry point 'cgse.settings'
+        """
+        global_settings = load_global_settings(entry_point, force)
 
-        _THIS_FILE_LOCATION = pathlib.Path(__file__).resolve().parent
-
-        if location is None:
-
-            # Check if the yaml file is located at the location of the caller,
-            # if not, use the file that is located where the Settings module is located.
-
-            caller_dir = get_caller_info(level=2).filename
-            caller_dir = pathlib.Path(caller_dir).resolve().parent
+        # Load the LOCAL settings YAML file
 
-            if (caller_dir / filename).is_file():
-                yaml_location = caller_dir
-            else:
-                yaml_location = _THIS_FILE_LOCATION
-        else:
+        if add_local_settings:
+            local_settings = load_local_settings(force)
+            recursive_dict_update(global_settings, local_settings)
 
-            # The location was given as an argument
+        return global_settings
 
-            yaml_location = pathlib.Path(location).resolve()
+    @staticmethod
+    def _load_group(
+            group_name: str,
+            entry_point: str = 'cgse.settings',
+            add_local_settings: bool = False,
+            force: bool = False
+    ) -> attrdict:
 
-        _LOGGER.debug(f"yaml_location in Settings.load(location={location}) is {yaml_location}")
+        global_settings = load_global_settings(entry_point, force)
 
-        # Load the YAML global document
+        group_settings = attrdict(label=group_name)
 
-        try:
-            yaml_document_global = cls.read_configuration_file(
-                str(yaml_location / filename), force=force
+        if group_name in global_settings:
+            group_settings = attrdict(
+                {name: value for name, value in global_settings[group_name].items()},
+                label=group_name
             )
-        except FileNotFoundError as exc:
-            raise SettingsError(
-                f"Filename {filename} not found at location {yaml_location}."
-            ) from exc
-
-        # Check if there were any groups defined in the YAML document
 
-        if not yaml_document_global:
-            raise SettingsError(f"Empty YAML document {filename} at {yaml_location}.")
+        if add_local_settings:
+            local_settings = load_local_settings(force)
+            if group_name in local_settings:
+                recursive_dict_update(group_settings, local_settings[group_name])
 
-        # Load the LOCAL settings YAML file
+        if not group_settings:
+            raise SettingsError(
+                f"Group name '{group_name}' is not defined in the global nor in the local settings."
+            )
 
-        local_settings = {}
+        return group_settings
 
-        if add_local_settings:
-            try:
-                local_settings_location = get_local_settings()
-                if local_settings_location:
-                    _LOGGER.debug(f"Using {local_settings_location} to update global settings.")
-                    try:
-                        yaml_document_local = cls.read_configuration_file(
-                            local_settings_location, force=force
-                        )
-                        if yaml_document_local is None:
-                            raise FileIsEmptyError()
-                        local_settings = AttributeDict(
-                            {name: value for name, value in yaml_document_local.items()}
-                        )
-                    except FileNotFoundError as exc:
-                        raise SettingsError(
-                            f"Local settings YAML file '{local_settings_location}' not found. "
-                            f"Check your environment variable {get_local_settings_env_name()}."
-                        ) from exc
-                    except FileIsEmptyError:
-                        _LOGGER.warning(f"Local settings YAML file '{local_settings_location}' is empty. "
-                                       f"No local settings were loaded.")
-            except KeyError:
-                _LOGGER.debug(f"The environment variable {get_local_settings_env_name()} is not defined.")
-
-        if group_name in (None, ""):
-            global_settings = AttributeDict(
-                {name: value for name, value in yaml_document_global.items()}
-            )
-            if add_local_settings:
-                recursive_dict_update(global_settings, local_settings)
-            return global_settings
+    @staticmethod
+    def _load_one(location: str, filename: str, force=False) -> attrdict:
 
-        # Check if the requested group is defined in the YAML document
+        return load_settings_file(Path(location).expanduser(), filename, force)
 
-        if group_name not in yaml_document_global:
-            raise SettingsError(
-                f"Group name '{group_name}' is not defined in the YAML "
-                f"document '{filename}' at '{yaml_location}."
-            )
+    @classmethod
+    def load(
+            cls,
+            group_name=None, filename="settings.yaml", location=None,
+            *, add_local_settings=True, force=False
+    ):
+        """
+        Load the settings for the given group. When no group is provided, the
+        complete configuration is returned.
 
-        # Check if the group has any settings
+        The Settings are loaded from entry-points that are defined in each of the
+        packages that provide a Settings file.
 
-        if not yaml_document_global[group_name]:
-            raise SettingsError(f"Empty group in YAML document {filename} at {yaml_location}.")
+        If a location is explicitly provided, the Settings will be loaded from that
+        location, using the given filename or the default (which is settings.yaml).
 
-        group_settings = AttributeDict(
-            {name: value for name, value in yaml_document_global[group_name].items()}
-        )
+        Args:
+            group_name (str): the name of one of the main groups from the YAML file
+            filename (str): the name of the YAML file to read [default=settings.yaml]
+            location (str, Path): the path to the location of the YAML file
+            force (bool): force reloading the file
+            add_local_settings (bool): update the Settings with site specific local settings
 
-        if add_local_settings and group_name in local_settings:
-            recursive_dict_update(group_settings, local_settings[group_name])
+        Returns:
+            a dynamically created class with the configuration parameters as instance variables.
 
-        return group_settings
+        Raises:
+            a SettingsError when the group is not defined in the YAML file.
+        """
+        if group_name:
+            return cls._load_group(group_name, add_local_settings=add_local_settings, force=force)
+        elif location:
+            return cls._load_one(location=location, filename=filename, force=force)
+        else:
+            return cls._load_all(add_local_settings=add_local_settings, force=force)
 
     @classmethod
     def to_string(cls):
@@ -312,34 +382,15 @@ class Settings:
                     trunc += " ..."
                 msg += f"   {field}: {trunc}\n"
 
-        return msg
-
-    @classmethod
-    def set_profiling(cls, flag):
-        cls.__profile = flag
-
-    @classmethod
-    def profiling(cls):
-        return cls.__profile
-
-    @classmethod
-    def set_simulation_mode(cls, flag: bool):
-        cls.__simulation = flag
-
-    @classmethod
-    def simulation_mode(cls) -> bool:
-        return cls.__simulation
-
-
-ignore_m_warning('egse.settings')
+        return msg.rstrip()
 
-if __name__ == "__main__":
 
+def main(args: list | None = None):  # pragma: no cover
     # We provide convenience to inspect the settings by calling this module directly from Python.
     #
     # python -m egse.settings
     #
-    # Use the '--help' option to see the what your choices are.
+    # Use the '--help' option to see what your choices are.
 
     logging.basicConfig(level=20)
 
@@ -352,7 +403,10 @@ if __name__ == "__main__":
         ),
     )
     parser.add_argument("--local", action="store_true", help="print only the local settings.")
-    args = parser.parse_args()
+    parser.add_argument("--global", action="store_true",
+                        help="print only the global settings, don't include local settings.")
+    parser.add_argument("--group", help="print only settings for this group")
+    args = parser.parse_args(args or [])
 
     # The following import will activate the pretty printing of the AttributeDict
     # through the __rich__ method.
@@ -360,15 +414,22 @@ if __name__ == "__main__":
     from rich import print
 
     if args.local:
-        location = get_local_settings()
+        location = get_local_settings_path()
         if location:
+            location = str(Path(location).expanduser().resolve())
             settings = Settings.load(filename=location)
             print(settings)
             print(f"Loaded from [purple]{location}.")
         else:
             print("[red]No local settings defined.")
     else:
-        settings = Settings.load()
+        # if the global option is given we don't want to include local settings
+        add_local_settings = False if vars(args)["global"] else True
+
+        if args.group:
+            settings = Settings.load(args.group, add_local_settings=add_local_settings)
+        else:
+            settings = Settings.load(add_local_settings=add_local_settings)
         print(settings)
         print("[blue]Memoized locations:")
         locations = Settings.get_memoized_locations()
@@ -379,3 +440,9 @@ def get_site_id() -> str:
 
     site = Settings.load("SITE")
     return site.ID
+
+
+# ignore_m_warning('egse.settings')
+
+if __name__ == "__main__":
+    main()

egse/settings.yaml

@@ -1,7 +1,5 @@
-SITE:
-    ID:                            XXX
-    SSH_SERVER:              localhost         # The IP address of the SSH server on your site
-    SSH_PORT:                       22         # The TCP/IP port on which the SSH server is listening
+# This is a stub file. Don't add any settings to this file, use the settings.yaml file
+# in your projects module, like e.g. `cgse_common/settings.yaml` for this module.
 
-PROCESS:
-    METRICS_INTERVAL:              10
+SITE:
+    ID:  XXXX