cgse-core 2025.0.4__py3-none-any.whl → 2025.0.5__py3-none-any.whl

{cgse_core-2025.0.4.dist-info → cgse_core-2025.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse-core
-Version: 2025.0.4
+Version: 2025.0.5
 Summary: Core services for the CGSE framework
 Author-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>
 License-Expression: MIT
@@ -13,10 +13,6 @@ Requires-Dist: gitpython>=3.1.44
 Requires-Dist: prometheus-client>=0.21.1
 Requires-Dist: pyzmq==23.2.1
 Requires-Dist: rich>=13.9.4
-Provides-Extra: dev
-Requires-Dist: pytest; extra == 'dev'
-Requires-Dist: pytest-cov; extra == 'dev'
-Requires-Dist: pytest-mock; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # The core services for the CGSE platform

{cgse_core-2025.0.4.dist-info → cgse_core-2025.0.5.dist-info}/RECORD

@@ -1,20 +1,21 @@
-egse/confman/__init__.py,sha256=KfsXYirEodvWN7BlzDjfGorZOu8qWER6VPVwibpPCg4,37279
+egse/settings.yaml,sha256=OtloyzII85CFJ1mG00QrRts7jtWAMHBOeXBb0QVc9kY,2663
+egse/confman/__init__.py,sha256=Woeffl3sRL-r3tfEgbtCSwZBd2GnctSt0zxULeE10c8,37831
 egse/confman/__main__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 egse/confman/confman.yaml,sha256=V3TnUHEVlrHt2VZ1HFOGgRVjq_fhWAIAg5eslKddMEs,2909
-egse/confman/confman_cs.py,sha256=Up6C1sPXMtOhfpJUh001iTnRnNvissUKGw2HmRLCc-w,6594
+egse/confman/confman_cs.py,sha256=Yyhj0tvG0zQEEcsWZgPCsAW3-MDKL-9nAqXkFhVkoR4,6404
 egse/logger/__init__.py,sha256=lkwftajGyfsUQe-iwi0IDU9w5rsIirdZPqAataKC3rU,8163
 egse/logger/__main__.py,sha256=RFCl0vmeNmHeXNDbziUP5sDwC014TqwJfc8xtDvZ_as,370
-egse/logger/log_cs.py,sha256=HjY8_4NynzpN_wgWIBysBBcPp2-7K2YAjJU3bgPbFJw,9297
-egse/procman/__init__.py,sha256=0t6_tB1cHX9NCN_YmpA9mPrxDbIeOhuNBbe7o-jj2i4,27398
+egse/logger/log_cs.py,sha256=lFZe5z5U_k28L3qp3P1rQt1Ux0OxM3pLdNXIvKSiF9s,9303
+egse/procman/__init__.py,sha256=VJ0gkBC7U2LkwKY_JpL7_--7kQHWdVYbNgHB_FhRjog,27494
 egse/procman/procman.yaml,sha256=ig7SKFAh3gif2EUhSLAA39sdbPf66LfduJ2OcInxNYU,1608
 egse/procman/procman_cs.py,sha256=Ht2xQA36-K2yUNA0ac0HIBopnZv0X8oSIz5HaAqPeKI,6001
-egse/storage/__init__.py,sha256=B3E7iyPzYWAmIhgINGzul4nM8f2_uhaja41U0CgDGIQ,40012
+egse/storage/__init__.py,sha256=m-ooUTD6_WBssSDMAYPIBgWph2YEpf6abvxghnbd49c,40013
 egse/storage/__main__.py,sha256=LI9fxlsFWmEd5LcWUB0xA8i7Yt6UHgnblB4G0aTi3pI,28
 egse/storage/persistence.py,sha256=e4kMTszUXQxqPWBfnA_3elRHaKQmXJvKri7LwXQZXdk,18246
 egse/storage/storage.yaml,sha256=zTRtRFbuMLBILsnlIphG6iWjI1Nav6tW7uOm4cUvFuk,2593
-egse/storage/storage_cs.py,sha256=tF1BBDbj7sWNXAw6arSHTOnDEbi8-4tEfOhI0Jf-tnk,5506
+egse/storage/storage_cs.py,sha256=Z-HnpeqF5QgRLjhEnW4jIQ6QKFdY55jK1XBUttxww_k,5447
 scripts/cgse.py,sha256=euz9Lw3Lh0X-DZct7jywTX6JQJPiBua_zIRhsERI-64,5182
-cgse_core-2025.0.4.dist-info/METADATA,sha256=J-3IZgzIkadkgYWKixfVee_BuCNTSnUWS4PueKnr34I,748
-cgse_core-2025.0.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-cgse_core-2025.0.4.dist-info/entry_points.txt,sha256=gcg5ck-Lh7byq1S88agmzwxFG5OJBsIDH439gaUPkHE,215
-cgse_core-2025.0.4.dist-info/RECORD,,
+cgse_core-2025.0.5.dist-info/METADATA,sha256=Q2WzcHoPQp3LOA2p5n51zlvvrXG4WltGK142K2xEAUg,605
+cgse_core-2025.0.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+cgse_core-2025.0.5.dist-info/entry_points.txt,sha256=gcg5ck-Lh7byq1S88agmzwxFG5OJBsIDH439gaUPkHE,215
+cgse_core-2025.0.5.dist-info/RECORD,,

egse/confman/__init__.py

@@ -128,18 +128,21 @@ from egse.command import ClientServerCommand
 from egse.command import stringify_function_call
 from egse.config import find_file
 from egse.config import find_files
-from egse.config import get_common_egse_root
 from egse.control import ControlServer
-from egse.control import Failure
-from egse.control import Response
-from egse.control import Success
 from egse.control import is_control_server_active
 from egse.decorators import dynamic_interface
 from egse.decorators import static_vars
+from egse.env import get_conf_data_location
+from egse.env import get_conf_repo_location
+from egse.env import get_project_name
+from egse.env import get_site_id
 from egse.exceptions import InternalError
 from egse.obsid import ObservationIdentifier
 from egse.protocol import CommandProtocol
 from egse.proxy import Proxy
+from egse.response import Failure
+from egse.response import Response
+from egse.response import Success
 from egse.settings import Settings
 from egse.settings import SettingsError
 from egse.setup import Setup
@@ -147,17 +150,17 @@ from egse.setup import load_last_setup_id
 from egse.setup import save_last_setup_id
 from egse.system import filter_by_attr
 from egse.system import format_datetime
-from egse.system import replace_environment_variable
 from egse.version import VERSION
 from egse.zmq_ser import bind_address
 from egse.zmq_ser import connect_address
 
 LOGGER = logging.getLogger(__name__)
 
+HERE = Path(__file__).parent
+
 CTRL_SETTINGS = Settings.load("Configuration Manager Control Server")
-SITE = Settings.load("SITE")
-COMMAND_SETTINGS = Settings.load(filename="confman.yaml")
-REPO = Settings.load("REPO")
+SITE_ID = get_site_id()
+COMMAND_SETTINGS = Settings.load(location=HERE, filename="confman.yaml")
 
 CM_SETUP_ID = Gauge("CM_SETUP_ID", 'Setup ID')
 CM_TEST_ID = Gauge("CM_TEST_ID", 'Test ID')
@@ -174,14 +177,14 @@ def _push_setup_to_repo(filename: str, commit_msg: str) -> Failure | Success:
         None.
     """
 
-    repo_workdir = REPO.CGSE_CONF
-    repo_workdir = replace_environment_variable(repo_workdir)
+    repo_workdir = get_conf_repo_location()
+
     if repo_workdir is None:
         msg = textwrap.dedent(
-            """\
-            Couldn't determine the repository location for cgse-conf. 
+            f"""\
+            Couldn't determine the repository location for configuration data. 
 
-            Check if the environment variable 'CGSE_CONF_REPO_LOCATION' is set 
+            Check if the environment variable '{get_project_name()}_CONF_REPO_LOCATION' is set and valid 
             before starting the configuration manager.
             """
         )
@@ -285,8 +288,14 @@ def _populate_cached_setup_info():
 
     LOGGER.info("Populating cache with Setup Info.")
 
-    location = replace_environment_variable(CTRL_SETTINGS.FILE_STORAGE_LOCATION)
-    data_conf_location = Path(location) if location else get_common_egse_root()
+    location = get_conf_data_location()
+    if location:
+        data_conf_location = Path(location)
+    else:
+        raise ValueError(
+            "Couldn't determine location of the configuration data with 'get_conf_data_location()'. "
+            "Check if the environment is properly defined."
+        )
 
     setup_info = {}
 
@@ -550,8 +559,13 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
 
         # Find the location for the configuration data
 
-        location = replace_environment_variable(CTRL_SETTINGS.FILE_STORAGE_LOCATION)
-        self._data_conf_location = Path(location) if location else get_common_egse_root()
+        location = get_conf_data_location()
+        if location:
+            self._data_conf_location = Path(location)
+        else:
+            raise ValueError(
+                "The location for the configuration data is not defined. Please check your environment."
+            )
 
         # Populate the cache with information from the available Setups. This will also load each
         # Setup and cache them with the lru_cache decorator. Since this takes about 5s for 100
@@ -590,7 +604,7 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
             last_obsid = self._storage.read({"origin": "obsid", "select": "last_line"})
             last_obsid = last_obsid.return_code if isinstance(last_obsid, Success) else None
 
-        self._obsid = create_obsid(last_obsid, SITE.ID, self._setup_id)
+        self._obsid = create_obsid(last_obsid, SITE_ID, self._setup_id)
 
         if self._storage:
             response = self._storage.start_observation(self._obsid, self._sut_name)
@@ -696,14 +710,14 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
 
         setup_files = list(
             find_files(
-                pattern=f"SETUP_{SITE.ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
+                pattern=f"SETUP_{SITE_ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
             )
         )
 
         if len(setup_files) != 1:
             LOGGER.error(
                 msg := f"Expected to find just one Setup YAML file, found {len(setup_files)}. "
-                       f"[{SITE.ID = }, {setup_id = }, data_conf_location={self._data_conf_location}]"
+                       f"[{SITE_ID = }, {setup_id = }, data_conf_location={self._data_conf_location}]"
             )
             return Failure("Loading Setup", InternalError(msg))
 
@@ -745,7 +759,7 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
 
             setup_files = list(
                 find_files(
-                    pattern=f"SETUP_{SITE.ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
+                    pattern=f"SETUP_{SITE_ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
                 )
             )
 
@@ -786,7 +800,7 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
             The Site identifier as a string.
         """
 
-        return SITE.ID
+        return SITE_ID
 
     def reload_setups(self):
         """
@@ -839,7 +853,7 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
             try:
                 setup_id = int(rc.return_code[-1].split(maxsplit=3)[2])
                 setup_file = find_file(
-                    name=f"SETUP_{SITE.ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
+                    name=f"SETUP_{SITE_ID}_{setup_id:05d}_*.yaml", root=self._data_conf_location
                 )
                 setup = Setup.from_yaml_file(setup_file)
             except (IndexError, SettingsError):
@@ -867,7 +881,7 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
 
         setup_id = self.get_next_setup_id_for_site(site)
 
-        filename = _construct_filename(SITE.ID, setup_id)
+        filename = _construct_filename(SITE_ID, setup_id)
 
         if not hasattr(setup, "history"):
             setup.history = {}
@@ -876,6 +890,12 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
         setup.set_private_attribute("_setup_id", setup_id)
         setup.to_yaml_file(self._data_conf_location / filename)
 
+        # No repository is defined. This should not break, but a warning is in place.
+        # The warnings are issued by the get_conf_repo_location() function.
+
+        if get_conf_repo_location() is None:
+            return
+
         try:
             rc = _push_setup_to_repo(filename, description)
             if isinstance(rc, Failure):
@@ -896,12 +916,13 @@ class ConfigurationManagerController(ConfigurationManagerInterface):
         return setup
 
     def get_next_setup_id_for_site(self, site: str) -> int:
-        """Return the next available Setup ID for the given Site.
+        """
+        Return the next available Setup ID for the given Site.
 
         Args:
             site (str): site identification, e.g. CSL, SRON, ...
         """
-        site = site or SITE.ID
+        site = site or SITE_ID
         files = sorted(find_files(pattern=f"SETUP_{site}_*.yaml", root=self._data_conf_location))
         last_file = files[-1]
         setup_id = last_file.name.split("_")[2]

egse/confman/confman_cs.py

@@ -16,15 +16,15 @@ from pathlib import Path
 import click
 import rich
 import zmq
-from egse.control import ControlServer
-from egse.control import Response
-from egse.process import SubProcess
-from egse.settings import Settings
-from egse.system import replace_environment_variable
 from prometheus_client import start_http_server
 
 from egse.confman import ConfigurationManagerProtocol
 from egse.confman import ConfigurationManagerProxy
+from egse.control import ControlServer
+from egse.env import get_conf_data_location
+from egse.process import SubProcess
+from egse.response import Response
+from egse.settings import Settings
 
 # Use explicit name here otherwise the logger will probably be called __main__
 
@@ -205,14 +205,11 @@ def check_prerequisites():
 
     # We need a proper location for storing the configuration data.
 
-    location = CTRL_SETTINGS.FILE_STORAGE_LOCATION
-    location = replace_environment_variable(location)
+    location = get_conf_data_location()
 
     if not location:
         raise RuntimeError(
-            "The environment variable referenced in the Settings.yaml file for the "
-            "FILE_STORAGE_LOCATION of the Configuration Manager does not exist, please set "
-            "the environment variable."
+            "The location for the configuration data is not defined. Please check your environment."
         )
 
     location = Path(location)

egse/logger/log_cs.py

@@ -119,9 +119,9 @@ def start():
     stream_handler = StreamHandler()
     stream_handler.setFormatter(logging.Formatter(fmt=LOG_FORMAT_STREAM))
 
-    # Log records are also sent to the cutelog listening server
+    # Log records are also sent to the textualog listening server
 
-    socket_handler = SocketHandler(CTRL_SETTINGS.CUTELOG_IP_ADDRESS, CTRL_SETTINGS.CUTELOG_LISTENING_PORT)
+    socket_handler = SocketHandler(CTRL_SETTINGS.TEXTUALOG_IP_ADDRESS, CTRL_SETTINGS.TEXTUALOG_LISTENING_PORT)
     socket_handler.setFormatter(file_formatter)
 
     context = zmq.Context()

egse/procman/__init__.py

@@ -7,11 +7,14 @@ import pickle
 import time
 
 import sys
+from pathlib import Path
+
 import zmq
 
 from egse.command import ClientServerCommand
 from egse.confman import ConfigurationManagerProxy
-from egse.control import ControlServer, Success, Failure
+from egse.control import ControlServer
+from egse.response import Success, Failure
 from egse.decorators import dynamic_interface
 from egse.dpu import fitsgen
 from egse.dpu.dpu_cs import is_dpu_cs_active
@@ -29,8 +32,10 @@ from egse.system import format_datetime
 from egse.zmq_ser import bind_address
 from egse.zmq_ser import connect_address
 
+HERE = Path(__file__).parent
+
 CTRL_SETTINGS = Settings.load("Process Manager Control Server")
-COMMAND_SETTINGS = Settings.load(filename="procman.yaml")
+COMMAND_SETTINGS = Settings.load(location=HERE, filename="procman.yaml")
 
 LOGGER = logging.getLogger(__name__)
 

egse/settings.yaml

@@ -0,0 +1,32 @@
+Logging Control Server:                          # LOG_CS
+
+    PROTOCOL:                       tcp
+    HOSTNAME:                 localhost          # The hostname that client shall connect to, e.g. pleiad01 @ KU Leuven
+    LOGGING_PORT:                  7000
+    COMMANDING_PORT:               7001
+    METRICS_PORT:                  7003          # The HTTP port where Prometheus will connect to for retrieving metrics
+    MAX_NR_LOG_FILES:                20          # The maximum number of log files that will be maintained in a roll-over
+    MAX_SIZE_LOG_FILES:              20          # The maximum size one log file can become
+    TEXTUALOG_IP_ADDRESS:     127.0.0.1          # The IP address of the textualog listening server
+    TEXTUALOG_LISTENING_PORT:     19996          # The port number on which the textualog server is listening
+
+Configuration Manager Control Server:            # CM_CS
+
+    PROTOCOL:                       tcp
+    HOSTNAME:                 localhost          # The hostname that client shall connect to, e.g. pleiad01 @ KU Leuven
+    COMMANDING_PORT:               6000          # The port on which the controller listens to commands - REQ-REP
+    MONITORING_PORT:               6001          # The port on which the controller sends periodic status information of the device - PUB-SUB
+    SERVICE_PORT:                  6002          # The port on which the controller listens for configuration and administration - REQ-REP
+    METRICS_PORT:                  6003          # The HTTP port where Prometheus will connect to for retrieving metrics
+    DELAY:                            1          # The delay time between publishing status information [seconds]
+    STORAGE_MNEMONIC:                CM          # The mnemonic to be used in the filename storing the housekeeping data
+
+Storage Control Server:                          # SM_CS
+
+    PROTOCOL:                       tcp
+    HOSTNAME:                 localhost          # The hostname that client shall connect to, e.g. pleiad01 @ KU Leuven
+    COMMANDING_PORT:               6100          # The port on which the controller listens to commands - REQ-REP
+    MONITORING_PORT:               6101          # The port on which the controller sends periodic status information of the device - PUB-SUB
+    SERVICE_PORT:                  6102          # The port on which the controller listens for configuration and administration - REQ-REP
+    METRICS_PORT:                  6103          # The HTTP port where Prometheus will connect to for retrieving metrics
+    DELAY:                            1          # The delay time between publishing status information [seconds]

egse/storage/__init__.py

@@ -112,9 +112,10 @@ from egse.bits import humanize_bytes
 from egse.command import ClientServerCommand
 from egse.config import find_files
 from egse.control import ControlServer
-from egse.control import Failure
-from egse.control import Response
-from egse.control import Success
+from egse.env import get_site_id
+from egse.response import Failure
+from egse.response import Response
+from egse.response import Success
 from egse.control import is_control_server_active
 from egse.decorators import dynamic_interface
 from egse.env import get_data_storage_location
@@ -132,11 +133,11 @@ from egse.zmq_ser import connect_address
 
 logger = logging.getLogger(__name__)
 
+HERE = Path(__file__).parent
+
 CTRL_SETTINGS = Settings.load("Storage Control Server")
-SITE = Settings.load("SITE")
-COMMAND_SETTINGS = Settings.load(filename="storage.yaml")
-DEVICE_SETTINGS = Settings.load(filename="storage.yaml")
-CCD_SETTINGS = Settings.load("CCD")
+SITE_ID = get_site_id()
+DEVICE_SETTINGS = COMMAND_SETTINGS = Settings.load(location=HERE, filename="storage.yaml")
 
 __all__ = [
     "is_storage_manager_active",
@@ -543,7 +544,7 @@ def _construct_filename(
         The full path to the file as a `PurePath`.
     """
 
-    site_id = site_id or SITE.ID
+    site_id = site_id or SITE_ID
     location = location or get_data_storage_location(site_id=site_id)
 
     if obsid:
@@ -854,7 +855,7 @@ class StorageController(StorageInterface):
             self._registry.register(item["origin"], item)
 
             if "filename" in item:
-                location = Path(get_data_storage_location(site_id=SITE.ID))
+                location = Path(get_data_storage_location(site_id=SITE_ID))
                 filename = location / item["filename"]
             else:
                 filename = _construct_filename(item["origin"], item["persistence_class"].extension,
@@ -972,7 +973,7 @@ class StorageController(StorageInterface):
                 )
 
     def get_storage_location(self):
-        return get_data_storage_location(site_id=SITE.ID)
+        return get_data_storage_location(site_id=SITE_ID)
 
     def get_filenames(self, item: dict) -> List[Path]:
         registered_item = self._registry.get(item["origin"])
@@ -995,7 +996,7 @@ class StorageController(StorageInterface):
 
     def get_disk_usage(self):
 
-        location = Path(get_data_storage_location(site_id=SITE.ID))
+        location = Path(get_data_storage_location(site_id=SITE_ID))
         total, used, free = shutil.disk_usage(location)
         return total, used, free
 

egse/storage/storage_cs.py

@@ -20,6 +20,7 @@ from pytz import utc
 from egse.bits import humanize_bytes
 from egse.control import ControlServer
 from egse.env import get_data_storage_location
+from egse.env import get_site_id
 from egse.process import SubProcess
 from egse.settings import Settings
 from egse.storage import StorageProtocol
@@ -33,7 +34,7 @@ from egse.system import replace_environment_variable
 logger = logging.getLogger("egse.storage.storage_cs")
 
 CTRL_SETTINGS = Settings.load("Storage Control Server")
-SITE = Settings.load("SITE")
+SITE_ID = get_site_id()
 
 
 class StorageControlServer(ControlServer):
@@ -159,12 +160,11 @@ def check_prerequisites():
     # We need a proper location for storing the data, this directory shall contain
     # two subfolders: 'daily' and 'obs'.
 
-    location = get_data_storage_location(site_id=SITE.ID)
+    location = get_data_storage_location(site_id=SITE_ID)
 
     if not location:
         raise RuntimeError(
-            "The environment variable referenced in the Settings.yaml file for the "
-            "FILE_STORAGE_LOCATION does not exist, please set the environment variable."
+            "The data storage location is not defined. Please check your environment."
         )
 
     location = Path(location)