cgse 0.13.1__tar.gz → 0.14.0__tar.gz

{cgse-0.13.1 → cgse-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cgse
-Version: 0.13.1
+Version: 0.14.0
 Summary: Generic Common-EGSE: Commanding and monitoring lab equipment
 Author: IvS KU Leuven
 Maintainer-email: Rik Huygen <rik.huygen@kuleuven.be>, Sara Regibo <sara.regibo@kuleuven.be>

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cgse-common"
-version = "0.13.1"
+version = "0.14.0"
 description = "Software framework to support hardware testing"
 authors = [
     {name = "IvS KU Leuven"}

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/cgse_common/cgse.py

@@ -184,6 +184,7 @@ def main(ctx: typer.Context, verbose: bool = False):
         # print("Try 'cgse --help' for a list of commands.")
         typer.echo(ctx.get_help())
 
+
 build_app()
 
 

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/calibration.py

@@ -5,8 +5,8 @@ This module provides functions to calibrate sensor values.
 from __future__ import annotations
 
 import numpy as np
-
-from egse.setup import navdict, Setup, SetupError
+from navdict import navdict
+from egse.setup import Setup, SetupError
 
 
 def apply_gain_offset(counts: float, gain: float, offset: float) -> float:

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/config.py

@@ -3,27 +3,15 @@ This module provides convenience functions to properly configure the CGSE
 and to find paths and resources.
 """
 
-from __future__ import annotations
-
-import errno
 import fnmatch
-import logging
 import os
-from functools import lru_cache
-from os.path import exists
-from os.path import join
 from pathlib import Path
 from pathlib import PurePath
-from typing import List
-from typing import Optional
+from typing import Generator
 from typing import Tuple
 from typing import Union
-from typing import Generator
 
-import git
-from egse.decorators import deprecate
-
-_logger = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def find_first_occurrence_of_dir(pattern: str, root: Path | str) -> Path | None:
@@ -252,18 +240,6 @@ def find_root(
     return Path(default) if default is not None else None
 
 
-def set_logger_levels(logger_levels: List[Tuple] = None):
-    """
-    Set the logging level for the given loggers.
-
-    """
-    logger_levels = logger_levels or []
-
-    for name, level in logger_levels:
-        a_logger = logging.getLogger(name)
-        a_logger.setLevel(level)
-
-
 class WorkingDirectory:
     """
     WorkingDirectory is a context manager to temporarily change the working directory while
@@ -301,7 +277,7 @@ class WorkingDirectory:
         try:
             os.chdir(self._current_dir)
         except OSError as exc:
-            _logger.warning(f"Change back to previous directory failed: {exc}")
+            logger.warning(f"Change back to previous directory failed: {exc}")
 
     @property
     def path(self):

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/counter.py

@@ -2,8 +2,6 @@
 This module manages files that have a counter in their filename.
 """
 
-from __future__ import annotations
-
 __all__ = [
     "counter_exists",
     "counter_filename",
@@ -11,12 +9,10 @@ __all__ = [
     "new_counter",
 ]
 
-import logging
 from pathlib import Path
 
 from egse.config import find_files
-
-_LOGGER = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def counter_filename(location: Path, filename: Path | str) -> Path:
@@ -133,7 +129,7 @@ def get_next_counter(filename: Path) -> int:
     return counter
 
 
-def determine_counter_from_dir_list(location: str, pattern: str, index: int = -1) -> int:
+def determine_counter_from_dir_list(location: Path | str, pattern: str, index: int = -1) -> int:
     """
     Determine counter for a new file at the given location and with the given pattern.
     The next counter is determined from the sorted list of files that match the given pattern.
@@ -170,9 +166,9 @@ def determine_counter_from_dir_list(location: str, pattern: str, index: int = -1
         #  <anything here>_counter>.<extension>
 
         counter = int(parts[index].split(".")[0]) + 1
-        _LOGGER.debug(f"{counter = }")
+        logger.debug(f"{counter = }")
         return counter
 
     except ValueError:
-        _LOGGER.warning("ValueError", exc_info=True)
+        logger.warning("ValueError", exc_info=True)
         return 1

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/decorators.py

@@ -17,8 +17,7 @@ import rich
 
 from egse.settings import Settings
 from egse.system import get_caller_info
-
-_LOGGER = logging.getLogger(__name__)
+from egse.log import logger
 
 
 def static_vars(**kwargs):
@@ -161,14 +160,14 @@ def average_time(*, name: str = "average_time", level: int = logging.INFO, preci
         def _report_average_time():
             if func._call_count:
                 average_time = func._run_time / func._call_count
-                _LOGGER.log(
+                logger.log(
                     level,
                     f"{name}: "
                     f"average runtime of {func.__name__!r} is {average_time:.{precision}f}s, "
                     f"#calls = {func._call_count}.",
                 )
             else:
-                _LOGGER.log(level, f"{name}: function {func.__name__!r} was never called.")
+                logger.log(level, f"{name}: function {func.__name__!r} was never called.")
 
             return func._run_time, func._call_count
 
@@ -210,7 +209,7 @@ def timer(*, name: str = "timer", level: int = logging.INFO, precision: int = 4)
             value = func(*args, **kwargs)
             end_time = time.perf_counter()
             run_time = end_time - start_time
-            _LOGGER.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
+            logger.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
             return value
 
         return wrapper_timer
@@ -235,7 +234,7 @@ def async_timer(*, name: str = "timer", level: int = logging.INFO, precision: in
             value = await func(*args, **kwargs)
             end_time = time.perf_counter()
             run_time = end_time - start_time
-            _LOGGER.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
+            logger.log(level, f"{name}: Finished {func.__name__!r} in {run_time:.{precision}f} secs")
             return value
 
         return wrapper_timer
@@ -307,9 +306,9 @@ def debug(func):
             args_repr = [repr(a) for a in args]
             kwargs_repr = [f"{k}={v!r}" for k, v in kwargs.items()]
             signature = ", ".join(args_repr + kwargs_repr)
-            _LOGGER.debug(f"Calling {func.__name__}({signature})")
+            logger.debug(f"Calling {func.__name__}({signature})")
             value = func(*args, **kwargs)
-            _LOGGER.debug(f"{func.__name__!r} returned {value!r}")
+            logger.debug(f"{func.__name__!r} returned {value!r}")
         else:
             value = func(*args, **kwargs)
         return value
@@ -523,7 +522,7 @@ def to_be_implemented(func):
 
     @functools.wraps(func)
     def wrapper_tbi(*args, **kwargs):
-        _LOGGER.warning(f"The function/method {func.__name__} is not yet implemented.")
+        logger.warning(f"The function/method {func.__name__} is not yet implemented.")
         return func(*args, **kwargs)
 
     return wrapper_tbi
@@ -736,12 +735,12 @@ def retry_with_exponential_backoff(
             while attempt < max_attempts:
                 try:
                     response = func(*args, **kwargs)  # Attempt to call the function
-                    _LOGGER.info(f"{func.__name__} successfully executed.")
+                    logger.info(f"{func.__name__} successfully executed.")
                     return response
                 except tuple(exceptions) as exc:
                     last_exception = exc
                     attempt += 1
-                    _LOGGER.info(
+                    logger.info(
                         f"Retry {attempt}: {func.__name__} will be executing again in {wait_time * backoff_factor}s. "
                         f"Received a {last_exception!r}."
                     )
@@ -805,7 +804,7 @@ def retry(times: int = 3, wait: float = 10.0, exceptions: List = None) -> Callab
                 except tuple(exceptions) as exc:
                     previous_exception = exc
                 if n < times:
-                    _LOGGER.info(
+                    logger.info(
                         f"Retry {n + 1}: {func.__name__} will be executing again in {wait}s. "
                         f"Received a {previous_exception!r}."
                     )

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/device.py

@@ -3,7 +3,6 @@ This module defines the generic interfaces to connect devices.
 """
 
 from enum import IntEnum
-from typing import List
 
 from egse.decorators import dynamic_interface
 from egse.exceptions import Error

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/dicts.py

@@ -9,14 +9,14 @@ __all__ = [
 ]
 import logging
 
-from egse.system import capture_rich_output
-from egse.system import flatten_dict
 from rich.table import Table
 
-LOGGER = logging.getLogger(__name__)
+from egse.log import logger
+from egse.system import capture_rich_output
+from egse.system import flatten_dict
 
 
-def log_differences(dict_1, dict_2):
+def log_differences(dict_1, dict_2, log_level: int = logging.INFO):
     """
     Takes two flattened dictionaries and compares them. This function only compares those
     keys that are common to both dictionaries. The key-value pairs that are unique to one
@@ -49,13 +49,14 @@ def log_differences(dict_1, dict_2):
         for name in sorted(mismatched):
             table.add_row(name, str(dict_1[name]), str(dict_2[name]))
 
-        LOGGER.info(capture_rich_output(table))
-        # rich.print(table)
+        logger.log(log_level, capture_rich_output(table))
     else:
-        LOGGER.info(f"No differences between the two flattened dictionaries, {len(all_keys)} values compared.")
+        logger.log(
+            log_level, f"No differences between the two flattened dictionaries, {len(all_keys)} values compared."
+        )
 
 
-def log_key_differences(dict_1, dict_2):
+def log_key_differences(dict_1, dict_2, log_level: int = logging.INFO):
     """
     Takes two dictionaries and compares the top-level keys. The differences are logged in a Rich Table at level=INFO.
     Keys that are present on both dictionaries are not logged.
@@ -81,7 +82,7 @@ def log_key_differences(dict_1, dict_2):
     not_in_s1 = s2 - s1
 
     if not not_in_s1 and not not_in_s2:
-        LOGGER.info("Both dictionaries contains the same keys.")
+        logger.log(log_level, "Both dictionaries contains the same keys.")
 
     table = Table("Dictionary 1", "Dictionary 2", title="Key differences", title_justify="left")
 
@@ -91,11 +92,11 @@ def log_key_differences(dict_1, dict_2):
     for key in not_in_s1:
         table.add_row("", str(key))
 
-    LOGGER.info(capture_rich_output(table))
+    logger.log(log_level, capture_rich_output(table))
 
 
 if __name__ == "__main__":
-    import egse.logger  # noqa activates the logger
+    logging.basicConfig(level=logging.DEBUG)
 
     d1 = {
         "A": 1,
@@ -115,7 +116,7 @@ if __name__ == "__main__":
         "C": 3,
     }
 
-    log_differences(d1, d2)
-    log_key_differences(d1, d2)
+    log_differences(d1, d2, logging.DEBUG)
+    log_key_differences(d1, d2, logging.DEBUG)
 
-    log_differences(d1, d3)
+    log_differences(d1, d3, logging.DEBUG)

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/env.py

@@ -63,11 +63,11 @@ __all__ = [
 ]
 
 import contextlib
-import logging
 import os
 import warnings
 from pathlib import Path
 
+from egse.log import logger
 from egse.system import all_logging_disabled
 from egse.system import get_caller_info
 from egse.system import ignore_m_warning
@@ -75,7 +75,6 @@ from egse.system import ignore_m_warning
 from rich.console import Console
 
 console = Console(width=100)
-logger = logging.getLogger("egse.env")
 
 # Every project shall have a PROJECT and a SITE_ID environment variable set. This variable will be used to
 # create the other environment variables that are specific to the project.
@@ -484,7 +483,7 @@ def get_local_settings_path() -> str or None:
 
 def has_conf_repo_location() -> bool:
     location = _env.get("CONF_REPO_LOCATION")
-    return True if location else False
+    return True if location in (None, NoValue) else False
 
 
 def get_conf_repo_location_env_name() -> str:

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/heartbeat.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import contextlib
-import logging
 import pickle
 import queue
 import random
@@ -13,8 +12,6 @@ import zmq
 from egse.system import format_datetime
 from egse.zmq_ser import MessageIdentifier
 
-_LOGGER = logging.getLogger(__name__)
-
 
 def _g_tick(period: float):
     """Generator for ticks every period [s]."""

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/hk.py

@@ -10,7 +10,6 @@ __all__ = [
 ]
 import csv
 import datetime
-import logging
 from enum import Enum
 from pathlib import Path
 from typing import Optional
@@ -19,9 +18,11 @@ from typing import Union
 import dateutil.parser as date_parser
 import numpy as np
 import pandas as pd
+
 from egse.config import find_files
 from egse.env import get_data_storage_location
 from egse.env import get_site_id
+from egse.log import logger
 from egse.obsid import ObservationIdentifier
 from egse.obsid import obsid_from_storage
 from egse.setup import Setup
@@ -33,8 +34,6 @@ from egse.system import read_last_lines
 from egse.system import str_to_datetime
 from egse.system import time_since_epoch_1958
 
-_LOGGER = logging.getLogger(__name__)
-
 
 class TmDictionaryColumns(str, Enum):
     """Enumeration of the relevant columns in the TM dictionary spreadsheet.
@@ -438,7 +437,7 @@ def _get_housekeeping_daily(hk_name: str, data_dir, time_window: int = None, set
         # (those will have to be read entirely)
 
         else:
-            _LOGGER.warning(f"No HK available for {origin} on {start_time.day}/{start_time.month}/{start_time.year}")
+            logger.warning(f"No HK available for {origin} on {start_time.day}/{start_time.month}/{start_time.year}")
 
         day = (start_time + datetime.timedelta(days=1)).date()  # The day after the first day
         last_day = datetime.date(now.year, now.month, now.day)  # Today
@@ -463,7 +462,7 @@ def _get_housekeeping_daily(hk_name: str, data_dir, time_window: int = None, set
                         hk_array = np.append(hk_array, row[hk_index])
 
             else:
-                _LOGGER.warning(f"No HK available for {origin} on {day.day}/{day.month}/{day.year}")
+                logger.warning(f"No HK available for {origin} on {day.day}/{day.month}/{day.year}")
 
             day += datetime.timedelta(days=1)
 
@@ -673,7 +672,7 @@ def read_conversion_dict(storage_mnemonic: str, use_site: bool = False, setup: O
 
     else:
         if len(original_name_col) != len(correct_name_col):
-            _LOGGER.error(
+            logger.error(
                 f"Name columns in TM dictionary have different length: "
                 f"{len(original_name_col)} != {len(correct_name_col)}"
             )

cgse-0.14.0/libs/cgse-common/src/egse/log.py

@@ -0,0 +1,132 @@
+"""
+Configure local logging for the CGSE.
+
+I use 'local logging' here because the CGSE also has a Logger server which stores
+all CGSE log messages in a rotated file. That server is part of the `cgse-core` package.
+
+Environment variables that affect logging:
+
+  - LOG_FORMAT: full | FULL
+  - LOG_LEVEL: an integer [10, 50] or a level name DEBUG, INFO, WARNING, CRITICAL, ERROR
+
+"""
+
+__all__ = [
+    "LOG_FORMAT_FULL",
+    "logger",
+    "root_logger",
+    "egse_logger",
+]
+
+import logging
+import os
+import textwrap
+
+import rich
+
+LOG_FORMAT_STYLE = "{"
+LOG_FORMAT_FULL = (
+    "{asctime:19s}.{msecs:03.0f} : {processName:20s} : {levelname:8s} : {name:^25s} : {lineno:6d} : {filename:20s} : {"
+    "message}"
+)
+LOG_FORMAT_CLEAN = "{asctime} [{levelname:>8s}] {message} ({filename}:{lineno:d})"
+
+LOG_DATE_FORMAT_FULL = "%Y-%m-%d %H:%M:%S"
+LOG_DATE_FORMAT_CLEAN = "%Y-%m-%d %H:%M:%S"
+
+
+class EGSEFilter(logging.Filter):
+    def filter(self, record):
+        return record.name.startswith("egse")
+
+
+egse_filter = EGSEFilter()
+
+
+class NonEGSEFilter(logging.Filter):
+    def filter(self, record):
+        return not record.name.startswith("egse")
+
+
+root_filter = NonEGSEFilter()
+
+
+def get_log_level_from_env(env_var: str = "LOG_LEVEL", default: int = logging.INFO):
+    """Read the log level from an environment variable."""
+    log_level_str = os.getenv(env_var, default)
+
+    # Try to convert to integer first (for numeric levels)
+    try:
+        log_level = int(log_level_str)
+
+        if 10 <= log_level <= 50:
+            return log_level
+        else:
+            logging.warning(
+                f"Log level {log_level} outside standard range (10-50). Using {logging.getLevelName(default)}."
+            )
+            return default
+
+    except ValueError:
+        log_level_str = log_level_str.upper()
+        try:
+            return getattr(logging, log_level_str)
+        except AttributeError:
+            logging.error(f"Invalid LOG_LEVEL '{log_level_str}'. Using {logging.getLevelName(default)}.")
+            return default
+
+
+egse_logger = logging.getLogger("egse")
+egse_logger.level = get_log_level_from_env()  # We might want to choose another env e.g. CGSE_LOG_LEVEL
+
+root_logger = logging.getLogger()
+root_logger.level = get_log_level_from_env()
+
+egse_handler = logging.StreamHandler()
+if os.getenv("LOG_FORMAT", "").lower() == "full":
+    egse_formatter = logging.Formatter(fmt=LOG_FORMAT_FULL, datefmt=LOG_DATE_FORMAT_FULL, style=LOG_FORMAT_STYLE)
+else:
+    egse_formatter = logging.Formatter(fmt=LOG_FORMAT_CLEAN, datefmt=LOG_DATE_FORMAT_CLEAN, style=LOG_FORMAT_STYLE)
+
+egse_handler.setFormatter(egse_formatter)
+egse_handler.addFilter(egse_filter)
+
+root_logger.addHandler(egse_handler)
+
+for handler in root_logger.handlers:
+    if handler != egse_handler:  # Don't filter our new handler
+        handler.addFilter(root_filter)
+
+logger = egse_logger
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=get_log_level_from_env(), format=LOG_FORMAT_CLEAN, datefmt=LOG_DATE_FORMAT_CLEAN, style="{"
+    )
+
+    root_logger = logging.getLogger()
+
+    rich.print(
+        textwrap.dedent(
+            """
+            Example logging statements
+              - logging level set to INFO
+              - fields are separated by a colon ':'
+              - fields: date & time: process name : level : logger name : lineno : filename : message
+            """
+        )
+    )
+
+    if os.getenv("LOG_FORMAT_FULL") == "true":
+        rich.print(
+            f"[b]{'Date & Time':^23s} : {'Process Name':20s} : {'Level':8s} : {'Logger Name':^25s} : {' Line '} : "
+            f"{'Filename':20s} : {'Message'}[/]"
+        )
+    else:
+        rich.print(f"[b]{'Date & Time':^19s} [ Level  ] Message (filename:lineno)[/]")
+
+    rich.print("-" * 150)
+    for name, level in logging.getLevelNamesMapping().items():
+        logger.log(level, f"{name} logging message")
+
+    root_logger.info("This should come out of the root logger, not the egse logger.")

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/metrics.py

@@ -5,8 +5,6 @@ __all__ = [
     "get_metrics_repo",
 ]
 
-import logging
-from dataclasses import dataclass
 from typing import Any
 from typing import Optional
 from typing import Protocol
@@ -16,6 +14,7 @@ import numpy as np
 from prometheus_client import Gauge
 
 from egse.hk import TmDictionaryColumns
+from egse.log import logger
 from egse.plugin import load_plugins_fn
 from egse.settings import Settings
 from egse.setup import Setup
@@ -24,8 +23,6 @@ from egse.setup import load_setup
 from egse.system import format_datetime
 from egse.system import str_to_datetime
 
-logger = logging.getLogger("egse.metrics")
-
 SITE_ID = Settings.load("SITE").ID
 
 

{cgse-0.13.1 → cgse-0.14.0}/libs/cgse-common/src/egse/plugin.py

@@ -11,7 +11,6 @@ __all__ = [
 ]
 
 import importlib.util
-import logging
 import os
 import sys
 import textwrap
@@ -20,6 +19,10 @@ import types
 from functools import lru_cache
 from pathlib import Path
 
+import click
+import rich
+
+from egse.log import logger
 from egse.system import type_name
 
 if sys.version_info >= (3, 12):  # Use the standard library version (Python 3.10+)
@@ -30,10 +33,7 @@ else:
     from importlib_metadata import entry_points as lib_entry_points
     from importlib_metadata import EntryPoint
 
-import click
-import rich
 
-logger = logging.getLogger(__name__)
 HERE = Path(__file__).parent
 
 
@@ -201,7 +201,7 @@ class HierarchicalEntryPoints:
     def _discover_groups(self):
         """Discover all groups that match the base group pattern."""
         all_eps = lib_entry_points()
-        # rich.print(f"{type(all_eps) = }")
+        # print(f"{type(all_eps) = }")
 
         self.groups = {}
         self.flat_entries = []
@@ -209,14 +209,18 @@ class HierarchicalEntryPoints:
         for ep in all_eps:
             # rich.print(f"{type(ep) = }, {dir(ep) = }")
             if ep.group == self.base_group or ep.group.startswith(f"{self.base_group}."):
-                self.groups[ep.group] = ep
+                # print(f"{ep.group = }, {ep = }")
+                if ep.group in self.groups:
+                    self.groups[ep.group].append(ep)
+                else:
+                    self.groups[ep.group] = [ep]
                 self.flat_entries.append(ep)
 
-    def get_all_entry_points(self):
+    def get_all_entry_points(self) -> list:
         """Get all entry points as a flat list."""
         return self.flat_entries
 
-    def get_by_subgroup(self, subgroup=None):
+    def get_by_subgroup(self, subgroup=None) -> list:
         """Get entry points from a specific subgroup."""
         if subgroup is None:
             return self.groups.get(self.base_group, [])
@@ -224,11 +228,11 @@ class HierarchicalEntryPoints:
         full_group = f"{self.base_group}.{subgroup}"
         return self.groups.get(full_group, [])
 
-    def get_all_groups(self):
+    def get_all_groups(self) -> list:
         """Get all discovered group names."""
         return list(self.groups.keys())
 
-    def get_by_type(self, entry_type):
+    def get_by_type(self, entry_type) -> list:
         """Get entry points by type (assuming type is the subgroup name)."""
         return self.get_by_subgroup(entry_type)