shepherd-core 2023.12.1__py3-none-any.whl → 2024.4.2__py3-none-any.whl

shepherd_core/fw_tools/validation.py

@@ -1,6 +1,10 @@
-"""TODO: Work in Progress
+"""Helper-functions for firmware-validation.
 
+TODO: Work in Progress.
+    - Each arch should have its own file and
+    - detection-functions that register in main validator.
 """
+
 import tempfile
 from pathlib import Path
 
@@ -26,6 +30,7 @@ except ImportError as e:
 
 @validate_call
 def is_hex(file: Path) -> bool:
+    """Check if file is containing intel-hex data."""
     try:
         _ = IntelHex(file.as_posix())
     except ValueError:  # parsing
@@ -36,7 +41,9 @@ def is_hex(file: Path) -> bool:
 
 
 def is_hex_msp430(file: Path) -> bool:
-    """Observations:
+    """Try to detect specifics for that MCU.
+
+    Observations:
     - addresses begin at 0x4000
     - value @0xFFFE (IVT) is start_address (of pgm-code)
     """
@@ -57,7 +64,9 @@ def is_hex_msp430(file: Path) -> bool:
 
 
 def is_hex_nrf52(file: Path) -> bool:
-    """Observations:
+    """Try to detect specifics for that MCU.
+
+    Observations:
     - addresses begin at 0x0
     - only one segment (.get_segments), todo
     """
@@ -80,6 +89,7 @@ def is_hex_nrf52(file: Path) -> bool:
 
 @validate_call
 def is_elf(file: Path) -> bool:
+    """Check if file is an ELF file."""
     if ELF is None:
         raise RuntimeError(elf_error_text)
     if not file.is_file():
@@ -92,7 +102,11 @@ def is_elf(file: Path) -> bool:
     return True
 
 
-def is_elf_msp430(file: Path) -> bool:  # TODO: allow detection without conversion
+def is_elf_msp430(file: Path) -> bool:
+    """Check if file is an ELF for that MCU.
+
+    TODO: allow detection without conversion
+    """
     if is_elf(file):
         with tempfile.TemporaryDirectory() as path:
             file_hex = Path(path) / "file.hex"
@@ -103,7 +117,11 @@ def is_elf_msp430(file: Path) -> bool:  # TODO: allow detection without conversi
     return False
 
 
-def is_elf_nrf52(file: Path) -> bool:  # TODO: allow detection without conversion
+def is_elf_nrf52(file: Path) -> bool:
+    """Check if file is an ELF for that MCU.
+
+    TODO: allow detection without conversion
+    """
     if is_elf(file):
         with tempfile.TemporaryDirectory() as path:
             file_hex = Path(path) / "file.hex"
@@ -115,6 +133,7 @@ def is_elf_nrf52(file: Path) -> bool:  # TODO: allow detection without conversio
 
 
 def determine_type(file: Path) -> FirmwareDType:
+    """Figure out file-type (hex or elf)."""
     if not file.is_file():
         raise ValueError("Fn needs an existing file as input")
     if is_hex(file):
@@ -125,6 +144,7 @@ def determine_type(file: Path) -> FirmwareDType:
 
 
 def determine_arch(file: Path) -> str:
+    """Figure out arch (msp430 or nrf52)."""
     file_t = determine_type(file)
     if file_t == FirmwareDType.path_elf:
         if is_elf_nrf52(file):

shepherd_core/inventory/__init__.py

@@ -1,8 +1,13 @@
-"""Creates an overview for shepherd-host-machines with:
+"""Creates an overview for shepherd-host-machines.
+
+This will collect:
 - relevant software-versions
 - system-parameters
-- hardware-config
+- hardware-config.
 """
+
+from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from typing import List
 
@@ -25,25 +30,39 @@ __all__ = [
 
 
 class Inventory(PythonInventory, SystemInventory, TargetInventory):
-    # has all child-parameters
+    """Complete inventory for one device.
+
+    Has all child-parameters.
+    """
+
+    hostname: str
+    created: datetime
 
     @classmethod
     def collect(cls) -> Self:
         # one by one for more precise error messages
-        pid = PythonInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
+        # NOTE: system is first, as it must take a precise timestamp
         sid = SystemInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
+        pid = PythonInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
         tid = TargetInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
         model = {**pid, **sid, **tid}
+        # make important metadata available at root level
+        model["created"] = sid["timestamp"]
+        model["hostname"] = sid["hostname"]
         return cls(**model)
 
 
 class InventoryList(ShpModel):
+    """Collection of inventories for several devices."""
+
     elements: Annotated[List[Inventory], Field(min_length=1)]
 
     def to_csv(self, path: Path) -> None:
-        """TODO: pretty messed up (raw lists and dicts for sub-elements)
+        """Generate a CSV.
+
+        TODO: pretty messed up (raw lists and dicts for sub-elements)
         numpy.savetxt -> too basic
-        np.concatenate(content).reshape((len(content), len(content[0])))
+        np.concatenate(content).reshape((len(content), len(content[0]))).
         """
         if path.is_dir():
             path = path / "inventory.yaml"
@@ -53,3 +72,44 @@ class InventoryList(ShpModel):
                 content = list(item.model_dump().values())
                 content = ["" if value is None else str(value) for value in content]
                 fd.write(", ".join(content) + "\r\n")
+
+    def warn(self) -> dict:
+        warnings = {}
+        ts_earl = min([_e.created.timestamp() for _e in self.elements])
+        for _e in self.elements:
+            if _e.uptime > timedelta(hours=30).total_seconds():
+                warnings["uptime"] = f"[{self.hostname}] restart is recommended"
+            if (_e.created.timestamp() - ts_earl) > 10:
+                warnings["time_delta"] = f"[{self.hostname}] time-sync has failed"
+
+        # turn  dict[hostname][type] = val
+        # to    dict[type][val] = list[hostnames]
+        _inp = {
+            _e.hostname: _e.model_dump(exclude_unset=True, exclude_defaults=True)
+            for _e in self.elements
+        }
+        result = {}
+        for _host, _types in _inp.items():
+            for _type, _val in _types.items():
+                if _type not in result:
+                    result[_type] = {}
+                if _val not in result[_type]:
+                    result[_type][_val] = []
+                result[_type][_val].append(_host)
+        rescnt = {_key: len(_val) for _key, _val in result.items()}
+        t_unique = [
+            "h5py",
+            "numpy",
+            "pydantic",
+            "python",
+            "shepherd_core",
+            "shepherd_sheep",
+            "yaml",
+            "zstandard",
+        ]
+        for _key in t_unique:
+            if rescnt[_key] > 1:
+                warnings[_key] = f"[{_key}] VersionMismatch - {result[_key]}"
+
+        # TODO: finish with more potential warnings
+        return warnings

shepherd_core/inventory/python.py

@@ -1,3 +1,5 @@
+"""Python related inventory model."""
+
 import platform
 from contextlib import suppress
 from importlib import import_module
@@ -10,6 +12,8 @@ from ..data_models import ShpModel
 
 
 class PythonInventory(ShpModel):
+    """Python related inventory model."""
+
     # program versions
     h5py: Optional[str] = None
     numpy: Optional[str] = None

shepherd_core/inventory/system.py

@@ -1,12 +1,16 @@
+"""System / OS related inventory model."""
+
 import platform
 import subprocess
 import time
 from contextlib import suppress
+from datetime import datetime
 from typing import Optional
 
 from typing_extensions import Self
 
-from .. import logger
+from ..data_models.base.timezone import local_now
+from ..logger import logger
 
 try:
     import psutil
@@ -20,8 +24,13 @@ from ..data_models import ShpModel
 
 
 class SystemInventory(ShpModel):
+    """System / OS related inventory model."""
+
     uptime: PositiveInt
     # ⤷ seconds
+    timestamp: datetime
+    # time_delta: timedelta = timedelta(0)  # noqa: ERA001
+    # ⤷ lag behind earliest observer, TODO: wrong place
 
     system: str
     release: str
@@ -43,6 +52,8 @@ class SystemInventory(ShpModel):
 
     @classmethod
     def collect(cls) -> Self:
+        ts = local_now()
+
         if psutil is None:
             ifs2 = {}
             uptime = 0
@@ -58,6 +69,7 @@ class SystemInventory(ShpModel):
 
         model_dict = {
             "uptime": round(uptime),
+            "timestamp": ts,
             "system": platform.system(),
             "release": platform.release(),
             "version": platform.version(),

shepherd_core/inventory/target.py

@@ -1,3 +1,5 @@
+"""Hardware related inventory model."""
+
 from typing import List
 from typing import Optional
 
@@ -8,6 +10,8 @@ from ..data_models import ShpModel
 
 
 class TargetInventory(ShpModel):
+    """Hardware related inventory model."""
+
     cape: Optional[str] = None
     targets: List[str] = []  # noqa: RUF012
 

shepherd_core/logger.py

@@ -1,3 +1,5 @@
+"""Log handler of shepherd."""
+
 import logging
 import logging.handlers
 from typing import Union
@@ -12,10 +14,12 @@ verbose_level: int = 2
 
 
 def get_verbose_level() -> int:
+    """Get log level of shepherd."""
     return verbose_level
 
 
 def set_log_verbose_level(log_: Union[logging.Logger, logging.Handler], verbose: int) -> None:
+    """Set log level of shepherd."""
     if verbose == 0:
         log_.setLevel(logging.ERROR)
         logging.basicConfig(level=logging.ERROR)
@@ -39,6 +43,7 @@ def set_log_verbose_level(log_: Union[logging.Logger, logging.Handler], verbose:
 
 
 def increase_verbose_level(verbose: int) -> None:
+    """Increase log level of shepherd."""
     global verbose_level  # noqa: PLW0603
     if verbose >= verbose_level:
         verbose_level = min(max(verbose, 0), 3)

shepherd_core/reader.py

@@ -1,6 +1,7 @@
-"""
-Reader-Baseclass
-"""
+"""Reader-Baseclass."""
+
+from __future__ import annotations
+
 import contextlib
 import errno
 import logging
@@ -8,7 +9,7 @@ import math
 import os
 from itertools import product
 from pathlib import Path
-from types import TracebackType
+from typing import TYPE_CHECKING
 from typing import Any
 from typing import ClassVar
 from typing import Dict
@@ -31,13 +32,18 @@ from .data_models.base.calibration import CalibrationSeries
 from .data_models.content.energy_environment import EnergyDType
 from .decoder_waveform import Uart
 
+if TYPE_CHECKING:
+    from types import TracebackType
+
 
 class Reader:
     """Sequentially Reads shepherd-data from HDF5 file.
 
     Args:
+    ----
         file_path: Path of hdf5 file containing shepherd data with iv-samples, iv-curves or isc&voc
         verbose: more debug-info during usage, 'None' skips the setter
+
     """
 
     samples_per_buffer: int = 10_000
@@ -95,7 +101,8 @@ class Reader:
                 self.h5file = h5py.File(self.file_path, "r")  # = readonly
                 self._reader_opened = True
             except OSError as _xcp:
-                raise TypeError(f"Unable to open HDF5-File '{self.file_path.name}'") from _xcp
+                msg = f"Unable to open HDF5-File '{self.file_path.name}'"
+                raise TypeError(msg) from _xcp
 
             if self.is_valid():
                 self._logger.debug("File is available now")
@@ -158,7 +165,7 @@ class Reader:
         )
 
     def _refresh_file_stats(self) -> None:
-        """update internal states, helpful after resampling or other changes in data-group"""
+        """Update internal states, helpful after resampling or other changes in data-group."""
         self.h5file.flush()
         if (self.ds_time.shape[0] > 1) and (self.ds_time[1] != self.ds_time[0]):
             # this assumes isochronal sampling
@@ -180,17 +187,20 @@ class Reader:
         is_raw: bool = False,
         omit_ts: bool = False,
     ) -> Generator[tuple, None, None]:
-        """Generator that reads the specified range of buffers from the hdf5 file.
-        can be configured on first call
-        TODO: reconstruct - start/end mark samples and
-            each call can request a certain number of samples
+        """Read the specified range of buffers from the hdf5 file.
+
+        Generator - can be configured on first call
+        TODO: reconstruct - start/end mark samples &
+         each call can request a certain number of samples.
 
         Args:
+        ----
             :param start_n: (int) Index of first buffer to be read
             :param end_n: (int) Index of last buffer to be read
             :param is_raw: (bool) output original data, not transformed to SI-Units
             :param omit_ts: (bool) optimize reading if timestamp is never used
         Yields: Buffers between start and end (tuple with time, voltage, current)
+
         """
         if end_n is None:
             end_n = int(self.ds_voltage.shape[0] // self.samples_per_buffer)
@@ -215,7 +225,7 @@ class Reader:
                 )
 
     def get_calibration_data(self) -> CalibrationSeries:
-        """Reads calibration-data from hdf5 file.
+        """Read calibration-data from hdf5 file.
 
         :return: Calibration data as CalibrationSeries object
         """
@@ -245,12 +255,14 @@ class Reader:
         try:
             if "datatype" in self.h5file["data"].attrs:
                 return EnergyDType[self.h5file["data"].attrs["datatype"]]
-            return None
         except KeyError:
             return None
+        else:
+            return None
 
     def get_hrv_config(self) -> dict:
-        """essential info for harvester
+        """Essential info for harvester.
+
         :return: config-dict directly for vHarvester to be used during emulation
         """
         return {
@@ -259,7 +271,7 @@ class Reader:
         }
 
     def is_valid(self) -> bool:
-        """checks file for plausibility
+        """Check file for plausibility, validity / correctness.
 
         :return: state of validity
         """
@@ -387,7 +399,7 @@ class Reader:
         return True
 
     def __getitem__(self, key: str) -> Any:
-        """returns attribute or (if none found) a handle for a group or dataset (if found)
+        """Query attribute or (if none found) a handle for a group or dataset (if found).
 
         :param key: attribute, group, dataset
         :return: value of that key, or handle of object
@@ -399,9 +411,10 @@ class Reader:
         raise KeyError
 
     def energy(self) -> float:
-        """determine the recorded energy of the trace
+        """Determine the recorded energy of the trace.
+
         # multiprocessing: https://stackoverflow.com/a/71898911
-        # -> failed with multiprocessing.pool and pathos.multiprocessing.ProcessPool
+        # -> failed with multiprocessing.pool and pathos.multiprocessing.ProcessPool.
 
         :return: sampled energy in Ws (watt-seconds)
         """
@@ -427,7 +440,8 @@ class Reader:
     def _dset_statistics(
         self, dset: h5py.Dataset, cal: Optional[CalibrationPair] = None
     ) -> Dict[str, float]:
-        """some basic stats for a provided dataset
+        """Create basic stats for a provided dataset.
+
         :param dset: dataset to evaluate
         :param cal: calibration (if wanted)
         :return: dict with entries for mean, min, max, std
@@ -469,8 +483,9 @@ class Reader:
         return stats
 
     def _data_timediffs(self) -> List[float]:
-        """calculate list of (unique) time-deltas between buffers [s]
-            -> optimized version that only looks at the start of each buffer
+        """Calculate list of unique time-deltas [s] between buffers.
+
+        Optimized version that only looks at the start of each buffer.
 
         :return: list of (unique) time-deltas between buffers [s]
         """
@@ -500,8 +515,9 @@ class Reader:
         return list(diffs)
 
     def check_timediffs(self) -> bool:
-        """validate equal time-deltas
-        -> unexpected time-jumps hint at a corrupted file or faulty measurement
+        """Validate equal time-deltas.
+
+        Unexpected time-jumps hint at a corrupted file or faulty measurement.
 
         :return: True if OK
         """
@@ -529,7 +545,8 @@ class Reader:
         *,
         minimal: bool = False,
     ) -> Dict[str, dict]:
-        """recursive FN to capture the structure of the file
+        """Recursive FN to capture the structure of the file.
+
         :param node: starting node, leave free to go through whole file
         :param minimal: just provide a bare tree (much faster)
         :return: structure of that node with everything inside it
@@ -580,7 +597,7 @@ class Reader:
         return metadata
 
     def save_metadata(self, node: Union[h5py.Dataset, h5py.Group, None] = None) -> dict:
-        """get structure of file and dump content to yaml-file with same name as original
+        """Get structure of file and dump content to yaml-file with same name as original.
 
         :param node: starting node, leave free to go through whole file
         :return: structure of that node with everything inside it
@@ -609,8 +626,9 @@ class Reader:
 
     @staticmethod
     def get_filter_for_redundant_states(data: np.ndarray) -> np.ndarray:
-        """input is 1D state-vector, kep only first from identical & sequential states
-        algo: create an offset-by-one vector and compare against original
+        """Input is 1D state-vector, kep only first from identical & sequential states.
+
+        Algo: create an offset-by-one vector and compare against original.
         """
         if len(data.shape) > 1:
             ValueError("Array must be 1D")

shepherd_core/testbed_client/__init__.py

@@ -1,3 +1,5 @@
+"""Client to access a testbed-instance for controlling experiments."""
+
 from .client import tb_client
 from .user_model import User
 

shepherd_core/testbed_client/cache_path.py

@@ -0,0 +1,17 @@
+"""Generalized path for the file-cache."""
+
+import os
+from pathlib import Path
+
+
+def _get_xdg_path(variable_name: str, default_path: Path) -> Path:
+    _value = os.environ.get(variable_name)
+    if _value is None or _value == "":
+        return default_path
+    return Path(_value)
+
+
+user_path = Path("~").expanduser()
+
+cache_xdg_path = _get_xdg_path("XDG_CACHE_HOME", user_path / ".cache")
+cache_user_path = cache_xdg_path / "shepherd_datalib"

shepherd_core/testbed_client/client.py

@@ -1,3 +1,5 @@
+"""Client-Class to access a testbed instance."""
+
 from importlib import import_module
 from pathlib import Path
 from typing import Optional
@@ -16,6 +18,8 @@ from .user_model import User
 
 
 class TestbedClient:
+    """Client-Class to access a testbed instance."""
+
     _instance: Optional[Self] = None
 
     def __init__(self, server: Optional[str] = None, token: Union[str, Path, None] = None) -> None:
@@ -41,9 +45,10 @@ class TestbedClient:
 
     @validate_call
     def connect(self, server: Optional[str] = None, token: Union[str, Path, None] = None) -> bool:
-        """
+        """Establish connection to testbed-server.
+
         server: either "local" to use demo-fixtures or something like "https://HOST:PORT"
-        token: your account validation
+        token: your account validation.
         """
         if isinstance(token, Path):
             with token.resolve().open() as file:
@@ -78,19 +83,19 @@ class TestbedClient:
 
     def query_ids(self, model_type: str) -> list:
         if self._connected:
-            raise RuntimeError("Not Implemented, TODO")
+            raise NotImplementedError("TODO")
         return list(self._fixtures[model_type].elements_by_id.keys())
 
     def query_names(self, model_type: str) -> list:
         if self._connected:
-            raise RuntimeError("Not Implemented, TODO")
+            raise NotImplementedError("TODO")
         return list(self._fixtures[model_type].elements_by_name.keys())
 
     def query_item(
         self, model_type: str, uid: Optional[int] = None, name: Optional[str] = None
     ) -> dict:
         if self._connected:
-            raise RuntimeError("Not Implemented, TODO")
+            raise NotImplementedError("TODO")
         if uid is not None:
             return self._fixtures[model_type].query_id(uid)
         if name is not None:
@@ -116,11 +121,11 @@ class TestbedClient:
 
     def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
         if self._connected:
-            raise RuntimeError("Not Implemented, TODO")
+            raise NotImplementedError("TODO")
         return self._fixtures[model_type].inheritance(values)
 
     def try_completing_model(self, model_type: str, values: dict) -> (dict, list):
-        """init by name/id, for none existing instances raise Exception"""
+        """Init by name/id, for none existing instances raise Exception."""
         if len(values) == 1 and next(iter(values.keys())) in {"id", "name"}:
             value = next(iter(values.values()))
             if (
@@ -132,7 +137,8 @@ class TestbedClient:
                 # TODO: still depending on _fixture
                 values = self.query_item(model_type, uid=value)
             else:
-                raise ValueError(f"Query {model_type} by name / ID failed - {values} is unknown!")
+                msg = f"Query {model_type} by name / ID failed - {values} is unknown!"
+                raise ValueError(msg)
         return self.try_inheritance(model_type, values)
 
     def fill_in_user_data(self, values: dict) -> dict: