shepherd-core 2025.4.1__py3-none-any.whl → 2025.5.2__py3-none-any.whl

shepherd_core/reader.py

@@ -9,14 +9,11 @@ import math
 import os
 from itertools import product
 from pathlib import Path
+from types import MappingProxyType
 from typing import TYPE_CHECKING
+from typing import Annotated
 from typing import Any
-from typing import ClassVar
-from typing import Dict
-from typing import Generator
-from typing import List
 from typing import Optional
-from typing import Type
 from typing import Union
 
 import h5py
@@ -25,14 +22,18 @@ import yaml
 from pydantic import validate_call
 from tqdm import trange
 from typing_extensions import Self
+from typing_extensions import deprecated
 
-from .commons import samplerate_sps_default
+from .commons import SAMPLERATE_SPS_DEFAULT
 from .data_models.base.calibration import CalibrationPair
 from .data_models.base.calibration import CalibrationSeries
 from .data_models.content.energy_environment import EnergyDType
 from .decoder_waveform import Uart
 
 if TYPE_CHECKING:
+    from collections.abc import Generator
+    from collections.abc import Mapping
+    from collections.abc import Sequence
     from types import TracebackType
 
 
@@ -46,28 +47,27 @@ class Reader:
 
     """
 
-    samples_per_buffer: int = 10_000
+    CHUNK_SAMPLES_N: int = 10_000
 
-    mode_dtype_dict: ClassVar[dict] = {
-        "harvester": [
-            EnergyDType.ivsample,
-            EnergyDType.ivcurve,
-            EnergyDType.isc_voc,
-        ],
-        "emulator": [EnergyDType.ivsample],
-    }
+    MODE_TO_DTYPE: Mapping[str, Sequence[EnergyDType]] = MappingProxyType(
+        {
+            "harvester": (
+                EnergyDType.ivsample,
+                EnergyDType.ivcurve,
+                EnergyDType.isc_voc,
+            ),
+            "emulator": (EnergyDType.ivsample,),
+        }
+    )
 
     @validate_call
     def __init__(
         self,
-        file_path: Optional[Path],
+        file_path: Path,
         *,
-        verbose: Optional[bool] = True,
+        verbose: bool = True,
     ) -> None:
-        if not hasattr(self, "file_path"):
-            self.file_path: Optional[Path] = None
-            if isinstance(file_path, (Path, str)):
-                self.file_path = Path(file_path).resolve()
+        self.file_path: Path = file_path.resolve()
 
         if not hasattr(self, "_logger"):
             self._logger: logging.Logger = logging.getLogger("SHPCore.Reader")
@@ -75,7 +75,7 @@ class Reader:
             self._logger.setLevel(logging.DEBUG if verbose else logging.INFO)
 
         if not hasattr(self, "samplerate_sps"):
-            self.samplerate_sps: int = samplerate_sps_default
+            self.samplerate_sps: int = SAMPLERATE_SPS_DEFAULT
         self.sample_interval_ns: int = round(10**9 // self.samplerate_sps)
         self.sample_interval_s: float = 1 / self.samplerate_sps
 
@@ -84,10 +84,13 @@ class Reader:
 
         # init stats
         self.runtime_s: float = 0
-        self.buffers_n: int = 0
+        self.samples_n: int = 0
+        self.chunks_n: int = 0
         self.file_size: int = 0
         self.data_rate: float = 0
 
+        self.buffers_n: Annotated[int, deprecated("use .chunk_n instead")] = 0
+
         # open file (if not already done by writer)
         self._reader_opened: bool = False
         if not hasattr(self, "h5file"):
@@ -115,7 +118,8 @@ class Reader:
                 )
 
         if not isinstance(self.h5file, h5py.File):
-            raise TypeError("Type of opened file is not h5py.File, for %s", self.file_path.name)
+            msg = (f"Type of opened file is not h5py.File, for {self.file_path.name}",)
+            raise TypeError(msg)
 
         self.ds_time: h5py.Dataset = self.h5file["data"]["time"]
         self.ds_voltage: h5py.Dataset = self.h5file["data"]["voltage"]
@@ -155,7 +159,7 @@ class Reader:
 
     def __exit__(
         self,
-        typ: Optional[Type[BaseException]] = None,
+        typ: Optional[type[BaseException]] = None,
         exc: Optional[BaseException] = None,
         tb: Optional[TracebackType] = None,
         extra_arg: int = 0,
@@ -171,59 +175,63 @@ class Reader:
     def _refresh_file_stats(self) -> None:
         """Update internal states, helpful after resampling or other changes in data-group."""
         self.h5file.flush()
-        sample_count = self.ds_time.shape[0]
+        self.samples_n = min(
+            self.ds_time.shape[0], self.ds_current.shape[0], self.ds_voltage.shape[0]
+        )
         duration_raw = (
-            (int(self.ds_time[sample_count - 1]) - int(self.ds_time[0])) if sample_count > 0 else 0
+            (int(self.ds_time[self.samples_n - 1]) - int(self.ds_time[0]))
+            if self.samples_n > 0
+            else 0
         )
         # above's typecasting prevents overflow in u64-format
-        if (sample_count > 0) and (duration_raw > 0):
-            # this assumes iso-chronous sampling
+        if (self.samples_n > 0) and (duration_raw > 0):
+            # this assumes iso-chronous sampling, TODO: not the best choice?
             duration_s = self._cal.time.raw_to_si(duration_raw)
-            self.sample_interval_s = duration_s / sample_count
+            self.sample_interval_s = duration_s / self.samples_n
             self.sample_interval_ns = round(10**9 * self.sample_interval_s)
-            self.samplerate_sps = max(round((sample_count - 1) / duration_s), 1)
-        self.runtime_s = round(self.ds_voltage.shape[0] / self.samplerate_sps, 1)
-        self.buffers_n = int(self.ds_voltage.shape[0] // self.samples_per_buffer)
+            self.samplerate_sps = max(round((self.samples_n - 1) / duration_s), 1)
+        self.runtime_s = round(self.samples_n / self.samplerate_sps, 1)
+        self.chunks_n = self.buffers_n = int(self.samples_n // self.CHUNK_SAMPLES_N)
         if isinstance(self.file_path, Path):
             self.file_size = self.file_path.stat().st_size
         else:
             self.file_size = 0
         self.data_rate = self.file_size / self.runtime_s if self.runtime_s > 0 else 0
 
-    def read_buffers(
+    def read(
         self,
         start_n: int = 0,
         end_n: Optional[int] = None,
-        n_samples_per_buffer: Optional[int] = None,
+        n_samples_per_chunk: Optional[int] = None,
         *,
         is_raw: bool = False,
-        omit_ts: bool = False,
+        omit_timestamps: bool = False,
     ) -> Generator[tuple, None, None]:
-        """Read the specified range of buffers from the hdf5 file.
+        """Read the specified range of chunks from the hdf5 file.
 
         Generator - can be configured on first call
 
         Args:
         ----
-            :param start_n: (int) Index of first buffer to be read
-            :param end_n: (int) Index of last buffer to be read
-            :param n_samples_per_buffer: (int) allows changing
+            :param start_n: (int) Index of first chunk to be read
+            :param end_n: (int) Index of last chunk to be read
+            :param n_samples_per_chunk: (int) allows changing
             :param is_raw: (bool) output original data, not transformed to SI-Units
-            :param omit_ts: (bool) optimize reading if timestamp is never used
-        Yields: Buffers between start and end (tuple with time, voltage, current)
+            :param omit_timestamps: (bool) optimize reading if timestamp is never used
+        Yields: chunks between start and end (tuple with time, voltage, current)
 
         """
-        if n_samples_per_buffer is None:
-            n_samples_per_buffer = self.samples_per_buffer
-        end_max = int(self.ds_voltage.shape[0] // n_samples_per_buffer)
+        if n_samples_per_chunk is None:
+            n_samples_per_chunk = self.CHUNK_SAMPLES_N
+        end_max = int(self.samples_n // n_samples_per_chunk)
         end_n = end_max if end_n is None else min(end_n, end_max)
-        self._logger.debug("Reading blocks %d to %d from source-file", start_n, end_n)
+        self._logger.debug("Reading chunk %d to %d from source-file", start_n, end_n)
         _raw = is_raw
-        _wts = not omit_ts
+        _wts = not omit_timestamps
 
         for i in range(start_n, end_n):
-            idx_start = i * n_samples_per_buffer
-            idx_end = idx_start + n_samples_per_buffer
+            idx_start = i * n_samples_per_chunk
+            idx_end = idx_start + n_samples_per_chunk
             if _raw:
                 yield (
                     self.ds_time[idx_start:idx_end] if _wts else None,
@@ -237,6 +245,24 @@ class Reader:
                     self._cal.current.raw_to_si(self.ds_current[idx_start:idx_end]),
                 )
 
+    @deprecated("use .read() instead")
+    def read_buffers(
+        self,
+        start_n: int = 0,
+        end_n: Optional[int] = None,
+        n_samples_per_buffer: Optional[int] = None,
+        *,
+        is_raw: bool = False,
+        omit_ts: bool = False,
+    ) -> Generator[tuple, None, None]:
+        return self.read(
+            start_n=start_n,
+            end_n=end_n,
+            n_samples_per_chunk=n_samples_per_buffer,
+            is_raw=is_raw,
+            omit_timestamps=omit_ts,
+        )
+
     def get_calibration_data(self) -> CalibrationSeries:
         """Read calibration-data from hdf5 file.
 
@@ -254,7 +280,7 @@ class Reader:
             return self.h5file.attrs["mode"]
         return ""
 
-    def get_config(self) -> Dict:
+    def get_config(self) -> dict:
         if "config" in self.h5file["data"].attrs:
             return yaml.safe_load(self.h5file["data"].attrs["config"])
         return {}
@@ -329,7 +355,7 @@ class Reader:
                     self.file_path.name,
                 )
                 return False
-            if self.h5file.attrs["mode"] not in self.mode_dtype_dict:
+            if self.h5file.attrs["mode"] not in self.MODE_TO_DTYPE:
                 self._logger.error(
                     "[FileValidation] unsupported mode '%s' in '%s'",
                     attr,
@@ -361,7 +387,7 @@ class Reader:
                         self.file_path.name,
                     )
                     return False
-        if self.get_datatype() not in self.mode_dtype_dict[self.get_mode()]:
+        if self.get_datatype() not in self.MODE_TO_DTYPE[self.get_mode()]:
             self._logger.error(
                 "[FileValidation] unsupported type '%s' for mode '%s'  in '%s'",
                 self.get_datatype(),
@@ -386,23 +412,22 @@ class Reader:
                 self.file_path.name,
             )
         # same length of datasets:
-        ds_volt_size = self.h5file["data"]["voltage"].shape[0]
         for dset in ["current", "time"]:
             ds_size = self.h5file["data"][dset].shape[0]
-            if ds_volt_size != ds_size:
+            if ds_size != self.samples_n:
                 self._logger.warning(
                     "[FileValidation] dataset '%s' has different size (=%d), "
-                    "compared to time-ds (=%d), in '%s'",
+                    "compared to smallest set (=%d), in '%s'",
                     dset,
                     ds_size,
-                    ds_volt_size,
+                    self.samples_n,
                     self.file_path.name,
                 )
-        # dataset-length should be multiple of buffersize
-        remaining_size = ds_volt_size % self.samples_per_buffer
+        # dataset-length should be multiple of chunk-size
+        remaining_size = self.samples_n % self.CHUNK_SAMPLES_N
         if remaining_size != 0:
             self._logger.warning(
-                "[FileValidation] datasets are not aligned with buffer-size in '%s'",
+                "[FileValidation] datasets are not aligned with chunk-size in '%s'",
                 self.file_path.name,
             )
         # check compression
@@ -457,10 +482,10 @@ class Reader:
 
         :return: sampled energy in Ws (watt-seconds)
         """
-        iterations = math.ceil(self.ds_voltage.shape[0] / self.max_elements)
+        iterations = math.ceil(self.samples_n / self.max_elements)
         job_iter = trange(
             0,
-            self.ds_voltage.shape[0],
+            self.samples_n,
             self.max_elements,
             desc="energy",
             leave=False,
@@ -468,7 +493,7 @@ class Reader:
         )
 
         def _calc_energy(idx_start: int) -> float:
-            idx_stop = min(idx_start + self.max_elements, self.ds_voltage.shape[0])
+            idx_stop = min(idx_start + self.max_elements, self.samples_n)
             vol_v = self._cal.voltage.raw_to_si(self.ds_voltage[idx_start:idx_stop])
             cur_a = self._cal.current.raw_to_si(self.ds_current[idx_start:idx_stop])
             return (vol_v[:] * cur_a[:]).sum() * self.sample_interval_s
@@ -478,7 +503,7 @@ class Reader:
 
     def _dset_statistics(
         self, dset: h5py.Dataset, cal: Optional[CalibrationPair] = None
-    ) -> Dict[str, float]:
+    ) -> dict[str, float]:
         """Create basic stats for a provided dataset.
 
         :param dset: dataset to evaluate
@@ -511,7 +536,7 @@ class Reader:
         if len(stats_list) < 1:
             return {}
         stats_nd = np.stack(stats_list)
-        stats: Dict[str, float] = {
+        stats: dict[str, float] = {
             # TODO: wrong calculation for ndim-datasets with n>1
             "mean": float(stats_nd[:, 0].mean()),
             "min": float(stats_nd[:, 1].min()),
@@ -521,17 +546,19 @@ class Reader:
         }
         return stats
 
-    def _data_timediffs(self) -> List[float]:
-        """Calculate list of unique time-deltas [s] between buffers.
+    def _data_timediffs(self) -> list[float]:
+        """Calculate list of unique time-deltas [s] between chunks.
 
-        Optimized version that only looks at the start of each buffer.
+        Optimized version that only looks at the start of each chunk.
+        Timestamps get converted to signed (it still fits > 100 years)
+        to allow calculating negative diffs.
 
-        :return: list of (unique) time-deltas between buffers [s]
+        :return: list of (unique) time-deltas between chunks [s]
         """
-        iterations = math.ceil(self.ds_time.shape[0] / self.max_elements)
+        iterations = math.ceil(self.samples_n / self.max_elements)
         job_iter = trange(
             0,
-            self.h5file["data"]["time"].shape[0],
+            self.samples_n,
             self.max_elements,
             desc="timediff",
             leave=False,
@@ -540,14 +567,14 @@ class Reader:
 
         def calc_timediffs(idx_start: int) -> list:
             ds_time = self.ds_time[
-                idx_start : (idx_start + self.max_elements) : self.samples_per_buffer
-            ]
+                idx_start : (idx_start + self.max_elements) : self.CHUNK_SAMPLES_N
+            ].astype(np.int64)
             diffs_np = np.unique(ds_time[1:] - ds_time[0:-1], return_counts=False)
             return list(np.array(diffs_np))
 
         diffs_ll = [calc_timediffs(i) for i in job_iter]
         diffs = {
-            round(self._cal.time.raw_to_si(j) / self.samples_per_buffer, 6)
+            round(self._cal.time.raw_to_si(j) / self.CHUNK_SAMPLES_N, 6)
             for i in diffs_ll
             for j in i
         }
@@ -565,7 +592,7 @@ class Reader:
             self._logger.warning(
                 "Time-jumps detected -> expected equal steps, but got: %s s", diffs
             )
-        return (len(diffs) <= 1) and diffs[0] == round(0.1 / self.samples_per_buffer, 6)
+        return (len(diffs) <= 1) and diffs[0] == round(0.1 / self.CHUNK_SAMPLES_N, 6)
 
     def count_errors_in_log(self, group_name: str = "sheep", min_level: int = 40) -> int:
         if group_name not in self.h5file:
@@ -583,7 +610,7 @@ class Reader:
         node: Union[h5py.Dataset, h5py.Group, None] = None,
         *,
         minimal: bool = False,
-    ) -> Dict[str, dict]:
+    ) -> dict[str, dict]:
         """Recursive FN to capture the structure of the file.
 
         :param node: starting node, leave free to go through whole file
@@ -594,7 +621,7 @@ class Reader:
             self._refresh_file_stats()
             return self.get_metadata(self.h5file, minimal=minimal)
 
-        metadata: Dict[str, dict] = {}
+        metadata: dict[str, dict] = {}
         if isinstance(node, h5py.Dataset) and not minimal:
             metadata["_dataset_info"] = {
                 "datatype": str(node.dtype),
@@ -616,7 +643,7 @@ class Reader:
                 with contextlib.suppress(yaml.YAMLError):
                     attr_value = yaml.safe_load(attr_value)
             elif "int" in str(type(attr_value)):
-                # TODO: why not isinstance? can it be List[int] other complex type?
+                # TODO: why not isinstance? can it be list[int] other complex type?
                 attr_value = int(attr_value)
             else:
                 attr_value = float(attr_value)
@@ -675,7 +702,7 @@ class Reader:
         return data != data_1
 
     def gpio_to_waveforms(self, name: Optional[str] = None) -> dict:
-        waveforms: Dict[str, np.ndarray] = {}
+        waveforms: dict[str, np.ndarray] = {}
         if "gpio" not in self.h5file:
             return waveforms
 

shepherd_core/testbed_client/client_abc_fix.py

@@ -17,11 +17,12 @@ TODO: Comfort functions missing
 
 from abc import ABC
 from abc import abstractmethod
-from typing import List
+from typing import Any
 from typing import Optional
 
-from ..data_models.base.shepherd import ShpModel
-from ..data_models.base.wrapper import Wrapper
+from shepherd_core.data_models.base.shepherd import ShpModel
+from shepherd_core.data_models.base.wrapper import Wrapper
+
 from .fixtures import Fixtures
 
 
@@ -40,11 +41,11 @@ class AbcClient(ABC):
         """
 
     @abstractmethod
-    def query_ids(self, model_type: str) -> List[int]:
+    def query_ids(self, model_type: str) -> list[int]:
         pass
 
     @abstractmethod
-    def query_names(self, model_type: str) -> List[str]:
+    def query_names(self, model_type: str) -> list[str]:
         pass
 
     @abstractmethod
@@ -54,11 +55,15 @@ class AbcClient(ABC):
         pass
 
     @abstractmethod
-    def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
+    def try_inheritance(
+        self, model_type: str, values: dict[str, Any]
+    ) -> tuple[dict[str, Any], list[str]]:
         # TODO: maybe internal? yes
         pass
 
-    def try_completing_model(self, model_type: str, values: dict) -> (dict, list):
+    def try_completing_model(
+        self, model_type: str, values: dict[str, Any]
+    ) -> tuple[dict[str, Any], list[str]]:
         """Init by name/id, for none existing instances raise Exception.
 
         This is the main entry-point for querying a model (used be the core-lib).
@@ -67,13 +72,12 @@ class AbcClient(ABC):
             try:
                 values = self.query_item(model_type, name=values.get("name"), uid=values.get("id"))
             except ValueError as err:
-                raise ValueError(
-                    "Query %s by name / ID failed - %s is unknown!", model_type, values
-                ) from err
+                msg = f"Query {model_type} by name / ID failed - {values} is unknown!"
+                raise ValueError(msg) from err
         return self.try_inheritance(model_type, values)
 
     @abstractmethod
-    def fill_in_user_data(self, values: dict) -> dict:
+    def fill_in_user_data(self, values: dict[str, Any]) -> dict[str, Any]:
         # TODO: is it really helpful and needed?
         pass
 
@@ -83,7 +87,7 @@ class FixturesClient(AbcClient):
 
     def __init__(self) -> None:
         super().__init__()
-        self._fixtures: Optional[Fixtures] = Fixtures()
+        self._fixtures: Fixtures = Fixtures()
 
     def insert(self, data: ShpModel) -> bool:
         wrap = Wrapper(
@@ -93,10 +97,10 @@ class FixturesClient(AbcClient):
         self._fixtures.insert_model(wrap)
         return True
 
-    def query_ids(self, model_type: str) -> List[int]:
+    def query_ids(self, model_type: str) -> list[int]:
         return list(self._fixtures[model_type].elements_by_id.keys())
 
-    def query_names(self, model_type: str) -> List[str]:
+    def query_names(self, model_type: str) -> list[str]:
         return list(self._fixtures[model_type].elements_by_name.keys())
 
     def query_item(
@@ -108,10 +112,12 @@ class FixturesClient(AbcClient):
             return self._fixtures[model_type].query_name(name)
         raise ValueError("Query needs either uid or name of object")
 
-    def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
+    def try_inheritance(
+        self, model_type: str, values: dict[str, Any]
+    ) -> tuple[dict[str, Any], list[str]]:
         return self._fixtures[model_type].inheritance(values)
 
-    def fill_in_user_data(self, values: dict) -> dict:
+    def fill_in_user_data(self, values: dict[str, Any]) -> dict[str, Any]:
         """Add fake user-data when offline-client is used.
 
         Hotfix until WebClient is working.

shepherd_core/testbed_client/client_web.py

@@ -2,15 +2,16 @@
 
 from importlib import import_module
 from pathlib import Path
-from typing import List
+from typing import Any
 from typing import Optional
 from typing import Union
 
 from pydantic import validate_call
 
-from ..commons import testbed_server_default
-from ..data_models.base.shepherd import ShpModel
-from ..data_models.base.wrapper import Wrapper
+from shepherd_core.commons import TESTBED_SERVER_URI
+from shepherd_core.data_models.base.shepherd import ShpModel
+from shepherd_core.data_models.base.wrapper import Wrapper
+
 from .client_abc_fix import AbcClient
 from .user_model import User
 
@@ -37,7 +38,7 @@ class WebClient(AbcClient):
         if not hasattr(self, "_token"):
             # add default values
             self._token: str = "basic_public_access"  # noqa: S105
-            self._server: str = testbed_server_default
+            self._server: str = TESTBED_SERVER_URI
             self._user: Optional[User] = None
             self._key: Optional[str] = None
             self._connected: bool = False
@@ -49,6 +50,8 @@ class WebClient(AbcClient):
     # ABC Functions below
 
     def insert(self, data: ShpModel) -> bool:
+        if self._req is None:
+            return False
         wrap = Wrapper(
             datatype=type(data).__name__,
             parameters=data.model_dump(),
@@ -57,10 +60,10 @@ class WebClient(AbcClient):
         r.raise_for_status()
         return True
 
-    def query_ids(self, model_type: str) -> List[int]:
+    def query_ids(self, model_type: str) -> list[int]:
         raise NotImplementedError("TODO")
 
-    def query_names(self, model_type: str) -> List[str]:
+    def query_names(self, model_type: str) -> list[str]:
         raise NotImplementedError("TODO")
 
     def query_item(
@@ -68,10 +71,14 @@ class WebClient(AbcClient):
     ) -> dict:
         raise NotImplementedError("TODO")
 
-    def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
+    def try_inheritance(
+        self, model_type: str, values: dict[str, Any]
+    ) -> tuple[dict[str, Any], list[str]]:
         raise NotImplementedError("TODO")
 
-    def fill_in_user_data(self, values: dict) -> dict:
+    def fill_in_user_data(self, values: dict[str, Any]) -> dict[str, Any]:
+        if self._user is None:
+            return values
         if values.get("owner") is None:
             values["owner"] = self._user.name
         if values.get("group") is None:
@@ -105,7 +112,7 @@ class WebClient(AbcClient):
         return self._query_user_data()
 
     def _query_session_key(self) -> bool:
-        if self._server:
+        if self._server and self._req is not None:
             r = self._req.get(self._server + "/session_key", timeout=2)
             r.raise_for_status()
             self._key = r.json()["value"]  # TODO: not finished
@@ -113,7 +120,7 @@ class WebClient(AbcClient):
         return False
 
     def _query_user_data(self) -> bool:
-        if self._server:
+        if self._server and self._req is not None:
             r = self._req.get(self._server + "/user?token=" + self._token, timeout=2)
             # TODO: possibly a security nightmare (send via json or encrypted via public key?)
             r.raise_for_status()