shepherd-data 2025.2.1__tar.gz → 2025.4.1__tar.gz

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.2.1
+Version: 2025.4.1
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -36,7 +36,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.02.1
+Requires-Dist: shepherd-core[inventory]>=2025.04.1
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     "pandas>=2.0.0",  # full-version, v2 is OK
     "pyYAML",
     "scipy",   # full-version
-    "shepherd-core[inventory]>=2025.02.1",  # libs are strongly coupled
+    "shepherd-core[inventory]>=2025.04.1",  # libs are strongly coupled
     "tqdm",    # full-version
 ]
 
@@ -103,3 +103,4 @@ source = ["shepherd_data"]
 [tool.mypy]
 python_version = 3.8
 ignore_missing_imports = true
+disable_error_code = ["call-arg", ]

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data/__init__.py

@@ -11,7 +11,7 @@ from shepherd_core import Writer
 
 from .reader import Reader
 
-__version__ = "2025.02.1"
+__version__ = "2025.04.1"
 
 __all__ = [
     "Reader",

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data/cli.py

@@ -1,7 +1,6 @@
 """Command definitions for CLI."""
 
 import logging
-import os
 import sys
 from contextlib import suppress
 from datetime import datetime
@@ -22,7 +21,7 @@ from .reader import Reader
 logger = logging.getLogger("SHPData.cli")
 
 
-def path_to_flist(data_path: Path) -> List[Path]:
+def path_to_flist(data_path: Path, *, recurse: bool = False) -> List[Path]:
     """Every path gets transformed to a list of paths.
 
     Transformations:
@@ -31,16 +30,14 @@ def path_to_flist(data_path: Path) -> List[Path]:
     - or else: empty list
     """
     data_path = Path(data_path).resolve()
-    h5files = []
+    h5files: list = []
     if data_path.is_file() and data_path.suffix.lower() == ".h5":
         h5files.append(data_path)
     elif data_path.is_dir():
-        flist = os.listdir(data_path)
-        for file in flist:
-            fpath = data_path / str(file)
-            if not fpath.is_file() or fpath.suffix.lower() != ".h5":
-                continue
-            h5files.append(fpath)
+        files = data_path.glob(
+            "**/*.h5" if recurse else "*.h5"
+        )  # for py>=3.12: case_sensitive=False
+        h5files = [file for file in files if file.is_file()]
     return h5files
 
 
@@ -70,9 +67,15 @@ def version() -> None:
 
 @cli.command(short_help="Validates a file or directory containing shepherd-recordings")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
-def validate(in_data: Path) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def validate(in_data: Path, *, recurse: bool = False) -> None:
     """Validate a file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()  # TODO: should be stored and passed in ctx
     valid_dir = True
     for file in files:
@@ -123,7 +126,13 @@ def validate(in_data: Path) -> None:
     "--raw",
     "-r",
     is_flag=True,
-    help="Plot only power instead of voltage, current & power",
+    help="Don't convert data to si-units",
+)
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
 )
 def extract(
     in_data: Path,
@@ -133,9 +142,10 @@ def extract(
     separator: str,
     *,
     raw: bool = False,
+    recurse: bool = False,
 ) -> None:
     """Extract recorded IVSamples and store them to csv."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     if not isinstance(ds_factor, (float, int)) or ds_factor < 1:
         ds_factor = 1000
@@ -162,10 +172,15 @@ def extract(
     type=click.STRING,
     help="Set an individual csv-separator",
 )
-# TODO: a recursive option would help!
-def extract_meta(in_data: Path, separator: str) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def extract_meta(in_data: Path, separator: str, *, recurse: bool = False) -> None:
     """Extract metadata and logs from file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting metadata & logs from '%s' ...", file.name)
@@ -195,9 +210,15 @@ def extract_meta(in_data: Path, separator: str) -> None:
     short_help="Extracts uart from gpio-trace in file or directory containing shepherd-recordings"
 )
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
-def extract_uart(in_data: Path) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def extract_uart(in_data: Path, *, recurse: bool = False) -> None:
     """Extract UART from GPIO-trace in file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting uart from gpio-trace from from '%s' ...", file.name)
@@ -220,6 +241,7 @@ def extract_uart(in_data: Path) -> None:
                             log_file.write(timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + ":")
                             # TODO: allow to skip Timestamp and export raw text
                             log_file.write(f"\t{str.encode(line[1])}")
+                            # TODO: does this produce "\tb'abc'"?
                             log_file.write("\n")
         except TypeError:
             logger.exception("ERROR: Will skip file. It caused an exception.")
@@ -233,10 +255,16 @@ def extract_uart(in_data: Path) -> None:
     default=";",
     type=click.STRING,
     help="Set an individual csv-separator",
+)  # TODO: also configure decimal point
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
 )
-def extract_gpio(in_data: Path, separator: str) -> None:
+def extract_gpio(in_data: Path, separator: str, *, recurse: bool = False) -> None:
     """Extract UART from gpio-trace in file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting gpio-trace from from '%s' ...", file.name)
@@ -281,15 +309,23 @@ def extract_gpio(in_data: Path, separator: str) -> None:
     type=click.FLOAT,
     help="End-point in seconds, will be max if omitted",
 )
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
 def downsample(
     in_data: Path,
     ds_factor: Optional[float],
     sample_rate: Optional[int],
     start: Optional[float],
     end: Optional[float],
+    *,
+    recurse: bool = False,
 ) -> None:
     """Create an array of down-sampled files from file or dir containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         try:
@@ -351,6 +387,12 @@ def downsample(
     is_flag=True,
     help="Plot only power instead of voltage, current & power",
 )
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
 def plot(
     in_data: Path,
     start: Optional[float],
@@ -360,9 +402,10 @@ def plot(
     *,
     multiplot: bool,
     only_power: bool,
+    recurse: bool = False,
 ) -> None:
     """Plot IV-trace from file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     multiplot = multiplot and len(files) > 1
     data = []

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data/mppt.py

@@ -3,6 +3,9 @@
 Might be exchanged by shepherds py-model of pru-harvesters.
 """
 
+from abc import ABC
+from abc import abstractmethod
+
 import numpy as np
 import pandas as pd
 
@@ -36,7 +39,7 @@ def find_oc(v_arr: np.ndarray, i_arr: np.ndarray, ratio: float = 0.05) -> np.nda
     return v_arr[np.argmax(i_arr < i_arr[0] * ratio)]
 
 
-class MPPTracker:
+class MPPTracker(ABC):
     """Prototype for a MPPT-class.
 
     :param v_max: Maximum voltage supported by shepherd
@@ -48,6 +51,7 @@ class MPPTracker:
         self.v_max: float = v_max
         self.v_proto: np.ndarray = np.linspace(0, v_max, pts_per_curve)
 
+    @abstractmethod
     def process(self, coeffs: pd.DataFrame) -> pd.DataFrame:
         """Apply harvesting model to input data.
 

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data/reader.py

@@ -4,7 +4,10 @@ import math
 from datetime import datetime
 from pathlib import Path
 from typing import Dict
+from typing import List
+from typing import Mapping
 from typing import Optional
+from typing import Sequence
 from typing import Union
 
 import h5py
@@ -56,21 +59,25 @@ class Reader(CoreReader):
         if csv_path.exists():
             self._logger.info("File already exists, will skip '%s'", csv_path.name)
             return 0
-        datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
+        datasets: List[str] = [
+            str(key) for key in h5_group if isinstance(h5_group[key], h5py.Dataset)
+        ]
         datasets.remove("time")
         datasets = ["time", *datasets]
         separator = separator.strip().ljust(2)
-        header = [h5_group[key].attrs["description"].replace(", ", separator) for key in datasets]
-        header = separator.join(header)
+        header_elements: List[str] = [
+            str(h5_group[key].attrs["description"]).replace(", ", separator) for key in datasets
+        ]
+        header: str = separator.join(header_elements)
         with csv_path.open("w", encoding="utf-8-sig") as csv_file:
             self._logger.info("CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name)
             csv_file.write(header + "\n")
             ts_gain = h5_group["time"].attrs.get("gain", 1e-9)
             # for converting data to si - if raw=false
-            gains: dict[str, float] = {
+            gains: Dict[str, float] = {
                 key: h5_group[key].attrs.get("gain", 1.0) for key in datasets[1:]
             }
-            offsets: dict[str, float] = {
+            offsets: Dict[str, float] = {
                 key: h5_group[key].attrs.get("offset", 1.0) for key in datasets[1:]
             }
             for idx, time_ns in enumerate(h5_group["time"][:]):
@@ -222,7 +229,7 @@ class Reader(CoreReader):
         for _iter in trange(
             0,
             iterations,
-            desc=f"downsampling {data_src.name}",
+            desc=f"downsampling {data_src.name if isinstance(data_src, h5py.Dataset) else ''}",
             leave=False,
             disable=iterations < 8,
         ):
@@ -242,7 +249,7 @@ class Reader(CoreReader):
         self,
         start_s: Optional[float],
         end_s: Optional[float],
-        ds_factor: Optional[float],
+        ds_factor: float,
     ) -> Path:
         """Cut source to given limits, downsample by factor and store result in separate file.
 
@@ -290,10 +297,7 @@ class Reader(CoreReader):
         else:
             cut_str = ""
 
-        if ds_factor > 1:  # noqa: SIM108
-            ds_str = f".downsample_x{round(ds_factor)}"
-        else:
-            ds_str = ""
+        ds_str = f".downsample_x{round(ds_factor)}" if ds_factor > 1 else ""
 
         dst_file = self.file_path.resolve().with_suffix(cut_str + ds_str + ".h5")
         if dst_file.exists():
@@ -403,7 +407,7 @@ class Reader(CoreReader):
             for _ in trange(
                 0,
                 iterations,
-                desc=f"resampling {data_src.name}",
+                desc=f"resampling {data_src.name if isinstance(data_src, h5py.Dataset) else ''}",
                 leave=False,
                 disable=iterations < 8,
             ):
@@ -506,7 +510,7 @@ class Reader(CoreReader):
 
     @staticmethod
     def assemble_plot(
-        data: Union[dict, list], width: int = 20, height: int = 10, *, only_pwr: bool = False
+        data: Union[Mapping, Sequence], width: int = 20, height: int = 10, *, only_pwr: bool = False
     ) -> plt.Figure:
         """Create the actual figure.
 
@@ -520,7 +524,7 @@ class Reader(CoreReader):
         :return: figure
         """
         # TODO: allow choosing freely from I, V, P, GPIO
-        if isinstance(data, dict):
+        if isinstance(data, Mapping):
             data = [data]
         if only_pwr:
             fig, ax = plt.subplots(1, 1, figsize=(width, height), layout="tight")

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.2.1
+Version: 2025.4.1
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -36,7 +36,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.02.1
+Requires-Dist: shepherd-core[inventory]>=2025.04.1
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.2.1 → shepherd_data-2025.4.1}/shepherd_data.egg-info/requires.txt

@@ -5,7 +5,7 @@ numpy
 pandas>=2.0.0
 pyYAML
 scipy
-shepherd-core[inventory]>=2025.02.1
+shepherd-core[inventory]>=2025.04.1
 tqdm
 
 [dev]