shepherd-data 2025.2.2__tar.gz → 2025.4.2__tar.gz

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.2.2
+Version: 2025.4.2
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -18,7 +18,6 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,7 +26,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Natural Language :: English
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: click
 Requires-Dist: h5py
@@ -36,7 +35,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.02.2
+Requires-Dist: shepherd-core[inventory]>=2025.04.2
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/pyproject.toml

@@ -18,7 +18,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Information Technology",
     "Intended Audience :: Science/Research",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
@@ -29,7 +28,7 @@ classifiers = [
     "Natural Language :: English",
 ]
 
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
     "click",
     "h5py",
@@ -38,7 +37,7 @@ dependencies = [
     "pandas>=2.0.0",  # full-version, v2 is OK
     "pyYAML",
     "scipy",   # full-version
-    "shepherd-core[inventory]>=2025.02.2",  # libs are strongly coupled
+    "shepherd-core[inventory]>=2025.04.2",  # libs are strongly coupled
     "tqdm",    # full-version
 ]
 
@@ -101,5 +100,10 @@ addopts = "-vvv --stepwise" # opts: verbose result for each tests
 source = ["shepherd_data"]
 
 [tool.mypy]
-python_version = 3.8
+python_version = 3.9
 ignore_missing_imports = true
+disable_error_code = ["call-arg", ]
+exclude = [
+    "build/",
+    ".egg-info/",
+]

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data/__init__.py

@@ -11,7 +11,7 @@ from shepherd_core import Writer
 
 from .reader import Reader
 
-__version__ = "2025.02.2"
+__version__ = "2025.04.2"
 
 __all__ = [
     "Reader",

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data/cli.py

@@ -1,12 +1,10 @@
 """Command definitions for CLI."""
 
 import logging
-import os
 import sys
 from contextlib import suppress
 from datetime import datetime
 from pathlib import Path
-from typing import List
 from typing import Optional
 
 import click
@@ -22,7 +20,7 @@ from .reader import Reader
 logger = logging.getLogger("SHPData.cli")
 
 
-def path_to_flist(data_path: Path) -> List[Path]:
+def path_to_flist(data_path: Path, *, recurse: bool = False) -> list[Path]:
     """Every path gets transformed to a list of paths.
 
     Transformations:
@@ -31,16 +29,14 @@ def path_to_flist(data_path: Path) -> List[Path]:
     - or else: empty list
     """
     data_path = Path(data_path).resolve()
-    h5files = []
+    h5files: list = []
     if data_path.is_file() and data_path.suffix.lower() == ".h5":
         h5files.append(data_path)
     elif data_path.is_dir():
-        flist = os.listdir(data_path)
-        for file in flist:
-            fpath = data_path / str(file)
-            if not fpath.is_file() or fpath.suffix.lower() != ".h5":
-                continue
-            h5files.append(fpath)
+        files = data_path.glob(
+            "**/*.h5" if recurse else "*.h5"
+        )  # for py>=3.12: case_sensitive=False
+        h5files = [file for file in files if file.is_file()]
     return h5files
 
 
@@ -70,9 +66,15 @@ def version() -> None:
 
 @cli.command(short_help="Validates a file or directory containing shepherd-recordings")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
-def validate(in_data: Path) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def validate(in_data: Path, *, recurse: bool = False) -> None:
     """Validate a file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()  # TODO: should be stored and passed in ctx
     valid_dir = True
     for file in files:
@@ -123,7 +125,13 @@ def validate(in_data: Path) -> None:
     "--raw",
     "-r",
     is_flag=True,
-    help="Plot only power instead of voltage, current & power",
+    help="Don't convert data to si-units",
+)
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
 )
 def extract(
     in_data: Path,
@@ -133,9 +141,10 @@ def extract(
     separator: str,
     *,
     raw: bool = False,
+    recurse: bool = False,
 ) -> None:
     """Extract recorded IVSamples and store them to csv."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     if not isinstance(ds_factor, (float, int)) or ds_factor < 1:
         ds_factor = 1000
@@ -162,10 +171,15 @@ def extract(
     type=click.STRING,
     help="Set an individual csv-separator",
 )
-# TODO: a recursive option would help!
-def extract_meta(in_data: Path, separator: str) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def extract_meta(in_data: Path, separator: str, *, recurse: bool = False) -> None:
     """Extract metadata and logs from file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting metadata & logs from '%s' ...", file.name)
@@ -195,9 +209,15 @@ def extract_meta(in_data: Path, separator: str) -> None:
     short_help="Extracts uart from gpio-trace in file or directory containing shepherd-recordings"
 )
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
-def extract_uart(in_data: Path) -> None:
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
+def extract_uart(in_data: Path, *, recurse: bool = False) -> None:
     """Extract UART from GPIO-trace in file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting uart from gpio-trace from from '%s' ...", file.name)
@@ -220,6 +240,7 @@ def extract_uart(in_data: Path) -> None:
                             log_file.write(timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + ":")
                             # TODO: allow to skip Timestamp and export raw text
                             log_file.write(f"\t{str.encode(line[1])}")
+                            # TODO: does this produce "\tb'abc'"?
                             log_file.write("\n")
         except TypeError:
             logger.exception("ERROR: Will skip file. It caused an exception.")
@@ -233,10 +254,16 @@ def extract_uart(in_data: Path) -> None:
     default=";",
     type=click.STRING,
     help="Set an individual csv-separator",
+)  # TODO: also configure decimal point
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
 )
-def extract_gpio(in_data: Path, separator: str) -> None:
+def extract_gpio(in_data: Path, separator: str, *, recurse: bool = False) -> None:
     """Extract UART from gpio-trace in file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         logger.info("Extracting gpio-trace from from '%s' ...", file.name)
@@ -281,15 +308,23 @@ def extract_gpio(in_data: Path, separator: str) -> None:
     type=click.FLOAT,
     help="End-point in seconds, will be max if omitted",
 )
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
 def downsample(
     in_data: Path,
     ds_factor: Optional[float],
     sample_rate: Optional[int],
     start: Optional[float],
     end: Optional[float],
+    *,
+    recurse: bool = False,
 ) -> None:
     """Create an array of down-sampled files from file or dir containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     for file in files:
         try:
@@ -351,6 +386,12 @@ def downsample(
     is_flag=True,
     help="Plot only power instead of voltage, current & power",
 )
+@click.option(
+    "--recurse",
+    "-a",
+    is_flag=True,
+    help="Also consider files in sub-folders",
+)
 def plot(
     in_data: Path,
     start: Optional[float],
@@ -360,9 +401,10 @@ def plot(
     *,
     multiplot: bool,
     only_power: bool,
+    recurse: bool = False,
 ) -> None:
     """Plot IV-trace from file or directory containing shepherd-recordings."""
-    files = path_to_flist(in_data)
+    files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     multiplot = multiplot and len(files) > 1
     data = []

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data/ivonne.py

@@ -17,14 +17,14 @@ import pickle
 from pathlib import Path
 from types import TracebackType
 from typing import Optional
-from typing import Type
 
 import numpy as np
 import pandas as pd
 from tqdm import trange
 from typing_extensions import Self
 
-from . import Writer
+from shepherd_core.writer import Writer as CoreWriter
+
 from .mppt import MPPTracker
 from .mppt import OptimalTracker
 from .mppt import iv_model
@@ -88,7 +88,7 @@ class Reader:
 
     def __exit__(
         self,
-        typ: Optional[Type[BaseException]] = None,
+        typ: Optional[type[BaseException]] = None,
         exc: Optional[BaseException] = None,
         tb: Optional[TracebackType] = None,
         extra_arg: int = 0,
@@ -132,7 +132,7 @@ class Reader:
 
         v_proto = np.linspace(0, v_max, pts_per_curve)
 
-        with Writer(shp_output, datatype="ivcurve", window_samples=pts_per_curve) as sfw:
+        with CoreWriter(shp_output, datatype="ivcurve", window_samples=pts_per_curve) as sfw:
             sfw.store_hostname("IVonne")
             curve_interval_us = round(sfw.sample_interval_ns * pts_per_curve / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns
@@ -207,7 +207,7 @@ class Reader:
                 v_max,
             )
 
-        with Writer(shp_output, datatype="ivsample") as sfw:
+        with CoreWriter(shp_output, datatype="ivsample") as sfw:
             sfw.store_hostname("IVonne")
             interval_us = round(sfw.sample_interval_ns / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns
@@ -260,7 +260,7 @@ class Reader:
             self._logger.info("File already exists, will skip '%s'", shp_output.name)
             return
 
-        with Writer(shp_output, datatype="isc_voc") as sfw:
+        with CoreWriter(shp_output, datatype="isc_voc") as sfw:
             sfw.store_hostname("IVonne")
             interval_us = round(sfw.sample_interval_ns / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data/mppt.py

@@ -3,6 +3,9 @@
 Might be exchanged by shepherds py-model of pru-harvesters.
 """
 
+from abc import ABC
+from abc import abstractmethod
+
 import numpy as np
 import pandas as pd
 
@@ -36,7 +39,7 @@ def find_oc(v_arr: np.ndarray, i_arr: np.ndarray, ratio: float = 0.05) -> np.nda
     return v_arr[np.argmax(i_arr < i_arr[0] * ratio)]
 
 
-class MPPTracker:
+class MPPTracker(ABC):
     """Prototype for a MPPT-class.
 
     :param v_max: Maximum voltage supported by shepherd
@@ -48,6 +51,7 @@ class MPPTracker:
         self.v_max: float = v_max
         self.v_proto: np.ndarray = np.linspace(0, v_max, pts_per_curve)
 
+    @abstractmethod
     def process(self, coeffs: pd.DataFrame) -> pd.DataFrame:
         """Apply harvesting model to input data.
 

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data/reader.py

@@ -1,9 +1,10 @@
 """Reader-Baseclass for opening shepherds hdf5-files."""
 
 import math
+from collections.abc import Mapping
+from collections.abc import Sequence
 from datetime import datetime
 from pathlib import Path
-from typing import Dict
 from typing import Optional
 from typing import Union
 
@@ -33,9 +34,9 @@ class Reader(CoreReader):
 
     def __init__(
         self,
-        file_path: Optional[Path],
+        file_path: Path,
         *,
-        verbose: Optional[bool] = True,
+        verbose: bool = True,
     ) -> None:
         super().__init__(file_path, verbose=verbose)
 
@@ -56,12 +57,16 @@ class Reader(CoreReader):
         if csv_path.exists():
             self._logger.info("File already exists, will skip '%s'", csv_path.name)
             return 0
-        datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
+        datasets: list[str] = [
+            str(key) for key in h5_group if isinstance(h5_group[key], h5py.Dataset)
+        ]
         datasets.remove("time")
         datasets = ["time", *datasets]
         separator = separator.strip().ljust(2)
-        header = [h5_group[key].attrs["description"].replace(", ", separator) for key in datasets]
-        header = separator.join(header)
+        header_elements: list[str] = [
+            str(h5_group[key].attrs["description"]).replace(", ", separator) for key in datasets
+        ]
+        header: str = separator.join(header_elements)
         with csv_path.open("w", encoding="utf-8-sig") as csv_file:
             self._logger.info("CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name)
             csv_file.write(header + "\n")
@@ -222,7 +227,7 @@ class Reader(CoreReader):
         for _iter in trange(
             0,
             iterations,
-            desc=f"downsampling {data_src.name}",
+            desc=f"downsampling {data_src.name if isinstance(data_src, h5py.Dataset) else ''}",
             leave=False,
             disable=iterations < 8,
         ):
@@ -242,7 +247,7 @@ class Reader(CoreReader):
         self,
         start_s: Optional[float],
         end_s: Optional[float],
-        ds_factor: Optional[float],
+        ds_factor: float,
     ) -> Path:
         """Cut source to given limits, downsample by factor and store result in separate file.
 
@@ -290,10 +295,7 @@ class Reader(CoreReader):
         else:
             cut_str = ""
 
-        if ds_factor > 1:  # noqa: SIM108
-            ds_str = f".downsample_x{round(ds_factor)}"
-        else:
-            ds_str = ""
+        ds_str = f".downsample_x{round(ds_factor)}" if ds_factor > 1 else ""
 
         dst_file = self.file_path.resolve().with_suffix(cut_str + ds_str + ".h5")
         if dst_file.exists():
@@ -403,7 +405,7 @@ class Reader(CoreReader):
             for _ in trange(
                 0,
                 iterations,
-                desc=f"resampling {data_src.name}",
+                desc=f"resampling {data_src.name if isinstance(data_src, h5py.Dataset) else ''}",
                 leave=False,
                 disable=iterations < 8,
             ):
@@ -456,7 +458,7 @@ class Reader(CoreReader):
         end_s: Optional[float] = None,
         *,
         relative_timestamp: bool = True,
-    ) -> Optional[Dict]:
+    ) -> Optional[dict]:
         """Provide down-sampled iv-data that can be fed into plot_to_file().
 
         :param start_s: time in seconds, relative to start of recording
@@ -506,7 +508,7 @@ class Reader(CoreReader):
 
     @staticmethod
     def assemble_plot(
-        data: Union[dict, list], width: int = 20, height: int = 10, *, only_pwr: bool = False
+        data: Union[Mapping, Sequence], width: int = 20, height: int = 10, *, only_pwr: bool = False
     ) -> plt.Figure:
         """Create the actual figure.
 
@@ -520,7 +522,7 @@ class Reader(CoreReader):
         :return: figure
         """
         # TODO: allow choosing freely from I, V, P, GPIO
-        if isinstance(data, dict):
+        if isinstance(data, Mapping):
             data = [data]
         if only_pwr:
             fig, ax = plt.subplots(1, 1, figsize=(width, height), layout="tight")

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.2.2
+Version: 2025.4.2
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -18,7 +18,6 @@ Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
 Classifier: Intended Audience :: Science/Research
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -27,7 +26,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Classifier: Natural Language :: English
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 Requires-Dist: click
 Requires-Dist: h5py
@@ -36,7 +35,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.02.2
+Requires-Dist: shepherd-core[inventory]>=2025.04.2
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.2.2 → shepherd_data-2025.4.2}/shepherd_data.egg-info/requires.txt

@@ -5,7 +5,7 @@ numpy
 pandas>=2.0.0
 pyYAML
 scipy
-shepherd-core[inventory]>=2025.02.2
+shepherd-core[inventory]>=2025.04.2
 tqdm
 
 [dev]