shepherd-data 2025.4.2__tar.gz → 2025.5.3__tar.gz

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.4.2
+Version: 2025.5.3
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -35,7 +35,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.04.2
+Requires-Dist: shepherd-core[inventory]>=2025.05.3
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "pandas>=2.0.0",  # full-version, v2 is OK
     "pyYAML",
     "scipy",   # full-version
-    "shepherd-core[inventory]>=2025.04.2",  # libs are strongly coupled
+    "shepherd-core[inventory]>=2025.05.3",  # libs are strongly coupled
     "tqdm",    # full-version
 ]
 

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data/__init__.py

@@ -11,7 +11,7 @@ from shepherd_core import Writer
 
 from .reader import Reader
 
-__version__ = "2025.04.2"
+__version__ = "2025.05.3"
 
 __all__ = [
     "Reader",

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data/cli.py

@@ -92,7 +92,7 @@ def validate(in_data: Path, *, recurse: bool = False) -> None:
     sys.exit(int(not valid_dir))
 
 
-@cli.command(short_help="Extracts recorded IVSamples and stores it to csv")
+@cli.command(short_help="Extracts recorded IVTrace and stores it to csv")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
 @click.option(
     "--start",
@@ -143,7 +143,7 @@ def extract(
     raw: bool = False,
     recurse: bool = False,
 ) -> None:
-    """Extract recorded IVSamples and store them to csv."""
+    """Extract recorded IVTrace and store them to csv."""
     files = path_to_flist(in_data, recurse=recurse)
     verbose_level = get_verbose_level()
     if not isinstance(ds_factor, (float, int)) or ds_factor < 1:
@@ -189,12 +189,12 @@ def extract_meta(in_data: Path, separator: str, *, recurse: bool = False) -> Non
             with Reader(file, verbose=verbose_level > 2) as shpr:
                 shpr.save_metadata()
                 csvs_depr = ["sysutil", "timesync"]
-                csvs = ["ptp", "sys_util", "pru_util"]
+                csvs = ["ptp", "phc2sys", "sys_util", "pru_util"]
                 for element in csvs + csvs_depr:
                     if element in shpr.h5file:
                         shpr.save_csv(shpr[element], separator)
                 logs_depr = ["shepherd-log", "dmesg", "exceptions"]
-                logs = ["sheep", "kernel", "phc2sys", "uart"]
+                logs = ["sheep", "kernel", "ntp", "uart"]
                 for element in logs + logs_depr:
                     if element in shpr.h5file:
                         shpr.save_log(shpr[element])

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data/ivonne.py

@@ -102,7 +102,7 @@ class Reader:
         self.file_size = self.file_path.stat().st_size
         self.data_rate = self.file_size / self.runtime_s if self.runtime_s > 0 else 0
 
-    def convert_2_ivcurves(
+    def convert_2_ivsurface(
         self,
         shp_output: Path,
         v_max: float = 5.0,
@@ -137,7 +137,9 @@ class Reader:
             curve_interval_us = round(sfw.sample_interval_ns * pts_per_curve / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns
             max_elements = math.ceil(sfw.max_elements // up_factor)
-            job_iter = trange(0, df_elements_n, max_elements, desc="generate ivcurves", leave=False)
+            job_iter = trange(
+                0, df_elements_n, max_elements, desc="generate ivsurface", leave=False
+            )
 
             for idx in job_iter:
                 idx_top = min(idx + max_elements, df_elements_n)
@@ -164,14 +166,14 @@ class Reader:
                 #   - time can be generated and set as a whole
                 #   - v_proto is repetitive, can also be set as a whole
 
-    def convert_2_ivsamples(
+    def convert_2_ivtrace(
         self,
         shp_output: Path,
         v_max: float = 5.0,
         duration_s: Optional[float] = None,
         tracker: Optional[MPPTracker] = None,
     ) -> None:
-        """Transform shepherd IV curves to shepherd IV samples / traces.
+        """Transform shepherd IV surface / curves to shepherd IV trace / samples .
 
         For the 'buck' and 'buck-boost' modes, shepherd takes voltage and current traces.
         These can be recorded with shepherd or generated from existing IV curves by, for
@@ -212,9 +214,7 @@ class Reader:
             interval_us = round(sfw.sample_interval_ns / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns
             max_elements = math.ceil(sfw.max_elements // up_factor)
-            job_iter = trange(
-                0, df_elements_n, max_elements, desc="generate ivsamples", leave=False
-            )
+            job_iter = trange(0, df_elements_n, max_elements, desc="generate ivtrace", leave=False)
 
             for idx in job_iter:
                 # select (max_elements + 1) elements, so upsampling is without gaps

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data/mppt.py

@@ -56,7 +56,7 @@ class MPPTracker(ABC):
         """Apply harvesting model to input data.
 
         :param coeffs: ivonne coefficients
-        :return: ivsample-data
+        :return: ivtrace-data
         """
 
 
@@ -76,7 +76,7 @@ class OpenCircuitTracker(MPPTracker):
         """Apply harvesting model to input data.
 
         :param coeffs: ivonne coefficients
-        :return: ivsample-data
+        :return: ivtrace-data
         """
         coeffs["icurve"] = coeffs.apply(lambda x: iv_model(self.v_proto, x), axis=1)
         if "voc" not in coeffs.columns:
@@ -106,7 +106,7 @@ class OptimalTracker(MPPTracker):
         """Apply harvesting model to input data.
 
         :param coeffs: ivonne coefficients
-        :return: ivsample-data
+        :return: ivtrace-data
         """
         coeffs["icurve"] = coeffs.apply(lambda x: iv_model(self.v_proto, x), axis=1)
         coeffs["pcurve"] = coeffs.apply(lambda x: self.v_proto * x["icurve"], axis=1)

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data/reader.py

@@ -16,6 +16,7 @@ from tqdm import trange
 from shepherd_core import Reader as CoreReader
 from shepherd_core import Writer as CoreWriter
 from shepherd_core import local_tz
+from shepherd_core.data_models import EnergyDType
 from shepherd_core.logger import get_verbose_level
 from shepherd_core.logger import logger
 
@@ -92,7 +93,7 @@ class Reader(CoreReader):
         return h5_group["time"][:].shape[0]
 
     def save_log(self, h5_group: h5py.Group, *, add_timestamp: bool = True) -> int:
-        """Save dataset from group as log, optimal for logged 'dmesg' and console-output.
+        """Save dataset from groups as log, optimal for logged kernel- and console-output.
 
         :param h5_group: can be external
         :param add_timestamp: can be external
@@ -175,7 +176,8 @@ class Reader(CoreReader):
     ) -> Union[None, h5py.Dataset, np.ndarray]:
         """Sample down iv-data.
 
-        Warning: only valid for IV-Stream, not IV-Curves
+        Warning: only valid for IV-Stream, not IV-Curves,
+        TODO: globally rename to IVTrace, IVSurface
 
         :param data_src: a h5-dataset to digest, can be external
         :param data_dst: can be a dataset, numpy-array or None (will be created internally then)
@@ -187,8 +189,8 @@ class Reader(CoreReader):
         """
         from scipy import signal  # here due to massive delay
 
-        if self.get_datatype() == "ivcurve":
-            self._logger.warning("Downsampling-Function was not written for IVCurves")
+        if self.get_datatype() == EnergyDType.ivsurface:
+            self._logger.warning("Downsampling-Function was not written for IVSurfaces")
         ds_factor = max(1, math.floor(ds_factor))
 
         if isinstance(end_n, (int, float)):
@@ -201,9 +203,9 @@ class Reader(CoreReader):
         if data_len == 0:
             self._logger.warning("downsampling failed because of data_len = 0")
             return data_dst
-        iblock_len = min(self.max_elements, data_len)
-        oblock_len = round(iblock_len / ds_factor)
-        iterations = math.ceil(data_len / iblock_len)
+        chunk_size_inp = min(self.max_elements, data_len)
+        chunk_size_out = round(chunk_size_inp / ds_factor)
+        iterations = math.ceil(data_len / chunk_size_inp)
         dest_len = math.floor(data_len / ds_factor)
         if data_dst is None:
             data_dst = np.empty((dest_len,))
@@ -222,8 +224,13 @@ class Reader(CoreReader):
         )
         # filter state - needed for sliced calculation
         f_state = np.zeros((filter_.shape[0], 2))
+        # prime the state to avoid starting from 0
+        if not is_time and ds_factor > 1:
+            slice_ds = data_src[start_n : start_n + self.CHUNK_SAMPLES_N]
+            slice_ds[:] = slice_ds[:].mean()
+            slice_ds, f_state = signal.sosfilt(filter_, slice_ds, zi=f_state)
 
-        slice_len = 0
+        output_pos = 0
         for _iter in trange(
             0,
             iterations,
@@ -231,16 +238,22 @@ class Reader(CoreReader):
             leave=False,
             disable=iterations < 8,
         ):
-            slice_ds = data_src[start_n + _iter * iblock_len : start_n + (_iter + 1) * iblock_len]
+            slice_ds = data_src[
+                start_n + _iter * chunk_size_inp : start_n + (_iter + 1) * chunk_size_inp
+            ]
             if not is_time and ds_factor > 1:
                 slice_ds, f_state = signal.sosfilt(filter_, slice_ds, zi=f_state)
             slice_ds = slice_ds[::ds_factor]
-            slice_len = min(dest_len - _iter * oblock_len, oblock_len)
-            data_dst[_iter * oblock_len : (_iter + 1) * oblock_len] = slice_ds[:slice_len]
+            slice_len = min(dest_len - _iter * chunk_size_out, chunk_size_out, len(slice_ds))
+            data_dst[output_pos : output_pos + slice_len] = slice_ds[:slice_len]
+            # workaround to allow processing last slice (often smaller than expected),
+            # wanted: [_iter * chunk_size_out : (_iter + 1) * chunk_size_out]
+            # this prevents future parallel processing!
+            output_pos += slice_len
         if isinstance(data_dst, np.ndarray):
-            data_dst.resize((oblock_len * (iterations - 1) + slice_len,), refcheck=False)
+            data_dst.resize((output_pos,), refcheck=False)
         else:
-            data_dst.resize((oblock_len * (iterations - 1) + slice_len,))
+            data_dst.resize((output_pos,))
         return data_dst
 
     def cut_and_downsample_to_file(
@@ -269,24 +282,20 @@ class Reader(CoreReader):
 
         # test input-parameters
         if end_sample < start_sample:
-            raise ValueError(
-                "Cut & downsample for %s failed because "
-                "end-mark (%.3f) is before start-mark (%.3f).",
-                self.file_path.name,
-                end_s,
-                start_s,
+            msg = (
+                f"Cut & downsample for {self.file_path.name} failed because "
+                f"end-mark ({end_s:.3f}) is before start-mark ({start_s:.3f})."
             )
+            raise ValueError(msg)
         if ds_factor < 1:
-            raise ValueError(
-                "Cut & downsample for %s failed because factor < 1",
-                self.file_path.name,
-            )
+            msg = f"Cut & downsample for {self.file_path.name} failed because factor < 1"
+            raise ValueError(msg)
         if ((end_sample - start_sample) / ds_factor) < 1000:
-            raise ValueError(
-                "Cut & downsample for %s failed because resulting sample-size is too small",
-                self.file_path.name,
+            msg = (
+                f"Cut & downsample for {self.file_path.name} failed because "
+                f"resulting sample-size is too small",
             )
-
+            raise ValueError(msg)
         # assemble file-name of output
         if start_s != 0.0 or end_s != self.runtime_s:
             start_str = f"{start_s:.3f}".replace(".", "s")
@@ -370,8 +379,8 @@ class Reader(CoreReader):
         :return: resampled iv-data
         """
         self._logger.error("Resampling is still under construction - do not use for now!")
-        if self.get_datatype() == "ivcurve":
-            self._logger.warning("Resampling-Function was not written for IVCurves")
+        if self.get_datatype() == EnergyDType.ivsurface:
+            self._logger.warning("Resampling-Function was not written for IVSurfaces")
             return data_dst
         if isinstance(end_n, (int, float)):
             _end_n = min(data_src.shape[0], round(end_n))
@@ -466,8 +475,8 @@ class Reader(CoreReader):
         :param relative_timestamp: treat
         :return: down-sampled size of ~ self.max_elements
         """
-        if self.get_datatype() == "ivcurve":
-            self._logger.warning("Plot-Function was not written for IVCurves.")
+        if self.get_datatype() == EnergyDType.ivsurface:
+            self._logger.warning("Plot-Function was not written for IVSurfaces.")
         if not isinstance(start_s, (float, int)):
             start_s = 0
         if not isinstance(end_s, (float, int)):
@@ -536,11 +545,20 @@ class Reader(CoreReader):
             # last axis is set below
 
         for date in data:
+            samples_n = min(len(date["time"]), len(date["voltage"]), len(date["current"]))
             if not only_pwr:
-                axs[0].plot(date["time"], date["voltage"], label=date["name"])
-                axs[1].plot(date["time"], date["current"] * 10**3, label=date["name"])
+                axs[0].plot(
+                    date["time"][:samples_n], date["voltage"][:samples_n], label=date["name"]
+                )
+                axs[1].plot(
+                    date["time"][:samples_n],
+                    date["current"][:samples_n] * 10**3,
+                    label=date["name"],
+                )
             axs[-1].plot(
-                date["time"], date["voltage"] * date["current"] * 10**3, label=date["name"]
+                date["time"][:samples_n],
+                date["voltage"][:samples_n] * date["current"][:samples_n] * 10**3,
+                label=date["name"],
             )
 
         if len(data) > 1:
@@ -551,6 +569,8 @@ class Reader(CoreReader):
             # deactivates offset-creation for ax-ticks
             ax.get_yaxis().get_major_formatter().set_useOffset(False)
             ax.get_xaxis().get_major_formatter().set_useOffset(False)
+            # add a thin and light gray grid, TODO: add option to switch off?
+            ax.grid(color="0.8", linewidth=0.5)
         return fig
 
     def plot_to_file(

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shepherd_data
-Version: 2025.4.2
+Version: 2025.5.3
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -35,7 +35,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory]>=2025.04.2
+Requires-Dist: shepherd-core[inventory]>=2025.05.3
 Requires-Dist: tqdm
 Provides-Extra: elf
 Requires-Dist: shepherd-core[elf]; extra == "elf"

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/shepherd_data.egg-info/requires.txt

@@ -5,7 +5,7 @@ numpy
 pandas>=2.0.0
 pyYAML
 scipy
-shepherd-core[inventory]>=2025.04.2
+shepherd-core[inventory]>=2025.05.3
 tqdm
 
 [dev]

{shepherd_data-2025.4.2 → shepherd_data-2025.5.3}/tests/test_ivonne.py

@@ -23,13 +23,13 @@ def test_convert_ivonne(tmp_path: Path, example_path: Path) -> None:
 
     with ivonne.Reader(inp_path) as ifr:
         ifr.upsample_2_isc_voc(isc_path, duration_s=20)
-        ifr.convert_2_ivcurves(ivc_path, duration_s=20)
+        ifr.convert_2_ivsurface(ivc_path, duration_s=20)
 
         tr_voc = mppt.OpenCircuitTracker(ratio=0.76)
         tr_opt = mppt.OptimalTracker()
 
-        ifr.convert_2_ivsamples(voc_path, tracker=tr_voc, duration_s=20)
-        ifr.convert_2_ivsamples(opt_path, tracker=tr_opt, duration_s=20)
+        ifr.convert_2_ivtrace(voc_path, tracker=tr_voc, duration_s=20)
+        ifr.convert_2_ivtrace(opt_path, tracker=tr_opt, duration_s=20)
 
     energies = {}
     for file_path in [isc_path, ivc_path, voc_path, opt_path]: