shepherd-data 2023.11.1__py3-none-any.whl → 2024.4.1__py3-none-any.whl

shepherd_data/__init__.py

@@ -4,11 +4,12 @@ Provides classes for storing and retrieving sampled IV data to/from
 HDF5 files.
 
 """
+
 from shepherd_core import Writer
 
 from .reader import Reader
 
-__version__ = "2023.11.1"
+__version__ = "2024.04.1"
 
 __all__ = [
     "Reader",

shepherd_data/cli.py

@@ -1,5 +1,5 @@
-"""Command definitions for CLI
-"""
+"""Command definitions for CLI"""
+
 import logging
 import os
 import sys
@@ -120,9 +120,7 @@ def extract(in_data: Path, ds_factor: float, separator: str) -> None:
                 # will create a downsampled h5-file (if not existing) and then saving to csv
                 ds_file = file.with_suffix(f".downsampled_x{round(ds_factor)}.h5")
                 if not ds_file.exists():
-                    logger.info(
-                        "Downsampling '%s' by factor x%f ...", file.name, ds_factor
-                    )
+                    logger.info("Downsampling '%s' by factor x%f ...", file.name, ds_factor)
                     with Writer(
                         ds_file,
                         mode=shpr.get_mode(),
@@ -140,12 +138,8 @@ def extract(in_data: Path, ds_factor: float, separator: str) -> None:
                             ds_factor=ds_factor,
                             is_time=True,
                         )
-                        shpr.downsample(
-                            shpr.ds_voltage, shpw.ds_voltage, ds_factor=ds_factor
-                        )
-                        shpr.downsample(
-                            shpr.ds_current, shpw.ds_current, ds_factor=ds_factor
-                        )
+                        shpr.downsample(shpr.ds_voltage, shpw.ds_voltage, ds_factor=ds_factor)
+                        shpr.downsample(shpr.ds_current, shpw.ds_current, ds_factor=ds_factor)
 
                 with Reader(ds_file, verbose=verbose_level > 2) as shpd:
                     shpd.save_csv(shpd["data"], separator)
@@ -184,6 +178,8 @@ def extract_meta(in_data: Path, separator: str) -> None:
                 for element in logs + logs_depr:
                     if element in shpr.h5file:
                         shpr.save_log(shpr[element])
+                        # TODO: allow omitting timestamp,
+                        #       also test if segmented uart is correctly written
                         shpr.warn_logs(element, show=True)
         except TypeError as _xpc:
             logger.error("ERROR: will skip file, caught exception: %s", _xpc)
@@ -212,21 +208,16 @@ def extract_uart(in_data: Path) -> None:
                 with log_path.open("w") as log_file:
                     for line in lines:
                         with suppress(TypeError):
-                            timestamp = datetime.fromtimestamp(
-                                float(line[0]), tz=local_tz()
-                            )
-                            log_file.write(
-                                timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + ":"
-                            )
+                            timestamp = datetime.fromtimestamp(float(line[0]), tz=local_tz())
+                            log_file.write(timestamp.strftime("%Y-%m-%d %H:%M:%S.%f") + ":")
+                            # TODO: allow to skip Timestamp and export raw text
                             log_file.write(f"\t{str.encode(line[1])}")
                             log_file.write("\n")
         except TypeError as _xpc:
             logger.error("ERROR: will skip file, caught exception: %s", _xpc)
 
 
-@cli.command(
-    short_help="Extracts gpio-trace from file or directory containing shepherd-recordings"
-)
+@cli.command(short_help="Extracts gpio-trace from file or directory containing shepherd-recordings")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
 @click.option(
     "--separator",
@@ -269,9 +260,7 @@ def extract_gpio(in_data: Path, separator: str) -> None:
     type=click.INT,
     help="Alternative Input to determine a downsample-factor (Choose One)",
 )
-def downsample(
-    in_data: Path, ds_factor: Optional[float], sample_rate: Optional[int]
-) -> None:
+def downsample(in_data: Path, ds_factor: Optional[float], sample_rate: Optional[int]) -> None:
     """Creates an array of downsampling-files from file
     or directory containing shepherd-recordings
     """
@@ -289,7 +278,7 @@ def downsample(
         try:
             with Reader(file, verbose=verbose_level > 2) as shpr:
                 for _factor in ds_list:
-                    if shpr.ds_time.shape[0] / _factor < 1000:
+                    if shpr.ds_voltage.shape[0] / _factor < 1000:
                         logger.warning(
                             "will skip downsampling for %s because "
                             "resulting sample-size is too small",
@@ -299,9 +288,7 @@ def downsample(
                     ds_file = file.with_suffix(f".downsampled_x{round(_factor)}.h5")
                     if ds_file.exists():
                         continue
-                    logger.info(
-                        "Downsampling '%s' by factor x%f ...", file.name, _factor
-                    )
+                    logger.info("Downsampling '%s' by factor x%f ...", file.name, _factor)
                     with Writer(
                         ds_file,
                         mode=shpr.get_mode(),
@@ -313,22 +300,14 @@ def downsample(
                         shpw["ds_factor"] = _factor
                         shpw.store_hostname(shpr.get_hostname())
                         shpw.store_config(shpr.get_config())
-                        shpr.downsample(
-                            shpr.ds_time, shpw.ds_time, ds_factor=_factor, is_time=True
-                        )
-                        shpr.downsample(
-                            shpr.ds_voltage, shpw.ds_voltage, ds_factor=_factor
-                        )
-                        shpr.downsample(
-                            shpr.ds_current, shpw.ds_current, ds_factor=_factor
-                        )
+                        shpr.downsample(shpr.ds_time, shpw.ds_time, ds_factor=_factor, is_time=True)
+                        shpr.downsample(shpr.ds_voltage, shpw.ds_voltage, ds_factor=_factor)
+                        shpr.downsample(shpr.ds_current, shpw.ds_current, ds_factor=_factor)
         except TypeError as _xpc:
             logger.error("ERROR: will skip file, caught exception: %s", _xpc)
 
 
-@cli.command(
-    short_help="Plots IV-trace from file or directory containing shepherd-recordings"
-)
+@cli.command(short_help="Plots IV-trace from file or directory containing shepherd-recordings")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
 @click.option(
     "--start",
@@ -382,13 +361,11 @@ def plot(
         try:
             with Reader(file, verbose=verbose_level > 2) as shpr:
                 if multiplot:
-                    data.append(
-                        shpr.generate_plot_data(start, end, relative_timestamp=True)
-                    )
+                    data.append(shpr.generate_plot_data(start, end, relative_timestamp=True))
                 else:
                     shpr.plot_to_file(start, end, width, height)
         except TypeError as _xpc:
-            logger.error("ERROR: will skip file, caught exception: %s", _xpc)
+            logger.exception("ERROR: will skip file, caught exception: %s", _xpc)
     if multiplot:
         logger.info("Got %d datasets to plot", len(data))
         mpl_path = Reader.multiplot_to_file(data, in_data, width, height)

shepherd_data/debug_resampler.py

@@ -24,9 +24,7 @@ with shpd.Reader(file) as shpr:
         window_samples=shpr.get_window_samples(),
         cal_data=shpr.get_calibration_data(),
     ) as shpw:
-        shpr.resample(
-            shpr.ds_time, shpw.ds_time, samplerate_dst=samplerate_sps, is_time=True
-        )
+        shpr.resample(shpr.ds_time, shpw.ds_time, samplerate_dst=samplerate_sps, is_time=True)
         shpr.resample(shpr.ds_voltage, shpw.ds_voltage, samplerate_dst=samplerate_sps)
         shpr.resample(shpr.ds_current, shpw.ds_current, samplerate_dst=samplerate_sps)
         shpw.save_metadata()

shepherd_data/ivonne.py

@@ -2,6 +2,7 @@
 to generate valid shepherd-data for emulation
 
 """
+
 import errno
 import logging
 import math
@@ -34,7 +35,7 @@ def get_isc(coeffs: pd.DataFrame):  # noqa: ANN201
 
 
 class Reader:
-    """container for converters to shepherd-data"""
+    """Container for converters that bridge the gap to shepherds data-files."""
 
     _logger: logging.Logger = logging.getLogger("SHPData.IVonne.Reader")
 
@@ -63,9 +64,7 @@ class Reader:
 
     def __enter__(self) -> Self:
         if not self.file_path.exists():
-            raise FileNotFoundError(
-                errno.ENOENT, os.strerror(errno.ENOENT), self.file_path.name
-            )
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.file_path.name)
         with self.file_path.open("rb") as ifr:
             self._df = pickle.load(ifr)  # noqa: S301
         self._refresh_file_stats()
@@ -104,8 +103,9 @@ class Reader:
         pts_per_curve: int = 1000,
         duration_s: Optional[float] = None,
     ) -> None:
-        """Transforms previously recorded parameters to shepherd hdf database with IV curves.
-        Shepherd should work with IV 'surfaces', where we have a stream of IV curves
+        """Transform recorded parameters to shepherd hdf database with IV curves.
+
+        Shepherd works with IV 'surfaces', which is a stream of IV curves.
 
         :param shp_output: Path where the resulting hdf file shall be stored
         :param v_max: Maximum voltage supported by shepherd
@@ -116,9 +116,7 @@ class Reader:
             raise RuntimeError("IVonne Context was not entered - file not open!")
         if isinstance(duration_s, (float, int)) and self.runtime_s > duration_s:
             self._logger.info("  -> gets trimmed to %f s", duration_s)
-            df_elements_n = min(
-                self._df.shape[0], int(duration_s * self.samplerate_sps)
-            )
+            df_elements_n = min(self._df.shape[0], int(duration_s * self.samplerate_sps))
         else:
             df_elements_n = self._df.shape[0]
 
@@ -128,9 +126,7 @@ class Reader:
 
         v_proto = np.linspace(0, v_max, pts_per_curve)
 
-        with Writer(
-            shp_output, datatype="ivcurve", window_samples=pts_per_curve
-        ) as sfw:
+        with Writer(shp_output, datatype="ivcurve", window_samples=pts_per_curve) as sfw:
             sfw.store_hostname("IVonne")
             curve_interval_us = round(sfw.sample_interval_ns * pts_per_curve / 1000)
             up_factor = self.sample_interval_ns // sfw.sample_interval_ns
@@ -140,9 +136,7 @@ class Reader:
             for idx in job_iter:
                 idx_top = min(idx + max_elements, df_elements_n)
                 df_slice = self._df.iloc[idx : idx_top + 1].copy()
-                df_slice["timestamp"] = pd.TimedeltaIndex(
-                    data=df_slice["time"], unit="s"
-                )
+                df_slice["timestamp"] = pd.to_timedelta(df_slice["time"], unit="s")
                 df_slice = df_slice.set_index("timestamp")
                 # warning: .interpolate does crash in debug-mode with typeError
                 df_slice = (
@@ -171,7 +165,7 @@ class Reader:
         duration_s: Optional[float] = None,
         tracker: Optional[MPPTracker] = None,
     ) -> None:
-        """Transforms shepherd IV curves to shepherd IV traces.
+        """Transform shepherd IV curves to shepherd IV samples / traces.
 
         For the 'buck' and 'buck-boost' modes, shepherd takes voltage and current traces.
         These can be recorded with shepherd or generated from existing IV curves by, for
@@ -188,14 +182,13 @@ class Reader:
         :param v_max: Maximum voltage supported by shepherd
         :param duration_s: time to stop in seconds, counted from beginning
         :param tracker: VOC or OPT
+
         """
         if self._df is None:
             raise RuntimeError("IVonne Context was not entered - file not open!")
         if isinstance(duration_s, (float, int)) and self.runtime_s > duration_s:
             self._logger.info("  -> gets trimmed to %f s", duration_s)
-            df_elements_n = min(
-                self._df.shape[0], int(duration_s * self.samplerate_sps)
-            )
+            df_elements_n = min(self._df.shape[0], int(duration_s * self.samplerate_sps))
         else:
             df_elements_n = self._df.shape[0]
 
@@ -223,17 +216,11 @@ class Reader:
                 df_slice.loc[:, "voc"] = get_voc(df_slice)
                 df_slice.loc[df_slice["voc"] >= v_max, "voc"] = v_max
                 df_slice = tracker.process(df_slice)
-                df_slice["timestamp"] = pd.TimedeltaIndex(
-                    data=df_slice["time"], unit="s"
-                )
-                df_slice = df_slice[["time", "v", "i", "timestamp"]].set_index(
-                    "timestamp"
-                )
+                df_slice["timestamp"] = pd.to_timedelta(df_slice["time"], unit="s")
+                df_slice = df_slice[["time", "v", "i", "timestamp"]].set_index("timestamp")
                 # warning: .interpolate does crash in debug-mode with typeError
                 df_slice = (
-                    df_slice.resample(f"{interval_us}us")
-                    .interpolate(method="cubic")
-                    .iloc[:-1]
+                    df_slice.resample(f"{interval_us}us").interpolate(method="cubic").iloc[:-1]
                 )
                 sfw.append_iv_data_si(
                     df_slice["time"].to_numpy(),
@@ -247,7 +234,7 @@ class Reader:
         v_max: float = 5.0,
         duration_s: Optional[float] = None,
     ) -> None:
-        """Transforms ivonne-parameters to upsampled version for shepherd
+        """Transform ivonne-parameters to up-sampled versions for shepherd.
 
         :param shp_output: Path where the resulting hdf file shall be stored
         :param v_max: Maximum voltage supported by shepherd
@@ -257,9 +244,7 @@ class Reader:
             raise RuntimeError("IVonne Context was not entered - file not open!")
         if isinstance(duration_s, (float, int)) and self.runtime_s > duration_s:
             self._logger.info("  -> gets trimmed to %f s", duration_s)
-            df_elements_n = min(
-                self._df.shape[0], int(duration_s * self.samplerate_sps)
-            )
+            df_elements_n = min(self._df.shape[0], int(duration_s * self.samplerate_sps))
         else:
             df_elements_n = self._df.shape[0]
 
@@ -282,17 +267,11 @@ class Reader:
                 df_slice.loc[:, "voc"] = get_voc(df_slice)
                 df_slice.loc[df_slice["voc"] >= v_max, "voc"] = v_max
                 df_slice.loc[:, "isc"] = get_isc(df_slice)
-                df_slice["timestamp"] = pd.TimedeltaIndex(
-                    data=df_slice["time"], unit="s"
-                )
-                df_slice = df_slice[["time", "voc", "isc", "timestamp"]].set_index(
-                    "timestamp"
-                )
+                df_slice["timestamp"] = pd.to_timedelta(df_slice["time"], unit="s")
+                df_slice = df_slice[["time", "voc", "isc", "timestamp"]].set_index("timestamp")
                 # warning: .interpolate does crash in debug-mode with typeError
                 df_slice = (
-                    df_slice.resample(f"{interval_us}us")
-                    .interpolate(method="cubic")
-                    .iloc[:-1]
+                    df_slice.resample(f"{interval_us}us").interpolate(method="cubic").iloc[:-1]
                 )
                 sfw.append_iv_data_si(
                     df_slice["time"].to_numpy(),

shepherd_data/mppt.py

@@ -1,6 +1,8 @@
 """Harvesters, simple and fast approach.
-Might be exchanged by shepherds py-model of pru-harvesters
+
+Might be exchanged by shepherds py-model of pru-harvesters.
 """
+
 import numpy as np
 import pandas as pd
 
@@ -8,16 +10,11 @@ from shepherd_core import Calc_t
 
 
 def iv_model(voltages: Calc_t, coeffs: pd.Series) -> Calc_t:
-    """Simple diode based model of a solar panel IV curve.
-
-    Args:
-    ----
-        :param voltages: Load voltage of the solar panel
-        :param coeffs: three generic coefficients
+    """Calculate simple diode based model (equivalent circuit diagram) of a solar panel IV curve.
 
-    Returns:
-    -------
-        Solar current at given load voltage
+    :param voltages: Load voltage of the solar panel
+    :param coeffs: three generic coefficients
+    :return: Solar current at given load voltage
     """
     currents = float(coeffs["a"]) - float(coeffs["b"]) * (
         np.exp(float(coeffs["c"]) * voltages) - 1.0
@@ -40,7 +37,7 @@ def find_oc(v_arr: np.ndarray, i_arr: np.ndarray, ratio: float = 0.05) -> np.nda
 
 
 class MPPTracker:
-    """Prototype
+    """Prototype for a MPPT-class.
 
     :param v_max: Maximum voltage supported by shepherd
     :param pts_per_curve: resolution of internal ivcurve
@@ -52,34 +49,34 @@ class MPPTracker:
         self.v_proto: np.ndarray = np.linspace(0, v_max, pts_per_curve)
 
     def process(self, coeffs: pd.DataFrame) -> pd.DataFrame:
-        """Apply harvesting model to input data
+        """Apply harvesting model to input data.
 
         :param coeffs: ivonne coefficients
-        :return:
+        :return: ivsample-data
         """
-        pass
 
 
 class OpenCircuitTracker(MPPTracker):
-    """Open-circuit based MPPT
+    """Open-circuit (-voltage) based MPPT.
 
     :param v_max: Maximum voltage supported by shepherd
     :param pts_per_curve: resolution of internal ivcurve
     :param ratio:  (float) Ratio of open-circuit voltage to track
     """
 
-    def __init__(
-        self, v_max: float = 5.0, pts_per_curve: int = 1000, ratio: float = 0.8
-    ) -> None:
+    def __init__(self, v_max: float = 5.0, pts_per_curve: int = 1000, ratio: float = 0.8) -> None:
         super().__init__(v_max, pts_per_curve)
         self.ratio = ratio
 
     def process(self, coeffs: pd.DataFrame) -> pd.DataFrame:
+        """Apply harvesting model to input data.
+
+        :param coeffs: ivonne coefficients
+        :return: ivsample-data
+        """
         coeffs["icurve"] = coeffs.apply(lambda x: iv_model(self.v_proto, x), axis=1)
         if "voc" not in coeffs.columns:
-            coeffs["voc"] = coeffs.apply(
-                lambda x: find_oc(self.v_proto, x["ivcurve"]), axis=1
-            )
+            coeffs["voc"] = coeffs.apply(lambda x: find_oc(self.v_proto, x["ivcurve"]), axis=1)
         coeffs["rvoc_pos"] = coeffs.apply(
             lambda x: np.argmax(self.v_proto[self.v_proto < self.ratio * x["voc"]]),
             axis=1,
@@ -90,7 +87,7 @@ class OpenCircuitTracker(MPPTracker):
 
 
 class OptimalTracker(MPPTracker):
-    """Optimal MPPT
+    """Optimal MPPT by looking at the whole curve.
 
     Calculates optimal harvesting voltage for every time and corresponding IV curve.
 
@@ -102,6 +99,11 @@ class OptimalTracker(MPPTracker):
         super().__init__(v_max, pts_per_curve)
 
     def process(self, coeffs: pd.DataFrame) -> pd.DataFrame:
+        """Apply harvesting model to input data.
+
+        :param coeffs: ivonne coefficients
+        :return: ivsample-data
+        """
         coeffs["icurve"] = coeffs.apply(lambda x: iv_model(self.v_proto, x), axis=1)
         coeffs["pcurve"] = coeffs.apply(lambda x: self.v_proto * x["icurve"], axis=1)
         coeffs["max_pos"] = coeffs.apply(lambda x: np.argmax(x["pcurve"]), axis=1)