shepherd-data 2024.9.1__py3-none-any.whl → 2024.11.1__py3-none-any.whl

shepherd_data/__init__.py

@@ -11,7 +11,7 @@ from shepherd_core import Writer
 
 from .reader import Reader
 
-__version__ = "2024.9.1"
+__version__ = "2024.11.1"
 
 __all__ = [
     "Reader",

shepherd_data/cli.py

@@ -10,13 +10,12 @@ from typing import List
 from typing import Optional
 
 import click
+import pydantic
 
 from shepherd_core import get_verbose_level
-from shepherd_core import increase_verbose_level
 from shepherd_core import local_tz
-from shepherd_core.commons import samplerate_sps_default
+from shepherd_core.logger import set_log_verbose_level
 
-from . import Writer
 from . import __version__
 from .reader import Reader
 
@@ -50,26 +49,25 @@ def path_to_flist(data_path: Path) -> List[Path]:
     "--verbose",
     "-v",
     is_flag=True,
-    help="4 Levels [0..3](Error, Warning, Info, Debug)",
-)
-@click.option(
-    "--version",
-    is_flag=True,
-    help="Prints version-info at start (combinable with -v)",
+    help="Switch from info- to debug-level",
 )
 @click.pass_context  # TODO: is the ctx-type correct?
-def cli(ctx: click.Context, *, verbose: bool, version: bool) -> None:
+def cli(ctx: click.Context, *, verbose: bool) -> None:
     """Shepherd: Synchronized Energy Harvesting Emulator and Recorder."""
-    if verbose:
-        increase_verbose_level(3)
-    if version:
-        logger.info("Shepherd-Data v%s", __version__)
-        logger.debug("Python v%s", sys.version)
-        logger.debug("Click v%s", click.__version__)
+    set_log_verbose_level(logger, 3 if verbose else 2)
     if not ctx.invoked_subcommand:
         click.echo("Please specify a valid command")
 
 
+@cli.command(short_help="Print version-info (combine with -v for more)")
+def version() -> None:
+    """Print version-info (combine with -v for more)."""
+    logger.info("Shepherd-Data v%s", __version__)
+    logger.debug("Python v%s", sys.version)
+    logger.debug("Click v%s", click.__version__)
+    logger.debug("Pydantic v%s", pydantic.__version__)
+
+
 @cli.command(short_help="Validates a file or directory containing shepherd-recordings")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
 def validate(in_data: Path) -> None:
@@ -94,6 +92,20 @@ def validate(in_data: Path) -> None:
 
 @cli.command(short_help="Extracts recorded IVSamples and stores it to csv")
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
+@click.option(
+    "--start",
+    "-s",
+    default=None,
+    type=click.FLOAT,
+    help="Start-point in seconds, will be 0 if omitted",
+)
+@click.option(
+    "--end",
+    "-e",
+    default=None,
+    type=click.FLOAT,
+    help="End-point in seconds, will be max if omitted",
+)
 @click.option(
     "--ds-factor",
     "-f",
@@ -103,12 +115,25 @@ def validate(in_data: Path) -> None:
 )
 @click.option(
     "--separator",
-    "-s",
     default=";",
     type=click.STRING,
     help="Set an individual csv-separator",
 )
-def extract(in_data: Path, ds_factor: float, separator: str) -> None:
+@click.option(
+    "--raw",
+    "-r",
+    is_flag=True,
+    help="Plot only power instead of voltage, current & power",
+)
+def extract(
+    in_data: Path,
+    start: Optional[float],
+    end: Optional[float],
+    ds_factor: float,
+    separator: str,
+    *,
+    raw: bool = False,
+) -> None:
     """Extract recorded IVSamples and store them to csv."""
     files = path_to_flist(in_data)
     verbose_level = get_verbose_level()
@@ -119,41 +144,10 @@ def extract(in_data: Path, ds_factor: float, separator: str) -> None:
         logger.info("Extracting IV-Samples from '%s' ...", file.name)
         try:
             with Reader(file, verbose=verbose_level > 2) as shpr:
-                # TODO: this code is very similar to data.reader.downsample()
-                if (shpr.ds_voltage.shape[0] / ds_factor) < 10:
-                    logger.warning(
-                        "will skip downsampling for %s because "
-                        "resulting sample-size is too small",
-                        file.name,
-                    )
-                    continue
-                # will create a downsampled h5-file (if not existing) and then saving to csv
-                ds_file = file.with_suffix(f".downsampled_x{round(ds_factor)}.h5")
-                if not ds_file.exists():
-                    logger.info("Downsampling '%s' by factor x%f ...", file.name, ds_factor)
-                    with Writer(
-                        ds_file,
-                        mode=shpr.get_mode(),
-                        datatype=shpr.get_datatype(),
-                        window_samples=shpr.get_window_samples(),
-                        cal_data=shpr.get_calibration_data(),
-                        verbose=verbose_level > 2,
-                    ) as shpw:
-                        shpw["ds_factor"] = ds_factor
-                        shpw.store_hostname(shpr.get_hostname())
-                        shpw.store_config(shpr.get_config())
-                        shpr.downsample(
-                            shpr.ds_time,
-                            shpw.ds_time,
-                            ds_factor=ds_factor,
-                            is_time=True,
-                        )
-                        shpr.downsample(shpr.ds_voltage, shpw.ds_voltage, ds_factor=ds_factor)
-                        shpr.downsample(shpr.ds_current, shpw.ds_current, ds_factor=ds_factor)
-
-                with Reader(ds_file, verbose=verbose_level > 2) as shpd:
-                    shpd.save_csv(shpd["data"], separator)
-        except TypeError:
+                out_file = shpr.cut_and_downsample_to_file(start, end, ds_factor=ds_factor)
+                with Reader(out_file, verbose=verbose_level > 2) as shpd:
+                    shpd.save_csv(shpd["data"], separator, raw=raw)
+        except (TypeError, ValueError):
             logger.exception("ERROR: Will skip file. It caused an exception.")
 
 
@@ -256,11 +250,10 @@ def extract_gpio(in_data: Path, separator: str) -> None:
 
 
 @cli.command(
-    short_help="Creates an array of downsampling-files from "
+    short_help="Creates an array of down-sampled files from "
     "file or directory containing shepherd-recordings"
 )
 @click.argument("in_data", type=click.Path(exists=True, resolve_path=True))
-# @click.option("--out_data", "-o", type=click.Path(resolve_path=True))
 @click.option(
     "--ds-factor",
     "-f",
@@ -274,48 +267,45 @@ def extract_gpio(in_data: Path, separator: str) -> None:
     type=click.INT,
     help="Alternative Input to determine a downsample-factor (Choose One)",
 )
-def downsample(in_data: Path, ds_factor: Optional[float], sample_rate: Optional[int]) -> None:
+@click.option(
+    "--start",
+    "-s",
+    default=None,
+    type=click.FLOAT,
+    help="Start-point in seconds, will be 0 if omitted",
+)
+@click.option(
+    "--end",
+    "-e",
+    default=None,
+    type=click.FLOAT,
+    help="End-point in seconds, will be max if omitted",
+)
+def downsample(
+    in_data: Path,
+    ds_factor: Optional[float],
+    sample_rate: Optional[int],
+    start: Optional[float],
+    end: Optional[float],
+) -> None:
     """Create an array of down-sampled files from file or dir containing shepherd-recordings."""
-    if ds_factor is None and sample_rate is not None and sample_rate >= 1:
-        ds_factor = int(samplerate_sps_default / sample_rate)
-        # TODO: shouldn't current sps be based on file rather than default?
-    if isinstance(ds_factor, (float, int)) and ds_factor >= 1:
-        ds_list = [ds_factor]
-    else:
-        ds_list = [5, 25, 100, 500, 2_500, 10_000, 50_000, 250_000, 1_000_000]
-
     files = path_to_flist(in_data)
     verbose_level = get_verbose_level()
     for file in files:
         try:
             with Reader(file, verbose=verbose_level > 2) as shpr:
+                if ds_factor is None and sample_rate is not None and sample_rate >= 1:
+                    ds_factor = shpr.samplerate_sps / sample_rate
+
+                if isinstance(ds_factor, (float, int)) and ds_factor >= 1:
+                    ds_list = [ds_factor]
+                else:
+                    ds_list = [5, 25, 100, 500, 2_500, 10_000, 50_000, 250_000, 1_000_000]
+
                 for _factor in ds_list:
-                    if (shpr.ds_voltage.shape[0] / _factor) < 1000:
-                        logger.warning(
-                            "will skip downsampling for %s because "
-                            "resulting sample-size is too small",
-                            file.name,
-                        )
-                        break
-                    ds_file = file.with_suffix(f".downsampled_x{round(_factor)}.h5")
-                    if ds_file.exists():
-                        continue
-                    logger.info("Downsampling '%s' by factor x%f ...", file.name, _factor)
-                    with Writer(
-                        ds_file,
-                        mode=shpr.get_mode(),
-                        datatype=shpr.get_datatype(),
-                        window_samples=shpr.get_window_samples(),
-                        cal_data=shpr.get_calibration_data(),
-                        verbose=verbose_level > 2,
-                    ) as shpw:
-                        shpw["ds_factor"] = _factor
-                        shpw.store_hostname(shpr.get_hostname())
-                        shpw.store_config(shpr.get_config())
-                        shpr.downsample(shpr.ds_time, shpw.ds_time, ds_factor=_factor, is_time=True)
-                        shpr.downsample(shpr.ds_voltage, shpw.ds_voltage, ds_factor=_factor)
-                        shpr.downsample(shpr.ds_current, shpw.ds_current, ds_factor=_factor)
-        except TypeError:
+                    path_file = shpr.cut_and_downsample_to_file(start, end, _factor)
+                    logger.info("Created %s", path_file.name)
+        except (TypeError, ValueError):  # noqa: PERF203
             logger.exception("ERROR: Will skip file. It caused an exception.")
 
 
@@ -326,14 +316,14 @@ def downsample(in_data: Path, ds_factor: Optional[float], sample_rate: Optional[
     "-s",
     default=None,
     type=click.FLOAT,
-    help="Start of plot in seconds, will be 0 if omitted",
+    help="Start-point in seconds, will be 0 if omitted",
 )
 @click.option(
     "--end",
     "-e",
     default=None,
     type=click.FLOAT,
-    help="End of plot in seconds, will be max if omitted",
+    help="End-point in seconds, will be max if omitted",
 )
 @click.option(
     "--width",

shepherd_data/reader.py

@@ -13,7 +13,9 @@ from matplotlib import pyplot as plt
 from tqdm import trange
 
 from shepherd_core import Reader as CoreReader
+from shepherd_core import Writer as CoreWriter
 from shepherd_core import local_tz
+from shepherd_core.logger import get_verbose_level
 from shepherd_core.logger import logger
 
 # import samplerate  # noqa: ERA001, TODO: just a test-fn for now
@@ -37,11 +39,12 @@ class Reader(CoreReader):
     ) -> None:
         super().__init__(file_path, verbose=verbose)
 
-    def save_csv(self, h5_group: h5py.Group, separator: str = ";") -> int:
+    def save_csv(self, h5_group: h5py.Group, separator: str = ";", *, raw: bool = False) -> int:
         """Extract numerical data from group and store it into csv.
 
         :param h5_group: can be external and should probably be downsampled
         :param separator: used between columns
+        :param raw: don't convert to si-units
         :return: number of processed entries
         """
         if ("time" not in h5_group) or (h5_group["time"].shape[0] < 1):
@@ -62,11 +65,21 @@ class Reader(CoreReader):
         with csv_path.open("w", encoding="utf-8-sig") as csv_file:
             self._logger.info("CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name)
             csv_file.write(header + "\n")
+            ts_gain = h5_group["time"].attrs.get("gain", 1e-9)
+            # for converting data to si - if raw=false
+            gains: dict[str, float] = {
+                key: h5_group[key].attrs.get("gain", 1.0) for key in datasets[1:]
+            }
+            offsets: dict[str, float] = {
+                key: h5_group[key].attrs.get("offset", 1.0) for key in datasets[1:]
+            }
             for idx, time_ns in enumerate(h5_group["time"][:]):
-                timestamp = datetime.fromtimestamp(time_ns / 1e9, tz=local_tz())
+                timestamp = datetime.fromtimestamp(time_ns * ts_gain, tz=local_tz())
                 csv_file.write(timestamp.strftime("%Y-%m-%d %H:%M:%S.%f"))
                 for key in datasets[1:]:
                     values = h5_group[key][idx]
+                    if not raw:
+                        values = values * gains[key] + offsets[key]
                     if isinstance(values, np.ndarray):
                         values = separator.join([str(value) for value in values])
                     csv_file.write(f"{separator}{values}")
@@ -225,6 +238,115 @@ class Reader(CoreReader):
             data_dst.resize((oblock_len * (iterations - 1) + slice_len,))
         return data_dst
 
+    def cut_and_downsample_to_file(
+        self,
+        start_s: Optional[float],
+        end_s: Optional[float],
+        ds_factor: Optional[float],
+    ) -> Path:
+        """Cut source to given limits, downsample by factor and store result in separate file.
+
+        Resulting file-name is derived from input-name by adding
+        - ".cut_x_to_y" and
+        - ".downsample_x"
+        when applicable.
+        """
+        # prepare cut
+        if not isinstance(start_s, (float, int)):
+            start_s = 0
+        if not isinstance(end_s, (float, int)):
+            end_s = self.runtime_s
+        start_s = max(0, start_s)
+        end_s = min(self.runtime_s, end_s)
+
+        start_sample = round(start_s * self.samplerate_sps)
+        end_sample = round(end_s * self.samplerate_sps)
+
+        # test input-parameters
+        if end_sample < start_sample:
+            raise ValueError(
+                "Cut & downsample for %s failed because "
+                "end-mark (%.3f) is before start-mark (%.3f).",
+                self.file_path.name,
+                end_s,
+                start_s,
+            )
+        if ds_factor < 1:
+            raise ValueError(
+                "Cut & downsample for %s failed because factor < 1",
+                self.file_path.name,
+            )
+        if ((end_sample - start_sample) / ds_factor) < 1000:
+            raise ValueError(
+                "Cut & downsample for %s failed because " "resulting sample-size is too small",
+                self.file_path.name,
+            )
+
+        # assemble file-name of output
+        if start_s != 0.0 or end_s != self.runtime_s:
+            start_str = f"{start_s:.3f}".replace(".", "s")
+            end_str = f"{end_s:.3f}".replace(".", "s")
+            cut_str = f".cut_{start_str}_to_{end_str}"
+        else:
+            cut_str = ""
+
+        if ds_factor > 1:  # noqa: SIM108
+            ds_str = f".downsample_x{round(ds_factor)}"
+        else:
+            ds_str = ""
+
+        dst_file = self.file_path.resolve().with_suffix(cut_str + ds_str + ".h5")
+        if dst_file.exists():
+            logger.warning(
+                "Cut & Downsample skipped because output-file %s already exists.", dst_file.name
+            )
+            return dst_file
+
+        logger.debug(
+            "Cut & Downsample '%s' from %.3f s to %.3f s with factor = %.1f ...",
+            self.file_path.name,
+            start_s,
+            end_s,
+            ds_factor,
+        )
+
+        # convert data
+        with CoreWriter(
+            dst_file,
+            mode=self.get_mode(),
+            datatype=self.get_datatype(),
+            window_samples=self.get_window_samples(),
+            cal_data=self.get_calibration_data(),
+            verbose=get_verbose_level() > 2,
+        ) as shpw:
+            shpw["ds_factor"] = ds_factor
+            shpw.store_hostname(self.get_hostname())
+            shpw.store_config(self.get_config())
+            self.downsample(
+                self.ds_time,
+                shpw.ds_time,
+                start_n=start_sample,
+                end_n=end_sample,
+                ds_factor=ds_factor,
+                is_time=True,
+            )
+            self.downsample(
+                self.ds_voltage,
+                shpw.ds_voltage,
+                start_n=start_sample,
+                end_n=end_sample,
+                ds_factor=ds_factor,
+            )
+            self.downsample(
+                self.ds_current,
+                shpw.ds_current,
+                start_n=start_sample,
+                end_n=end_sample,
+                ds_factor=ds_factor,
+            )
+
+        return dst_file
+
     def resample(
         self,
         data_src: Union[h5py.Dataset, np.ndarray],
@@ -348,6 +470,8 @@ class Reader(CoreReader):
             start_s = 0
         if not isinstance(end_s, (float, int)):
             end_s = self.runtime_s
+        start_s = max(0, start_s)
+        end_s = min(self.runtime_s, end_s)
         start_sample = round(start_s * self.samplerate_sps)
         end_sample = round(end_s * self.samplerate_sps)
         if end_sample - start_sample < 5:

{shepherd_data-2024.9.1.dist-info → shepherd_data-2024.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shepherd_data
-Version: 2024.9.1
+Version: 2024.11.1
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -36,7 +36,7 @@ Requires-Dist: numpy
 Requires-Dist: pandas >=2.0.0
 Requires-Dist: pyYAML
 Requires-Dist: scipy
-Requires-Dist: shepherd-core[inventory] >=2024.9.1
+Requires-Dist: shepherd-core[inventory] >=2024.11.1
 Requires-Dist: tqdm
 Provides-Extra: dev
 Requires-Dist: shepherd-core[dev] ; extra == 'dev'

shepherd_data-2024.11.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+shepherd_data/__init__.py,sha256=Bj7oLrhZ_tH1yjPc42MFFUP6uyTxS7cQtCQJFFiM4Rs,353
+shepherd_data/cli.py,sha256=6ndpKTZxXYOIogOTxD0Eqha6TtRSLn7_Imd_T6wzidA,13670
+shepherd_data/ivonne.py,sha256=IaHy7RizdaaLAo-cQS-S5xQW_TzHW3dcJjb9hkTLjjY,11900
+shepherd_data/mppt.py,sha256=588KSrLuJfNRKKnnL6ewePLi3zrwaO_PAZypikACrks,3925
+shepherd_data/reader.py,sha256=Fu5QziYNS8ffsWIc4S__Ln_0aF83v-endzpQ5s4seNE,24460
+shepherd_data-2024.11.1.dist-info/METADATA,sha256=jC6j7wcIexQp1zRyLspS7SI8kSNBuUAKwNcBICo8z8M,3440
+shepherd_data-2024.11.1.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
+shepherd_data-2024.11.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
+shepherd_data-2024.11.1.dist-info/top_level.txt,sha256=7-SCTY-TG1mLY72OVKCaqte1hy-X8woxknIUAD3OIxs,14
+shepherd_data-2024.11.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+shepherd_data-2024.11.1.dist-info/RECORD,,

{shepherd_data-2024.9.1.dist-info → shepherd_data-2024.11.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (74.1.2)
+Generator: setuptools (75.3.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

shepherd_data-2024.9.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-shepherd_data/__init__.py,sha256=7qoBbrIem27hArbLNss7PYbdNcvRCzuLegXcJx4skHA,352
-shepherd_data/cli.py,sha256=pv17ILFQCdPSyWgmw0rQIhdsYucqeiWFwJn8hft4ga0,15760
-shepherd_data/ivonne.py,sha256=IaHy7RizdaaLAo-cQS-S5xQW_TzHW3dcJjb9hkTLjjY,11900
-shepherd_data/mppt.py,sha256=588KSrLuJfNRKKnnL6ewePLi3zrwaO_PAZypikACrks,3925
-shepherd_data/reader.py,sha256=eShdgACQLwoOtOYIyB3CjvUgJlUEbRDcUso_3m02C9U,20098
-shepherd_data-2024.9.1.dist-info/METADATA,sha256=34fTRjrhjV1N_nyJtNCprz91K-IjafBywmdR_TBZZis,3438
-shepherd_data-2024.9.1.dist-info/WHEEL,sha256=cVxcB9AmuTcXqmwrtPhNK88dr7IR_b6qagTj0UvIEbY,91
-shepherd_data-2024.9.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
-shepherd_data-2024.9.1.dist-info/top_level.txt,sha256=7-SCTY-TG1mLY72OVKCaqte1hy-X8woxknIUAD3OIxs,14
-shepherd_data-2024.9.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-shepherd_data-2024.9.1.dist-info/RECORD,,