shepherd-data 2023.11.1__py3-none-any.whl → 2024.4.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
shepherd_data/reader.py CHANGED
@@ -1,5 +1,5 @@
1
- """Reader-Baseclass
2
- """
1
+ """Reader-Baseclass for opening shepherds hdf5-files."""
2
+
3
3
  import math
4
4
  from datetime import datetime
5
5
  from pathlib import Path
@@ -26,6 +26,7 @@ class Reader(CoreReader):
26
26
  ----
27
27
  file_path: Path of hdf5 file containing shepherd data with iv-samples, iv-curves or isc&voc
28
28
  verbose: more info during usage, 'None' skips the setter
29
+
29
30
  """
30
31
 
31
32
  def __init__(
@@ -37,7 +38,7 @@ class Reader(CoreReader):
37
38
  super().__init__(file_path, verbose=verbose)
38
39
 
39
40
  def save_csv(self, h5_group: h5py.Group, separator: str = ";") -> int:
40
- """Extract numerical data via csv
41
+ """Extract numerical data from group and store it into csv.
41
42
 
42
43
  :param h5_group: can be external and should probably be downsampled
43
44
  :param separator: used between columns
@@ -52,21 +53,14 @@ class Reader(CoreReader):
52
53
  if csv_path.exists():
53
54
  self._logger.info("File already exists, will skip '%s'", csv_path.name)
54
55
  return 0
55
- datasets = [
56
- key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group
57
- ]
56
+ datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
58
57
  datasets.remove("time")
59
58
  datasets = ["time", *datasets]
60
59
  separator = separator.strip().ljust(2)
61
- header = [
62
- h5_group[key].attrs["description"].replace(", ", separator)
63
- for key in datasets
64
- ]
60
+ header = [h5_group[key].attrs["description"].replace(", ", separator) for key in datasets]
65
61
  header = separator.join(header)
66
62
  with csv_path.open("w", encoding="utf-8-sig") as csv_file:
67
- self._logger.info(
68
- "CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name
69
- )
63
+ self._logger.info("CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name)
70
64
  csv_file.write(header + "\n")
71
65
  for idx, time_ns in enumerate(h5_group["time"][:]):
72
66
  timestamp = datetime.fromtimestamp(time_ns / 1e9, tz=local_tz())
@@ -80,7 +74,7 @@ class Reader(CoreReader):
80
74
  return h5_group["time"][:].shape[0]
81
75
 
82
76
  def save_log(self, h5_group: h5py.Group, *, add_timestamp: bool = True) -> int:
83
- """Save dataset in group as log, optimal for logged dmesg and exceptions
77
+ """Save dataset from group as log, optimal for logged 'dmesg' and console-output.
84
78
 
85
79
  :param h5_group: can be external
86
80
  :param add_timestamp: can be external
@@ -95,14 +89,10 @@ class Reader(CoreReader):
95
89
  if log_path.exists():
96
90
  self._logger.info("File already exists, will skip '%s'", log_path.name)
97
91
  return 0
98
- datasets = [
99
- key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group
100
- ]
92
+ datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
101
93
  datasets.remove("time")
102
94
  with log_path.open("w", encoding="utf-8-sig") as log_file:
103
- self._logger.info(
104
- "Log-Generator will save '%s' to '%s'", h5_group.name, log_path.name
105
- )
95
+ self._logger.info("Log-Generator will save '%s' to '%s'", h5_group.name, log_path.name)
106
96
  for idx, time_ns in enumerate(h5_group["time"][:]):
107
97
  if add_timestamp:
108
98
  timestamp = datetime.fromtimestamp(time_ns / 1e9, local_tz())
@@ -125,6 +115,7 @@ class Reader(CoreReader):
125
115
  *,
126
116
  show: bool = True,
127
117
  ) -> int:
118
+ """Print warning messages from log in data-group."""
128
119
  _count = self.count_errors_in_log(group_name, min_level)
129
120
  if _count < 1:
130
121
  return 0
@@ -156,7 +147,7 @@ class Reader(CoreReader):
156
147
 
157
148
  def downsample(
158
149
  self,
159
- data_src: h5py.Dataset,
150
+ data_src: Union[h5py.Dataset, np.ndarray],
160
151
  data_dst: Union[None, h5py.Dataset, np.ndarray],
161
152
  start_n: int = 0,
162
153
  end_n: Optional[int] = None,
@@ -164,15 +155,17 @@ class Reader(CoreReader):
164
155
  *,
165
156
  is_time: bool = False,
166
157
  ) -> Union[h5py.Dataset, np.ndarray]:
167
- """Warning: only valid for IV-Stream, not IV-Curves
158
+ """Sample down iv-data.
159
+
160
+ Warning: only valid for IV-Stream, not IV-Curves
168
161
 
169
162
  :param data_src: a h5-dataset to digest, can be external
170
163
  :param data_dst: can be a dataset, numpy-array or None (will be created internally then)
171
164
  :param start_n: start-sample
172
165
  :param end_n: ending-sample (not included)
173
- :param ds_factor: downsampling-factor
174
- :param is_time: time is not really downsamples, but just decimated
175
- :return: downsampled h5-dataset or numpy-array
166
+ :param ds_factor: sampling-factor
167
+ :param is_time: time is not really down-sampled, but decimated
168
+ :return: resampled h5-dataset or numpy-array
176
169
  """
177
170
  from scipy import signal # here due to massive delay
178
171
 
@@ -219,27 +212,21 @@ class Reader(CoreReader):
219
212
  leave=False,
220
213
  disable=iterations < 8,
221
214
  ):
222
- slice_ds = data_src[
223
- start_n + _iter * iblock_len : start_n + (_iter + 1) * iblock_len
224
- ]
215
+ slice_ds = data_src[start_n + _iter * iblock_len : start_n + (_iter + 1) * iblock_len]
225
216
  if not is_time and ds_factor > 1:
226
217
  slice_ds, f_state = signal.sosfilt(filter_, slice_ds, zi=f_state)
227
218
  slice_ds = slice_ds[::ds_factor]
228
219
  slice_len = min(dest_len - _iter * oblock_len, oblock_len)
229
- data_dst[_iter * oblock_len : (_iter + 1) * oblock_len] = slice_ds[
230
- :slice_len
231
- ]
220
+ data_dst[_iter * oblock_len : (_iter + 1) * oblock_len] = slice_ds[:slice_len]
232
221
  if isinstance(data_dst, np.ndarray):
233
- data_dst.resize(
234
- (oblock_len * (iterations - 1) + slice_len,), refcheck=False
235
- )
222
+ data_dst.resize((oblock_len * (iterations - 1) + slice_len,), refcheck=False)
236
223
  else:
237
224
  data_dst.resize((oblock_len * (iterations - 1) + slice_len,))
238
225
  return data_dst
239
226
 
240
227
  def resample(
241
228
  self,
242
- data_src: h5py.Dataset,
229
+ data_src: Union[h5py.Dataset, np.ndarray],
243
230
  data_dst: Union[None, h5py.Dataset, np.ndarray],
244
231
  start_n: int = 0,
245
232
  end_n: Optional[int] = None,
@@ -247,17 +234,17 @@ class Reader(CoreReader):
247
234
  *,
248
235
  is_time: bool = False,
249
236
  ) -> Union[h5py.Dataset, np.ndarray]:
250
- """:param data_src:
251
- :param data_dst:
252
- :param start_n:
253
- :param end_n:
254
- :param samplerate_dst:
255
- :param is_time:
256
- :return:
237
+ """Up- or down-sample the original trace-data.
238
+
239
+ :param data_src: original iv-data
240
+ :param data_dst: resampled iv-traces
241
+ :param start_n: start index of the source
242
+ :param end_n: end index of the source
243
+ :param samplerate_dst: desired sampling rate
244
+ :param is_time: time-array is handled differently than IV-Samples
245
+ :return: resampled iv-data
257
246
  """
258
- self._logger.error(
259
- "Resampling is still under construction - do not use for now!"
260
- )
247
+ self._logger.error("Resampling is still under construction - do not use for now!")
261
248
  if self.get_datatype() == "ivcurve":
262
249
  self._logger.warning("Resampling-Function was not written for IVCurves")
263
250
 
@@ -331,7 +318,6 @@ class Reader(CoreReader):
331
318
  slice_out_now = slice_out_nxt
332
319
  resampler.reset()
333
320
  """
334
- pass
335
321
 
336
322
  if isinstance(data_dst, np.ndarray):
337
323
  data_dst.resize((slice_out_now,), refcheck=False)
@@ -347,7 +333,7 @@ class Reader(CoreReader):
347
333
  *,
348
334
  relative_timestamp: bool = True,
349
335
  ) -> Dict:
350
- """Provides down-sampled iv-data that can be feed into plot_to_file()
336
+ """Provide down-sampled iv-data that can be fed into plot_to_file().
351
337
 
352
338
  :param start_s: time in seconds, relative to start of recording
353
339
  :param end_s: time in seconds, relative to start of recording
@@ -377,14 +363,10 @@ class Reader(CoreReader):
377
363
  )
378
364
  ).astype(float),
379
365
  "voltage": self._cal.voltage.raw_to_si(
380
- self.downsample(
381
- self.ds_voltage, None, start_sample, end_sample, ds_factor
382
- )
366
+ self.downsample(self.ds_voltage, None, start_sample, end_sample, ds_factor)
383
367
  ),
384
368
  "current": self._cal.current.raw_to_si(
385
- self.downsample(
386
- self.ds_current, None, start_sample, end_sample, ds_factor
387
- )
369
+ self.downsample(self.ds_current, None, start_sample, end_sample, ds_factor)
388
370
  ),
389
371
  "start_s": start_s,
390
372
  "end_s": end_s,
@@ -394,17 +376,16 @@ class Reader(CoreReader):
394
376
  return data
395
377
 
396
378
  @staticmethod
397
- def assemble_plot(
398
- data: Union[dict, list], width: int = 20, height: int = 10
399
- ) -> plt.Figure:
400
- """TODO: add power (if wanted)
379
+ def assemble_plot(data: Union[dict, list], width: int = 20, height: int = 10) -> plt.Figure:
380
+ """Create the actual figure.
401
381
 
402
382
  :param data: plottable / down-sampled iv-data with some meta-data
403
383
  -> created with generate_plot_data()
404
384
  :param width: plot-width
405
385
  :param height: plot-height
406
- :return:
386
+ :return: figure
407
387
  """
388
+ # TODO: add power (if wanted)
408
389
  if isinstance(data, dict):
409
390
  data = [data]
410
391
  fig, axes = plt.subplots(2, 1, sharex="all")
@@ -429,8 +410,9 @@ class Reader(CoreReader):
429
410
  width: int = 20,
430
411
  height: int = 10,
431
412
  ) -> None:
432
- """Creates (down-sampled) IV-Plot
433
- -> omitting start- and end-time will use the whole duration
413
+ """Create (down-sampled) IV-Plots.
414
+
415
+ Omitting start- and end-time will use the whole trace (full duration).
434
416
 
435
417
  :param start_s: time in seconds, relative to start of recording, optional
436
418
  :param end_s: time in seconds, relative to start of recording, optional
@@ -444,9 +426,7 @@ class Reader(CoreReader):
444
426
 
445
427
  start_str = f"{data[0]['start_s']:.3f}".replace(".", "s")
446
428
  end_str = f"{data[0]['end_s']:.3f}".replace(".", "s")
447
- plot_path = self.file_path.resolve().with_suffix(
448
- f".plot_{start_str}_to_{end_str}.png"
449
- )
429
+ plot_path = self.file_path.resolve().with_suffix(f".plot_{start_str}_to_{end_str}.png")
450
430
  if plot_path.exists():
451
431
  self._logger.warning("Plot exists, will skip & not overwrite!")
452
432
  return
@@ -460,7 +440,7 @@ class Reader(CoreReader):
460
440
  def multiplot_to_file(
461
441
  data: list, plot_path: Path, width: int = 20, height: int = 10
462
442
  ) -> Optional[Path]:
463
- """Creates (down-sampled) IV-Multi-Plot
443
+ """Create (down-sampled) IV-Multi-Plots (of more than one trace).
464
444
 
465
445
  :param data: plottable / down-sampled iv-data with some meta-data
466
446
  -> created with generate_plot_data()
@@ -471,9 +451,7 @@ class Reader(CoreReader):
471
451
  start_str = f"{data[0]['start_s']:.3f}".replace(".", "s")
472
452
  end_str = f"{data[0]['end_s']:.3f}".replace(".", "s")
473
453
  plot_path = (
474
- Path(plot_path)
475
- .resolve()
476
- .with_suffix(f".multiplot_{start_str}_to_{end_str}.png")
454
+ Path(plot_path).resolve().with_suffix(f".multiplot_{start_str}_to_{end_str}.png")
477
455
  )
478
456
  if plot_path.exists():
479
457
  logger.warning("Plot exists, will skip & not overwrite!")
@@ -0,0 +1,88 @@
1
+ Metadata-Version: 2.1
2
+ Name: shepherd_data
3
+ Version: 2024.4.1
4
+ Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
5
+ Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
6
+ Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
7
+ Project-URL: Documentation, https://github.com/orgua/shepherd-datalib/blob/main/README.md
8
+ Project-URL: Issues, https://pypi.org/project/shepherd-data/issues
9
+ Project-URL: Source, https://pypi.org/project/shepherd-data/
10
+ Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
11
+ Platform: unix
12
+ Platform: linux
13
+ Platform: osx
14
+ Platform: cygwin
15
+ Platform: win32
16
+ Platform: win64
17
+ Classifier: Development Status :: 5 - Production/Stable
18
+ Classifier: Intended Audience :: Developers
19
+ Classifier: Intended Audience :: Information Technology
20
+ Classifier: Intended Audience :: Science/Research
21
+ Classifier: Programming Language :: Python :: 3.8
22
+ Classifier: Programming Language :: Python :: 3.9
23
+ Classifier: Programming Language :: Python :: 3.10
24
+ Classifier: Programming Language :: Python :: 3.11
25
+ Classifier: Programming Language :: Python :: 3.12
26
+ Classifier: License :: OSI Approved :: MIT License
27
+ Classifier: Operating System :: OS Independent
28
+ Classifier: Natural Language :: English
29
+ Requires-Python: >=3.8
30
+ Description-Content-Type: text/markdown
31
+ Requires-Dist: click
32
+ Requires-Dist: h5py
33
+ Requires-Dist: matplotlib
34
+ Requires-Dist: numpy
35
+ Requires-Dist: pandas >=2.0.0
36
+ Requires-Dist: pyYAML
37
+ Requires-Dist: scipy
38
+ Requires-Dist: shepherd-core[inventory] >=2024.04.1
39
+ Requires-Dist: tqdm
40
+ Provides-Extra: dev
41
+ Requires-Dist: shepherd-core[dev] ; extra == 'dev'
42
+ Requires-Dist: pandas-stubs ; extra == 'dev'
43
+ Provides-Extra: elf
44
+ Requires-Dist: shepherd-core[elf] ; extra == 'elf'
45
+ Provides-Extra: test
46
+ Requires-Dist: shepherd-core[test] ; extra == 'test'
47
+ Requires-Dist: pytest ; extra == 'test'
48
+ Requires-Dist: pytest-click ; extra == 'test'
49
+ Requires-Dist: coverage ; extra == 'test'
50
+
51
+ # Shepherd-Data-Tool
52
+
53
+ [![PyPiVersion](https://img.shields.io/pypi/v/shepherd_data.svg)](https://pypi.org/project/shepherd_data)
54
+ [![image](https://img.shields.io/pypi/pyversions/shepherd_data.svg)](https://pypi.python.org/pypi/shepherd-data)
55
+ [![Pytest](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml/badge.svg)](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
56
+ [![CodeStyle](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
57
+
58
+ **Main Documentation**: <https://orgua.github.io/shepherd>
59
+
60
+ **Source Code**: <https://github.com/orgua/shepherd-datalib>
61
+
62
+ **Main Project**: <https://github.com/orgua/shepherd>
63
+
64
+ ---
65
+
66
+ `shepherd-data` eases the handling of hdf5-recordings used by the [shepherd](https://github.com/orgua/shepherd)-testbed. Users can read, validate and create files and also extract, down-sample and plot information.
67
+
68
+ ## Installation
69
+
70
+ ### PIP - Online
71
+
72
+ ```shell
73
+ pip3 install shepherd-data -U
74
+ ```
75
+
76
+ For bleeding-edge-features or dev-work it is possible to install directly from GitHub-Sources (here `dev`-branch):
77
+
78
+ ```Shell
79
+ pip install git+https://github.com/orgua/shepherd-datalib.git@dev#subdirectory=shepherd_data -U
80
+ ```
81
+
82
+ ## More
83
+
84
+ Please consult the [official documentation](https://orgua.github.io/shepherd) for more, it covers:
85
+
86
+ - general context
87
+ - command-line interface
88
+ - programming interface
@@ -0,0 +1,12 @@
1
+ shepherd_data/__init__.py,sha256=EZ_RzvcBkL2fMTCCWtKcbzMcrmmwgDb7SD-GZhvqUnA,243
2
+ shepherd_data/cli.py,sha256=2GDpTgd9cHJQS-GZJSvn2bhyeh6oG_srtITfc3FGAmg,15017
3
+ shepherd_data/debug_resampler.py,sha256=0VNGuUqOeKii6fvkRJUpv-27uv9p1VFsvShU3tvxwYQ,961
4
+ shepherd_data/ivonne.py,sha256=ArojlpWWYwaetkDtBhi-RdFidt79GHoMLZLRR6KljRo,11600
5
+ shepherd_data/mppt.py,sha256=588KSrLuJfNRKKnnL6ewePLi3zrwaO_PAZypikACrks,3925
6
+ shepherd_data/reader.py,sha256=JKTVemjH_6678MCNf2cDzRIsCyfD2B6CTUzBiG110QY,18531
7
+ shepherd_data-2024.4.1.dist-info/METADATA,sha256=ZuZcImaE9O7tvzyibfM4wVcToX_p7nImzS4lq8DEUoA,3388
8
+ shepherd_data-2024.4.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
9
+ shepherd_data-2024.4.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
10
+ shepherd_data-2024.4.1.dist-info/top_level.txt,sha256=7-SCTY-TG1mLY72OVKCaqte1hy-X8woxknIUAD3OIxs,14
11
+ shepherd_data-2024.4.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
12
+ shepherd_data-2024.4.1.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: bdist_wheel (0.41.3)
2
+ Generator: bdist_wheel (0.43.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,274 +0,0 @@
1
- Metadata-Version: 2.1
2
- Name: shepherd-data
3
- Version: 2023.11.1
4
- Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
5
- Home-page: https://pypi.org/project/shepherd-data/
6
- Author: Ingmar Splitt, Kai Geissdoerfer
7
- Author-email: ingmar.splitt@tu-dresden.de
8
- Maintainer-email: ingmar.splitt@tu-dresden.de
9
- License: MIT
10
- Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
11
- Project-URL: Source, https://github.com/orgua/shepherd-datalib
12
- Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
13
- Platform: unix
14
- Platform: linux
15
- Platform: osx
16
- Platform: cygwin
17
- Platform: win32
18
- Platform: win64
19
- Classifier: Development Status :: 5 - Production/Stable
20
- Classifier: Intended Audience :: Developers
21
- Classifier: Intended Audience :: Information Technology
22
- Classifier: Intended Audience :: Science/Research
23
- Classifier: Programming Language :: Python :: 3.8
24
- Classifier: Programming Language :: Python :: 3.9
25
- Classifier: Programming Language :: Python :: 3.10
26
- Classifier: Programming Language :: Python :: 3.11
27
- Classifier: Programming Language :: Python :: 3.12
28
- Classifier: License :: OSI Approved :: MIT License
29
- Classifier: Operating System :: OS Independent
30
- Classifier: Natural Language :: English
31
- Requires-Python: >=3.8
32
- Description-Content-Type: text/markdown
33
- Requires-Dist: h5py
34
- Requires-Dist: numpy
35
- Requires-Dist: pyYAML
36
- Requires-Dist: shepherd-core[inventory] >=2023.11.1
37
- Requires-Dist: click
38
- Requires-Dist: matplotlib
39
- Requires-Dist: pandas
40
- Requires-Dist: scipy
41
- Requires-Dist: tqdm
42
- Provides-Extra: dev
43
- Requires-Dist: shepherd-core[dev] ; extra == 'dev'
44
- Requires-Dist: pandas-stubs ; extra == 'dev'
45
- Provides-Extra: elf
46
- Requires-Dist: shepherd-core[elf] ; extra == 'elf'
47
- Provides-Extra: test
48
- Requires-Dist: shepherd-core[test] ; extra == 'test'
49
- Requires-Dist: pytest-click ; extra == 'test'
50
-
51
- # Data Module
52
-
53
- [![PyPiVersion](https://img.shields.io/pypi/v/shepherd_data.svg)](https://pypi.org/project/shepherd_data)
54
- [![image](https://img.shields.io/pypi/pyversions/shepherd_data.svg)](https://pypi.python.org/pypi/shepherd-data)
55
- [![Pytest](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml/badge.svg)](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
56
- [![CodeStyle](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
57
-
58
- **Documentation**: <https://orgua.github.io/shepherd/external/shepherd_data.html>
59
-
60
- **Source Code**: <https://github.com/orgua/shepherd-datalib>
61
-
62
- **Main Project**: <https://github.com/orgua/shepherd>
63
-
64
- ---
65
-
66
- This Python Module eases the handling of hdf5-recordings used by the [shepherd](https://github.com/orgua/shepherd)-testbed. Users can read, validate and create files and also extract, down-sample and plot information.
67
-
68
- ## Installation
69
-
70
- ### PIP - Online
71
-
72
- ```shell
73
- pip3 install shepherd-data -U
74
- ```
75
-
76
- For bleeding-edge-features or dev-work it is possible to install directly from GitHub-Sources (here `dev`-branch):
77
-
78
- ```Shell
79
- pip install git+https://github.com/orgua/shepherd-datalib.git@dev#subdirectory=shepherd_data -U
80
- ```
81
-
82
- ## Programming Interface
83
-
84
- ### Basic Usage (recommendation)
85
-
86
- ```python
87
- import shepherd_data as sd
88
-
89
- with sd.Reader("./hrv_sawtooth_1h.h5") as db:
90
- print(f"Mode: {db.get_mode()}")
91
- print(f"Window: {db.get_window_samples()}")
92
- print(f"Config: {db.get_config()}")
93
- ```
94
-
95
- ### Available Functionality
96
-
97
- - `Reader()`
98
- - file can be checked for plausibility and validity (`is_valid()`)
99
- - internal structure of h5file (`get_metadata()` or `save_metadata()` ... to yaml) with lots of additional data
100
- - access data and various converters, calculators
101
- - `read_buffers()` -> generator that provides one buffer per call, can be configured on first call
102
- - `get_calibration_data()`
103
- - `get_windows_samples()`
104
- - `get_mode()`
105
- - `get_config()`
106
- - direct access to root h5-structure via `reader['element']`
107
- - converters for raw / physical units: `si_to_raw()` & `raw_to_si()`
108
- - `energy()` sums up recorded power over time
109
- - `downsample()` (if needed) visualize recording (`plot_to_file()`)
110
- - `Writer()`
111
- - inherits all functionality from Reader
112
- - `append_iv_data_raw()`
113
- - `append_iv_data_si()`
114
- - `set_config()`
115
- - `set_windows_samples()`
116
- - IVonne Reader
117
- - `convert_2_ivcurves()` converts ivonne-recording into a shepherd ivcurve
118
- - `upsample_2_isc_voc()` TODO: for now a upsampled but unusable version of samples of short-circuit-current and open-circuit-voltage
119
- - `convert_2_ivsamples()` already applies a simple harvesting-algo and creates ivsamples
120
- - `./examples/`
121
- - `example_convert_ivonne.py` converts IVonne recording (`jogging_10m.iv`) to shepherd ivcurves, NOTE: slow implementation
122
- - `example_extract_logs.py` is analyzing all files in directory, saves logging-data and calculates cpu-load and data-rate
123
- - `example_generate_sawtooth.py` is using Writer to generate a 60s ramp with 1h repetition and uses Reader to dump metadata of that file
124
- - `example_plot_traces.py` demos some mpl-plots with various zoom levels
125
- - `example_repair_recordings.py` makes old recordings from shepherd 1.x fit for v2
126
- - `jogging_10m.iv`
127
- - 50 Hz measurement with Short-Circuit-Current and two other parameters
128
- - recorded with "IVonne"
129
-
130
- ### Functionality Update (WIP)
131
-
132
- - Core.`Reader`
133
- - `__repr__()`
134
- - `read_buffers`
135
- - `get_calibration_data`
136
- - `get_window_samples`
137
- - `get_mode`
138
- - `get_config`
139
- - `get_hostname`
140
- - `get_datatype`
141
- - `get_hrv_config`
142
- - `is_valid`
143
- - `energy()`
144
- - `check_timediffs()`
145
- - `get_metadata()`
146
- - `save_metadata()`
147
-
148
- - `Writer(Reader)` (core, data are the same)
149
- - `append_iv_data_raw`
150
- - `append_iv_data_si`
151
- - `store_config`
152
- - `store_hostname`
153
-
154
- - data.`Reader(CoreReader)`
155
- - `save_csv()`
156
- - `save_log()`
157
- - `downsample()`
158
- - `resample()`
159
- - `generate_plot_data()`
160
- - `assemble_plot()`
161
- - `plot_to_file()`
162
- - `multiplot_to_file()`
163
-
164
- ## CLI-Interface
165
-
166
- After installing the module the datalib offers some often needed functionality on the command line:
167
-
168
- **Validate Recordings**
169
-
170
- - takes a file or directory as an argument
171
-
172
- ```shell
173
- shepherd-data validate dir_or_file
174
-
175
- # examples:
176
- shepherd-data validate ./
177
- shepherd-data validate hrv_saw_1h.h5
178
- ```
179
-
180
- **Extract IV-Samples to csv**
181
-
182
- - takes a file or directory as an argument
183
- - can take down-sample-factor as an argument
184
-
185
- ```shell
186
- shepherd-data extract [-f ds-factor] [-s separator_symbol] dir_or_file
187
-
188
- # examples:
189
- shepherd-data extract ./
190
- shepherd-data extract -f 1000 -s ; hrv_saw_1h.h5
191
- ```
192
-
193
- **Extract meta-data and sys-logs**
194
-
195
- - takes a file or directory as an argument
196
-
197
- ```shell
198
- shepherd-data extract-meta dir_or_file
199
-
200
- # examples:
201
- shepherd-data extract-meta ./
202
- shepherd-data extract-meta hrv_saw_1h.h5
203
- ```
204
-
205
- **Plot IVSamples**
206
-
207
- - takes a file or directory as an argument
208
- - can take start- and end-time as an argument
209
- - can take image-width and -height as an argument
210
-
211
- ```shell
212
- shepherd-data plot [-s start_time] [-e end_time] [-w plot_width] [-h plot_height] [--multiplot] dir_or_file
213
-
214
- # examples:
215
- shepherd-data plot --multiplot ./
216
- shepherd-data plot -s10 -e20 hrv_saw_1h.h5
217
- ```
218
-
219
- **Downsample IVSamples (for later GUI-usage, TODO)**
220
-
221
- - generates a set of downsamplings (20 kHz to 0.1 Hz in x4 to x5 Steps)
222
- - takes a file or directory as an argument
223
- - can take down-sample-factor as an argument
224
-
225
- ```shell
226
- shepherd-data downsample [-f ds-factor] [-r sample-rate] dir_or_file
227
-
228
- # examples:
229
- shepherd-data downsample ./
230
- shepherd-data downsample -f 1000 hrv_saw_1h.h5
231
- shepherd-data downsample -r 100 hrv_saw_1h.h5
232
- ```
233
-
234
- ## Data-Layout and Design choices
235
-
236
- Details about the file-structure can be found in the [main-project](https://github.com/orgua/shepherd/blob/main/docs/user/data_format.rst).
237
-
238
- TODO:
239
- - update design of file
240
- - data dtype, mode, ...
241
-
242
- ### Modes and Datatypes
243
-
244
- - Mode `harvester` recorded a harvesting-source like solar with one of various algorithms
245
- - Datatype `ivsample` is directly usable by shepherd, input for virtual source / converter
246
- - Datatype `ivcurve` is directly usable by shepherd, input for a virtual harvester (output are ivsamples)
247
- - Datatype `isc_voc` is specially for solar-cells and needs to be (at least) transformed into ivcurves later
248
- - Mode `emulator` replayed a harvester-recording through a virtual converter and supplied a target while recording the power-consumption
249
- - Datatype `ivsample` is the only output of this mode
250
-
251
- ### Compression & Beaglebone
252
-
253
- - supported are uncompressed, lzf and gzip with level 1 (order of recommendation)
254
- - lzf seems better-suited due to lower load, or if space isn't a constraint: uncompressed (None as argument)
255
- - note: lzf seems to cause trouble with some third party hdf5-tools
256
- - compression is a heavy load for the beaglebone, but it got more performant with recent python-versions
257
- - size-experiment A: 24 h of ramping / sawtooth (data is repetitive with 1 minute ramp)
258
- - gzip-1: 49'646 MiB -> 588 KiB/s
259
- - lzf: 106'445 MiB -> 1262 KiB/s
260
- - uncompressed: 131'928 MiB -> 1564 KiB/s
261
- - cpu-load-experiments (input is 24h sawtooth, python 3.10 with most recent libs as of 2022-04)
262
- - warning: gpio-traffic and other logging-data can cause lots of load
263
-
264
- ```
265
- emu_120s_gz1_to_gz1.h5 -> emulator, cpu_util [%] = 65.59, data-rate = 352.0 KiB/s
266
- emu_120s_gz1_to_lzf.h5 -> emulator, cpu_util [%] = 57.37, data-rate = 686.0 KiB/s
267
- emu_120s_gz1_to_unc.h5 -> emulator, cpu_util [%] = 53.63, data-rate = 1564.0 KiB/s
268
- emu_120s_lzf_to_gz1.h5 -> emulator, cpu_util [%] = 63.18, data-rate = 352.0 KiB/s
269
- emu_120s_lzf_to_lzf.h5 -> emulator, cpu_util [%] = 58.60, data-rate = 686.0 KiB/s
270
- emu_120s_lzf_to_unc.h5 -> emulator, cpu_util [%] = 55.75, data-rate = 1564.0 KiB/s
271
- emu_120s_unc_to_gz1.h5 -> emulator, cpu_util [%] = 63.84, data-rate = 351.0 KiB/s
272
- emu_120s_unc_to_lzf.h5 -> emulator, cpu_util [%] = 57.28, data-rate = 686.0 KiB/s
273
- emu_120s_unc_to_unc.h5 -> emulator, cpu_util [%] = 51.69, data-rate = 1564.0 KiB/s
274
- ```
@@ -1,22 +0,0 @@
1
- shepherd_data/__init__.py,sha256=Ys41EYMRuO_55OepmLO6icu4xKGX9I9QJowHWa9jzuo,242
2
- shepherd_data/cli.py,sha256=u9ImWh-VLVlk1-TJaQs7govRHoiiuF5xIeOCLBzr8k8,15341
3
- shepherd_data/debug_resampler.py,sha256=Y4uxSXqWKoyyJ1H6TY1vZKe0pWMK_mjQY0xBr7dC_f4,983
4
- shepherd_data/ivonne.py,sha256=ZGCjctOFYJyMH9AsiF8oaFd_QuvVDbGdPuj3qoICHYg,12017
5
- shepherd_data/mppt.py,sha256=YWMw3lhReHbjmK1LVgrIo2S_SoBmIXI4a_sqUpHMuGE,3640
6
- shepherd_data/reader.py,sha256=toX3NLqmifUYfrSp81XrmDJkWm8jdhwUIWy8v530tRI,18440
7
- tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
8
- tests/conftest.py,sha256=BCtm2TbfVf85_AsGOUZLXd6TquicWRMgBq2-q5UA2UM,985
9
- tests/test_cli.py,sha256=PPbRwgOQbWjNcgR7mIXTnGdboCQ4QDC4Kh74kSaEFhc,181
10
- tests/test_cli_downsample.py,sha256=dsI04rJrFdNl50uW5DplmA4Uh1S2Y2hCYq-HkM47pG8,1803
11
- tests/test_cli_extract.py,sha256=XhXNNyiRRuXkDqkgWLaFTZkge6tv2a8jJyAd2zt8QlE,2532
12
- tests/test_cli_plot.py,sha256=IaN1Z37XYgfkxSzJkfej8Vp9FqSUeYCSog3ratnDePQ,3298
13
- tests/test_cli_validate.py,sha256=DOfaRq6r05eu8x4SnZOPrPfbbMNu0PTus7lSV8Gkzwo,405
14
- tests/test_examples.py,sha256=e_uc5h_I4wwJnIWuh-PjpSSTICkYemI831iqhxIy_R0,576
15
- tests/test_ivonne.py,sha256=cIMtpx7tjRFxL9hpr5OPPgroGLzRZxGC4x9Y7vWgze8,1367
16
- tests/test_reader.py,sha256=Xt__Khgldv8klZJ3OQcMZevC6_3gPH2_TpOorq2ytNI,90
17
- shepherd_data-2023.11.1.dist-info/METADATA,sha256=a0Jd5FvKexcOLiRcc46seLGm6vut-kcAHiwyWC2N-3U,9942
18
- shepherd_data-2023.11.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
19
- shepherd_data-2023.11.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
20
- shepherd_data-2023.11.1.dist-info/top_level.txt,sha256=Bk61YO7iYS43TSWtwdSKb2moGBvJqJIYfGaZsQsv09M,20
21
- shepherd_data-2023.11.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
22
- shepherd_data-2023.11.1.dist-info/RECORD,,
tests/__init__.py DELETED
File without changes