shepherd-data 2023.11.1__py3-none-any.whl → 2024.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shepherd_data/__init__.py +2 -1
- shepherd_data/cli.py +20 -43
- shepherd_data/debug_resampler.py +1 -3
- shepherd_data/ivonne.py +20 -41
- shepherd_data/mppt.py +24 -22
- shepherd_data/reader.py +45 -67
- shepherd_data-2024.4.1.dist-info/METADATA +88 -0
- shepherd_data-2024.4.1.dist-info/RECORD +12 -0
- {shepherd_data-2023.11.1.dist-info → shepherd_data-2024.4.1.dist-info}/WHEEL +1 -1
- {shepherd_data-2023.11.1.dist-info → shepherd_data-2024.4.1.dist-info}/top_level.txt +0 -1
- shepherd_data-2023.11.1.dist-info/METADATA +0 -274
- shepherd_data-2023.11.1.dist-info/RECORD +0 -22
- tests/__init__.py +0 -0
- tests/conftest.py +0 -33
- tests/test_cli.py +0 -8
- tests/test_cli_downsample.py +0 -51
- tests/test_cli_extract.py +0 -84
- tests/test_cli_plot.py +0 -128
- tests/test_cli_validate.py +0 -15
- tests/test_examples.py +0 -26
- tests/test_ivonne.py +0 -42
- tests/test_reader.py +0 -3
- {shepherd_data-2023.11.1.dist-info → shepherd_data-2024.4.1.dist-info}/entry_points.txt +0 -0
- {shepherd_data-2023.11.1.dist-info → shepherd_data-2024.4.1.dist-info}/zip-safe +0 -0
shepherd_data/reader.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
"""Reader-Baseclass
|
|
2
|
-
|
|
1
|
+
"""Reader-Baseclass for opening shepherds hdf5-files."""
|
|
2
|
+
|
|
3
3
|
import math
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from pathlib import Path
|
|
@@ -26,6 +26,7 @@ class Reader(CoreReader):
|
|
|
26
26
|
----
|
|
27
27
|
file_path: Path of hdf5 file containing shepherd data with iv-samples, iv-curves or isc&voc
|
|
28
28
|
verbose: more info during usage, 'None' skips the setter
|
|
29
|
+
|
|
29
30
|
"""
|
|
30
31
|
|
|
31
32
|
def __init__(
|
|
@@ -37,7 +38,7 @@ class Reader(CoreReader):
|
|
|
37
38
|
super().__init__(file_path, verbose=verbose)
|
|
38
39
|
|
|
39
40
|
def save_csv(self, h5_group: h5py.Group, separator: str = ";") -> int:
|
|
40
|
-
"""Extract numerical data
|
|
41
|
+
"""Extract numerical data from group and store it into csv.
|
|
41
42
|
|
|
42
43
|
:param h5_group: can be external and should probably be downsampled
|
|
43
44
|
:param separator: used between columns
|
|
@@ -52,21 +53,14 @@ class Reader(CoreReader):
|
|
|
52
53
|
if csv_path.exists():
|
|
53
54
|
self._logger.info("File already exists, will skip '%s'", csv_path.name)
|
|
54
55
|
return 0
|
|
55
|
-
datasets = [
|
|
56
|
-
key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group
|
|
57
|
-
]
|
|
56
|
+
datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
|
|
58
57
|
datasets.remove("time")
|
|
59
58
|
datasets = ["time", *datasets]
|
|
60
59
|
separator = separator.strip().ljust(2)
|
|
61
|
-
header = [
|
|
62
|
-
h5_group[key].attrs["description"].replace(", ", separator)
|
|
63
|
-
for key in datasets
|
|
64
|
-
]
|
|
60
|
+
header = [h5_group[key].attrs["description"].replace(", ", separator) for key in datasets]
|
|
65
61
|
header = separator.join(header)
|
|
66
62
|
with csv_path.open("w", encoding="utf-8-sig") as csv_file:
|
|
67
|
-
self._logger.info(
|
|
68
|
-
"CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name
|
|
69
|
-
)
|
|
63
|
+
self._logger.info("CSV-Generator will save '%s' to '%s'", h5_group.name, csv_path.name)
|
|
70
64
|
csv_file.write(header + "\n")
|
|
71
65
|
for idx, time_ns in enumerate(h5_group["time"][:]):
|
|
72
66
|
timestamp = datetime.fromtimestamp(time_ns / 1e9, tz=local_tz())
|
|
@@ -80,7 +74,7 @@ class Reader(CoreReader):
|
|
|
80
74
|
return h5_group["time"][:].shape[0]
|
|
81
75
|
|
|
82
76
|
def save_log(self, h5_group: h5py.Group, *, add_timestamp: bool = True) -> int:
|
|
83
|
-
"""Save dataset
|
|
77
|
+
"""Save dataset from group as log, optimal for logged 'dmesg' and console-output.
|
|
84
78
|
|
|
85
79
|
:param h5_group: can be external
|
|
86
80
|
:param add_timestamp: can be external
|
|
@@ -95,14 +89,10 @@ class Reader(CoreReader):
|
|
|
95
89
|
if log_path.exists():
|
|
96
90
|
self._logger.info("File already exists, will skip '%s'", log_path.name)
|
|
97
91
|
return 0
|
|
98
|
-
datasets = [
|
|
99
|
-
key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group
|
|
100
|
-
]
|
|
92
|
+
datasets = [key if isinstance(h5_group[key], h5py.Dataset) else [] for key in h5_group]
|
|
101
93
|
datasets.remove("time")
|
|
102
94
|
with log_path.open("w", encoding="utf-8-sig") as log_file:
|
|
103
|
-
self._logger.info(
|
|
104
|
-
"Log-Generator will save '%s' to '%s'", h5_group.name, log_path.name
|
|
105
|
-
)
|
|
95
|
+
self._logger.info("Log-Generator will save '%s' to '%s'", h5_group.name, log_path.name)
|
|
106
96
|
for idx, time_ns in enumerate(h5_group["time"][:]):
|
|
107
97
|
if add_timestamp:
|
|
108
98
|
timestamp = datetime.fromtimestamp(time_ns / 1e9, local_tz())
|
|
@@ -125,6 +115,7 @@ class Reader(CoreReader):
|
|
|
125
115
|
*,
|
|
126
116
|
show: bool = True,
|
|
127
117
|
) -> int:
|
|
118
|
+
"""Print warning messages from log in data-group."""
|
|
128
119
|
_count = self.count_errors_in_log(group_name, min_level)
|
|
129
120
|
if _count < 1:
|
|
130
121
|
return 0
|
|
@@ -156,7 +147,7 @@ class Reader(CoreReader):
|
|
|
156
147
|
|
|
157
148
|
def downsample(
|
|
158
149
|
self,
|
|
159
|
-
data_src: h5py.Dataset,
|
|
150
|
+
data_src: Union[h5py.Dataset, np.ndarray],
|
|
160
151
|
data_dst: Union[None, h5py.Dataset, np.ndarray],
|
|
161
152
|
start_n: int = 0,
|
|
162
153
|
end_n: Optional[int] = None,
|
|
@@ -164,15 +155,17 @@ class Reader(CoreReader):
|
|
|
164
155
|
*,
|
|
165
156
|
is_time: bool = False,
|
|
166
157
|
) -> Union[h5py.Dataset, np.ndarray]:
|
|
167
|
-
"""
|
|
158
|
+
"""Sample down iv-data.
|
|
159
|
+
|
|
160
|
+
Warning: only valid for IV-Stream, not IV-Curves
|
|
168
161
|
|
|
169
162
|
:param data_src: a h5-dataset to digest, can be external
|
|
170
163
|
:param data_dst: can be a dataset, numpy-array or None (will be created internally then)
|
|
171
164
|
:param start_n: start-sample
|
|
172
165
|
:param end_n: ending-sample (not included)
|
|
173
|
-
:param ds_factor:
|
|
174
|
-
:param is_time: time is not really
|
|
175
|
-
:return:
|
|
166
|
+
:param ds_factor: sampling-factor
|
|
167
|
+
:param is_time: time is not really down-sampled, but decimated
|
|
168
|
+
:return: resampled h5-dataset or numpy-array
|
|
176
169
|
"""
|
|
177
170
|
from scipy import signal # here due to massive delay
|
|
178
171
|
|
|
@@ -219,27 +212,21 @@ class Reader(CoreReader):
|
|
|
219
212
|
leave=False,
|
|
220
213
|
disable=iterations < 8,
|
|
221
214
|
):
|
|
222
|
-
slice_ds = data_src[
|
|
223
|
-
start_n + _iter * iblock_len : start_n + (_iter + 1) * iblock_len
|
|
224
|
-
]
|
|
215
|
+
slice_ds = data_src[start_n + _iter * iblock_len : start_n + (_iter + 1) * iblock_len]
|
|
225
216
|
if not is_time and ds_factor > 1:
|
|
226
217
|
slice_ds, f_state = signal.sosfilt(filter_, slice_ds, zi=f_state)
|
|
227
218
|
slice_ds = slice_ds[::ds_factor]
|
|
228
219
|
slice_len = min(dest_len - _iter * oblock_len, oblock_len)
|
|
229
|
-
data_dst[_iter * oblock_len : (_iter + 1) * oblock_len] = slice_ds[
|
|
230
|
-
:slice_len
|
|
231
|
-
]
|
|
220
|
+
data_dst[_iter * oblock_len : (_iter + 1) * oblock_len] = slice_ds[:slice_len]
|
|
232
221
|
if isinstance(data_dst, np.ndarray):
|
|
233
|
-
data_dst.resize(
|
|
234
|
-
(oblock_len * (iterations - 1) + slice_len,), refcheck=False
|
|
235
|
-
)
|
|
222
|
+
data_dst.resize((oblock_len * (iterations - 1) + slice_len,), refcheck=False)
|
|
236
223
|
else:
|
|
237
224
|
data_dst.resize((oblock_len * (iterations - 1) + slice_len,))
|
|
238
225
|
return data_dst
|
|
239
226
|
|
|
240
227
|
def resample(
|
|
241
228
|
self,
|
|
242
|
-
data_src: h5py.Dataset,
|
|
229
|
+
data_src: Union[h5py.Dataset, np.ndarray],
|
|
243
230
|
data_dst: Union[None, h5py.Dataset, np.ndarray],
|
|
244
231
|
start_n: int = 0,
|
|
245
232
|
end_n: Optional[int] = None,
|
|
@@ -247,17 +234,17 @@ class Reader(CoreReader):
|
|
|
247
234
|
*,
|
|
248
235
|
is_time: bool = False,
|
|
249
236
|
) -> Union[h5py.Dataset, np.ndarray]:
|
|
250
|
-
"""
|
|
251
|
-
|
|
252
|
-
:param
|
|
253
|
-
:param
|
|
254
|
-
:param
|
|
255
|
-
:param
|
|
256
|
-
:
|
|
237
|
+
"""Up- or down-sample the original trace-data.
|
|
238
|
+
|
|
239
|
+
:param data_src: original iv-data
|
|
240
|
+
:param data_dst: resampled iv-traces
|
|
241
|
+
:param start_n: start index of the source
|
|
242
|
+
:param end_n: end index of the source
|
|
243
|
+
:param samplerate_dst: desired sampling rate
|
|
244
|
+
:param is_time: time-array is handled differently than IV-Samples
|
|
245
|
+
:return: resampled iv-data
|
|
257
246
|
"""
|
|
258
|
-
self._logger.error(
|
|
259
|
-
"Resampling is still under construction - do not use for now!"
|
|
260
|
-
)
|
|
247
|
+
self._logger.error("Resampling is still under construction - do not use for now!")
|
|
261
248
|
if self.get_datatype() == "ivcurve":
|
|
262
249
|
self._logger.warning("Resampling-Function was not written for IVCurves")
|
|
263
250
|
|
|
@@ -331,7 +318,6 @@ class Reader(CoreReader):
|
|
|
331
318
|
slice_out_now = slice_out_nxt
|
|
332
319
|
resampler.reset()
|
|
333
320
|
"""
|
|
334
|
-
pass
|
|
335
321
|
|
|
336
322
|
if isinstance(data_dst, np.ndarray):
|
|
337
323
|
data_dst.resize((slice_out_now,), refcheck=False)
|
|
@@ -347,7 +333,7 @@ class Reader(CoreReader):
|
|
|
347
333
|
*,
|
|
348
334
|
relative_timestamp: bool = True,
|
|
349
335
|
) -> Dict:
|
|
350
|
-
"""
|
|
336
|
+
"""Provide down-sampled iv-data that can be fed into plot_to_file().
|
|
351
337
|
|
|
352
338
|
:param start_s: time in seconds, relative to start of recording
|
|
353
339
|
:param end_s: time in seconds, relative to start of recording
|
|
@@ -377,14 +363,10 @@ class Reader(CoreReader):
|
|
|
377
363
|
)
|
|
378
364
|
).astype(float),
|
|
379
365
|
"voltage": self._cal.voltage.raw_to_si(
|
|
380
|
-
self.downsample(
|
|
381
|
-
self.ds_voltage, None, start_sample, end_sample, ds_factor
|
|
382
|
-
)
|
|
366
|
+
self.downsample(self.ds_voltage, None, start_sample, end_sample, ds_factor)
|
|
383
367
|
),
|
|
384
368
|
"current": self._cal.current.raw_to_si(
|
|
385
|
-
self.downsample(
|
|
386
|
-
self.ds_current, None, start_sample, end_sample, ds_factor
|
|
387
|
-
)
|
|
369
|
+
self.downsample(self.ds_current, None, start_sample, end_sample, ds_factor)
|
|
388
370
|
),
|
|
389
371
|
"start_s": start_s,
|
|
390
372
|
"end_s": end_s,
|
|
@@ -394,17 +376,16 @@ class Reader(CoreReader):
|
|
|
394
376
|
return data
|
|
395
377
|
|
|
396
378
|
@staticmethod
|
|
397
|
-
def assemble_plot(
|
|
398
|
-
|
|
399
|
-
) -> plt.Figure:
|
|
400
|
-
"""TODO: add power (if wanted)
|
|
379
|
+
def assemble_plot(data: Union[dict, list], width: int = 20, height: int = 10) -> plt.Figure:
|
|
380
|
+
"""Create the actual figure.
|
|
401
381
|
|
|
402
382
|
:param data: plottable / down-sampled iv-data with some meta-data
|
|
403
383
|
-> created with generate_plot_data()
|
|
404
384
|
:param width: plot-width
|
|
405
385
|
:param height: plot-height
|
|
406
|
-
:return:
|
|
386
|
+
:return: figure
|
|
407
387
|
"""
|
|
388
|
+
# TODO: add power (if wanted)
|
|
408
389
|
if isinstance(data, dict):
|
|
409
390
|
data = [data]
|
|
410
391
|
fig, axes = plt.subplots(2, 1, sharex="all")
|
|
@@ -429,8 +410,9 @@ class Reader(CoreReader):
|
|
|
429
410
|
width: int = 20,
|
|
430
411
|
height: int = 10,
|
|
431
412
|
) -> None:
|
|
432
|
-
"""
|
|
433
|
-
|
|
413
|
+
"""Create (down-sampled) IV-Plots.
|
|
414
|
+
|
|
415
|
+
Omitting start- and end-time will use the whole trace (full duration).
|
|
434
416
|
|
|
435
417
|
:param start_s: time in seconds, relative to start of recording, optional
|
|
436
418
|
:param end_s: time in seconds, relative to start of recording, optional
|
|
@@ -444,9 +426,7 @@ class Reader(CoreReader):
|
|
|
444
426
|
|
|
445
427
|
start_str = f"{data[0]['start_s']:.3f}".replace(".", "s")
|
|
446
428
|
end_str = f"{data[0]['end_s']:.3f}".replace(".", "s")
|
|
447
|
-
plot_path = self.file_path.resolve().with_suffix(
|
|
448
|
-
f".plot_{start_str}_to_{end_str}.png"
|
|
449
|
-
)
|
|
429
|
+
plot_path = self.file_path.resolve().with_suffix(f".plot_{start_str}_to_{end_str}.png")
|
|
450
430
|
if plot_path.exists():
|
|
451
431
|
self._logger.warning("Plot exists, will skip & not overwrite!")
|
|
452
432
|
return
|
|
@@ -460,7 +440,7 @@ class Reader(CoreReader):
|
|
|
460
440
|
def multiplot_to_file(
|
|
461
441
|
data: list, plot_path: Path, width: int = 20, height: int = 10
|
|
462
442
|
) -> Optional[Path]:
|
|
463
|
-
"""
|
|
443
|
+
"""Create (down-sampled) IV-Multi-Plots (of more than one trace).
|
|
464
444
|
|
|
465
445
|
:param data: plottable / down-sampled iv-data with some meta-data
|
|
466
446
|
-> created with generate_plot_data()
|
|
@@ -471,9 +451,7 @@ class Reader(CoreReader):
|
|
|
471
451
|
start_str = f"{data[0]['start_s']:.3f}".replace(".", "s")
|
|
472
452
|
end_str = f"{data[0]['end_s']:.3f}".replace(".", "s")
|
|
473
453
|
plot_path = (
|
|
474
|
-
Path(plot_path)
|
|
475
|
-
.resolve()
|
|
476
|
-
.with_suffix(f".multiplot_{start_str}_to_{end_str}.png")
|
|
454
|
+
Path(plot_path).resolve().with_suffix(f".multiplot_{start_str}_to_{end_str}.png")
|
|
477
455
|
)
|
|
478
456
|
if plot_path.exists():
|
|
479
457
|
logger.warning("Plot exists, will skip & not overwrite!")
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: shepherd_data
|
|
3
|
+
Version: 2024.4.1
|
|
4
|
+
Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
|
|
5
|
+
Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
6
|
+
Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
7
|
+
Project-URL: Documentation, https://github.com/orgua/shepherd-datalib/blob/main/README.md
|
|
8
|
+
Project-URL: Issues, https://pypi.org/project/shepherd-data/issues
|
|
9
|
+
Project-URL: Source, https://pypi.org/project/shepherd-data/
|
|
10
|
+
Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
|
|
11
|
+
Platform: unix
|
|
12
|
+
Platform: linux
|
|
13
|
+
Platform: osx
|
|
14
|
+
Platform: cygwin
|
|
15
|
+
Platform: win32
|
|
16
|
+
Platform: win64
|
|
17
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
18
|
+
Classifier: Intended Audience :: Developers
|
|
19
|
+
Classifier: Intended Audience :: Information Technology
|
|
20
|
+
Classifier: Intended Audience :: Science/Research
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
26
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
27
|
+
Classifier: Operating System :: OS Independent
|
|
28
|
+
Classifier: Natural Language :: English
|
|
29
|
+
Requires-Python: >=3.8
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
Requires-Dist: click
|
|
32
|
+
Requires-Dist: h5py
|
|
33
|
+
Requires-Dist: matplotlib
|
|
34
|
+
Requires-Dist: numpy
|
|
35
|
+
Requires-Dist: pandas >=2.0.0
|
|
36
|
+
Requires-Dist: pyYAML
|
|
37
|
+
Requires-Dist: scipy
|
|
38
|
+
Requires-Dist: shepherd-core[inventory] >=2024.04.1
|
|
39
|
+
Requires-Dist: tqdm
|
|
40
|
+
Provides-Extra: dev
|
|
41
|
+
Requires-Dist: shepherd-core[dev] ; extra == 'dev'
|
|
42
|
+
Requires-Dist: pandas-stubs ; extra == 'dev'
|
|
43
|
+
Provides-Extra: elf
|
|
44
|
+
Requires-Dist: shepherd-core[elf] ; extra == 'elf'
|
|
45
|
+
Provides-Extra: test
|
|
46
|
+
Requires-Dist: shepherd-core[test] ; extra == 'test'
|
|
47
|
+
Requires-Dist: pytest ; extra == 'test'
|
|
48
|
+
Requires-Dist: pytest-click ; extra == 'test'
|
|
49
|
+
Requires-Dist: coverage ; extra == 'test'
|
|
50
|
+
|
|
51
|
+
# Shepherd-Data-Tool
|
|
52
|
+
|
|
53
|
+
[](https://pypi.org/project/shepherd_data)
|
|
54
|
+
[](https://pypi.python.org/pypi/shepherd-data)
|
|
55
|
+
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
56
|
+
[](https://github.com/astral-sh/ruff)
|
|
57
|
+
|
|
58
|
+
**Main Documentation**: <https://orgua.github.io/shepherd>
|
|
59
|
+
|
|
60
|
+
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
61
|
+
|
|
62
|
+
**Main Project**: <https://github.com/orgua/shepherd>
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
`shepherd-data` eases the handling of hdf5-recordings used by the [shepherd](https://github.com/orgua/shepherd)-testbed. Users can read, validate and create files and also extract, down-sample and plot information.
|
|
67
|
+
|
|
68
|
+
## Installation
|
|
69
|
+
|
|
70
|
+
### PIP - Online
|
|
71
|
+
|
|
72
|
+
```shell
|
|
73
|
+
pip3 install shepherd-data -U
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
For bleeding-edge-features or dev-work it is possible to install directly from GitHub-Sources (here `dev`-branch):
|
|
77
|
+
|
|
78
|
+
```Shell
|
|
79
|
+
pip install git+https://github.com/orgua/shepherd-datalib.git@dev#subdirectory=shepherd_data -U
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## More
|
|
83
|
+
|
|
84
|
+
Please consult the [official documentation](https://orgua.github.io/shepherd) for more, it covers:
|
|
85
|
+
|
|
86
|
+
- general context
|
|
87
|
+
- command-line interface
|
|
88
|
+
- programming interface
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
shepherd_data/__init__.py,sha256=EZ_RzvcBkL2fMTCCWtKcbzMcrmmwgDb7SD-GZhvqUnA,243
|
|
2
|
+
shepherd_data/cli.py,sha256=2GDpTgd9cHJQS-GZJSvn2bhyeh6oG_srtITfc3FGAmg,15017
|
|
3
|
+
shepherd_data/debug_resampler.py,sha256=0VNGuUqOeKii6fvkRJUpv-27uv9p1VFsvShU3tvxwYQ,961
|
|
4
|
+
shepherd_data/ivonne.py,sha256=ArojlpWWYwaetkDtBhi-RdFidt79GHoMLZLRR6KljRo,11600
|
|
5
|
+
shepherd_data/mppt.py,sha256=588KSrLuJfNRKKnnL6ewePLi3zrwaO_PAZypikACrks,3925
|
|
6
|
+
shepherd_data/reader.py,sha256=JKTVemjH_6678MCNf2cDzRIsCyfD2B6CTUzBiG110QY,18531
|
|
7
|
+
shepherd_data-2024.4.1.dist-info/METADATA,sha256=ZuZcImaE9O7tvzyibfM4wVcToX_p7nImzS4lq8DEUoA,3388
|
|
8
|
+
shepherd_data-2024.4.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
9
|
+
shepherd_data-2024.4.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
|
|
10
|
+
shepherd_data-2024.4.1.dist-info/top_level.txt,sha256=7-SCTY-TG1mLY72OVKCaqte1hy-X8woxknIUAD3OIxs,14
|
|
11
|
+
shepherd_data-2024.4.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
12
|
+
shepherd_data-2024.4.1.dist-info/RECORD,,
|
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: shepherd-data
|
|
3
|
-
Version: 2023.11.1
|
|
4
|
-
Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
|
|
5
|
-
Home-page: https://pypi.org/project/shepherd-data/
|
|
6
|
-
Author: Ingmar Splitt, Kai Geissdoerfer
|
|
7
|
-
Author-email: ingmar.splitt@tu-dresden.de
|
|
8
|
-
Maintainer-email: ingmar.splitt@tu-dresden.de
|
|
9
|
-
License: MIT
|
|
10
|
-
Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
|
|
11
|
-
Project-URL: Source, https://github.com/orgua/shepherd-datalib
|
|
12
|
-
Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
|
|
13
|
-
Platform: unix
|
|
14
|
-
Platform: linux
|
|
15
|
-
Platform: osx
|
|
16
|
-
Platform: cygwin
|
|
17
|
-
Platform: win32
|
|
18
|
-
Platform: win64
|
|
19
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
20
|
-
Classifier: Intended Audience :: Developers
|
|
21
|
-
Classifier: Intended Audience :: Information Technology
|
|
22
|
-
Classifier: Intended Audience :: Science/Research
|
|
23
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
24
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
25
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
26
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
27
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
28
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
29
|
-
Classifier: Operating System :: OS Independent
|
|
30
|
-
Classifier: Natural Language :: English
|
|
31
|
-
Requires-Python: >=3.8
|
|
32
|
-
Description-Content-Type: text/markdown
|
|
33
|
-
Requires-Dist: h5py
|
|
34
|
-
Requires-Dist: numpy
|
|
35
|
-
Requires-Dist: pyYAML
|
|
36
|
-
Requires-Dist: shepherd-core[inventory] >=2023.11.1
|
|
37
|
-
Requires-Dist: click
|
|
38
|
-
Requires-Dist: matplotlib
|
|
39
|
-
Requires-Dist: pandas
|
|
40
|
-
Requires-Dist: scipy
|
|
41
|
-
Requires-Dist: tqdm
|
|
42
|
-
Provides-Extra: dev
|
|
43
|
-
Requires-Dist: shepherd-core[dev] ; extra == 'dev'
|
|
44
|
-
Requires-Dist: pandas-stubs ; extra == 'dev'
|
|
45
|
-
Provides-Extra: elf
|
|
46
|
-
Requires-Dist: shepherd-core[elf] ; extra == 'elf'
|
|
47
|
-
Provides-Extra: test
|
|
48
|
-
Requires-Dist: shepherd-core[test] ; extra == 'test'
|
|
49
|
-
Requires-Dist: pytest-click ; extra == 'test'
|
|
50
|
-
|
|
51
|
-
# Data Module
|
|
52
|
-
|
|
53
|
-
[](https://pypi.org/project/shepherd_data)
|
|
54
|
-
[](https://pypi.python.org/pypi/shepherd-data)
|
|
55
|
-
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
56
|
-
[](https://github.com/psf/black)
|
|
57
|
-
|
|
58
|
-
**Documentation**: <https://orgua.github.io/shepherd/external/shepherd_data.html>
|
|
59
|
-
|
|
60
|
-
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
61
|
-
|
|
62
|
-
**Main Project**: <https://github.com/orgua/shepherd>
|
|
63
|
-
|
|
64
|
-
---
|
|
65
|
-
|
|
66
|
-
This Python Module eases the handling of hdf5-recordings used by the [shepherd](https://github.com/orgua/shepherd)-testbed. Users can read, validate and create files and also extract, down-sample and plot information.
|
|
67
|
-
|
|
68
|
-
## Installation
|
|
69
|
-
|
|
70
|
-
### PIP - Online
|
|
71
|
-
|
|
72
|
-
```shell
|
|
73
|
-
pip3 install shepherd-data -U
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
For bleeding-edge-features or dev-work it is possible to install directly from GitHub-Sources (here `dev`-branch):
|
|
77
|
-
|
|
78
|
-
```Shell
|
|
79
|
-
pip install git+https://github.com/orgua/shepherd-datalib.git@dev#subdirectory=shepherd_data -U
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
## Programming Interface
|
|
83
|
-
|
|
84
|
-
### Basic Usage (recommendation)
|
|
85
|
-
|
|
86
|
-
```python
|
|
87
|
-
import shepherd_data as sd
|
|
88
|
-
|
|
89
|
-
with sd.Reader("./hrv_sawtooth_1h.h5") as db:
|
|
90
|
-
print(f"Mode: {db.get_mode()}")
|
|
91
|
-
print(f"Window: {db.get_window_samples()}")
|
|
92
|
-
print(f"Config: {db.get_config()}")
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
### Available Functionality
|
|
96
|
-
|
|
97
|
-
- `Reader()`
|
|
98
|
-
- file can be checked for plausibility and validity (`is_valid()`)
|
|
99
|
-
- internal structure of h5file (`get_metadata()` or `save_metadata()` ... to yaml) with lots of additional data
|
|
100
|
-
- access data and various converters, calculators
|
|
101
|
-
- `read_buffers()` -> generator that provides one buffer per call, can be configured on first call
|
|
102
|
-
- `get_calibration_data()`
|
|
103
|
-
- `get_windows_samples()`
|
|
104
|
-
- `get_mode()`
|
|
105
|
-
- `get_config()`
|
|
106
|
-
- direct access to root h5-structure via `reader['element']`
|
|
107
|
-
- converters for raw / physical units: `si_to_raw()` & `raw_to_si()`
|
|
108
|
-
- `energy()` sums up recorded power over time
|
|
109
|
-
- `downsample()` (if needed) visualize recording (`plot_to_file()`)
|
|
110
|
-
- `Writer()`
|
|
111
|
-
- inherits all functionality from Reader
|
|
112
|
-
- `append_iv_data_raw()`
|
|
113
|
-
- `append_iv_data_si()`
|
|
114
|
-
- `set_config()`
|
|
115
|
-
- `set_windows_samples()`
|
|
116
|
-
- IVonne Reader
|
|
117
|
-
- `convert_2_ivcurves()` converts ivonne-recording into a shepherd ivcurve
|
|
118
|
-
- `upsample_2_isc_voc()` TODO: for now a upsampled but unusable version of samples of short-circuit-current and open-circuit-voltage
|
|
119
|
-
- `convert_2_ivsamples()` already applies a simple harvesting-algo and creates ivsamples
|
|
120
|
-
- `./examples/`
|
|
121
|
-
- `example_convert_ivonne.py` converts IVonne recording (`jogging_10m.iv`) to shepherd ivcurves, NOTE: slow implementation
|
|
122
|
-
- `example_extract_logs.py` is analyzing all files in directory, saves logging-data and calculates cpu-load and data-rate
|
|
123
|
-
- `example_generate_sawtooth.py` is using Writer to generate a 60s ramp with 1h repetition and uses Reader to dump metadata of that file
|
|
124
|
-
- `example_plot_traces.py` demos some mpl-plots with various zoom levels
|
|
125
|
-
- `example_repair_recordings.py` makes old recordings from shepherd 1.x fit for v2
|
|
126
|
-
- `jogging_10m.iv`
|
|
127
|
-
- 50 Hz measurement with Short-Circuit-Current and two other parameters
|
|
128
|
-
- recorded with "IVonne"
|
|
129
|
-
|
|
130
|
-
### Functionality Update (WIP)
|
|
131
|
-
|
|
132
|
-
- Core.`Reader`
|
|
133
|
-
- `__repr__()`
|
|
134
|
-
- `read_buffers`
|
|
135
|
-
- `get_calibration_data`
|
|
136
|
-
- `get_window_samples`
|
|
137
|
-
- `get_mode`
|
|
138
|
-
- `get_config`
|
|
139
|
-
- `get_hostname`
|
|
140
|
-
- `get_datatype`
|
|
141
|
-
- `get_hrv_config`
|
|
142
|
-
- `is_valid`
|
|
143
|
-
- `energy()`
|
|
144
|
-
- `check_timediffs()`
|
|
145
|
-
- `get_metadata()`
|
|
146
|
-
- `save_metadata()`
|
|
147
|
-
|
|
148
|
-
- `Writer(Reader)` (core, data are the same)
|
|
149
|
-
- `append_iv_data_raw`
|
|
150
|
-
- `append_iv_data_si`
|
|
151
|
-
- `store_config`
|
|
152
|
-
- `store_hostname`
|
|
153
|
-
|
|
154
|
-
- data.`Reader(CoreReader)`
|
|
155
|
-
- `save_csv()`
|
|
156
|
-
- `save_log()`
|
|
157
|
-
- `downsample()`
|
|
158
|
-
- `resample()`
|
|
159
|
-
- `generate_plot_data()`
|
|
160
|
-
- `assemble_plot()`
|
|
161
|
-
- `plot_to_file()`
|
|
162
|
-
- `multiplot_to_file()`
|
|
163
|
-
|
|
164
|
-
## CLI-Interface
|
|
165
|
-
|
|
166
|
-
After installing the module the datalib offers some often needed functionality on the command line:
|
|
167
|
-
|
|
168
|
-
**Validate Recordings**
|
|
169
|
-
|
|
170
|
-
- takes a file or directory as an argument
|
|
171
|
-
|
|
172
|
-
```shell
|
|
173
|
-
shepherd-data validate dir_or_file
|
|
174
|
-
|
|
175
|
-
# examples:
|
|
176
|
-
shepherd-data validate ./
|
|
177
|
-
shepherd-data validate hrv_saw_1h.h5
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
**Extract IV-Samples to csv**
|
|
181
|
-
|
|
182
|
-
- takes a file or directory as an argument
|
|
183
|
-
- can take down-sample-factor as an argument
|
|
184
|
-
|
|
185
|
-
```shell
|
|
186
|
-
shepherd-data extract [-f ds-factor] [-s separator_symbol] dir_or_file
|
|
187
|
-
|
|
188
|
-
# examples:
|
|
189
|
-
shepherd-data extract ./
|
|
190
|
-
shepherd-data extract -f 1000 -s ; hrv_saw_1h.h5
|
|
191
|
-
```
|
|
192
|
-
|
|
193
|
-
**Extract meta-data and sys-logs**
|
|
194
|
-
|
|
195
|
-
- takes a file or directory as an argument
|
|
196
|
-
|
|
197
|
-
```shell
|
|
198
|
-
shepherd-data extract-meta dir_or_file
|
|
199
|
-
|
|
200
|
-
# examples:
|
|
201
|
-
shepherd-data extract-meta ./
|
|
202
|
-
shepherd-data extract-meta hrv_saw_1h.h5
|
|
203
|
-
```
|
|
204
|
-
|
|
205
|
-
**Plot IVSamples**
|
|
206
|
-
|
|
207
|
-
- takes a file or directory as an argument
|
|
208
|
-
- can take start- and end-time as an argument
|
|
209
|
-
- can take image-width and -height as an argument
|
|
210
|
-
|
|
211
|
-
```shell
|
|
212
|
-
shepherd-data plot [-s start_time] [-e end_time] [-w plot_width] [-h plot_height] [--multiplot] dir_or_file
|
|
213
|
-
|
|
214
|
-
# examples:
|
|
215
|
-
shepherd-data plot --multiplot ./
|
|
216
|
-
shepherd-data plot -s10 -e20 hrv_saw_1h.h5
|
|
217
|
-
```
|
|
218
|
-
|
|
219
|
-
**Downsample IVSamples (for later GUI-usage, TODO)**
|
|
220
|
-
|
|
221
|
-
- generates a set of downsamplings (20 kHz to 0.1 Hz in x4 to x5 Steps)
|
|
222
|
-
- takes a file or directory as an argument
|
|
223
|
-
- can take down-sample-factor as an argument
|
|
224
|
-
|
|
225
|
-
```shell
|
|
226
|
-
shepherd-data downsample [-f ds-factor] [-r sample-rate] dir_or_file
|
|
227
|
-
|
|
228
|
-
# examples:
|
|
229
|
-
shepherd-data downsample ./
|
|
230
|
-
shepherd-data downsample -f 1000 hrv_saw_1h.h5
|
|
231
|
-
shepherd-data downsample -r 100 hrv_saw_1h.h5
|
|
232
|
-
```
|
|
233
|
-
|
|
234
|
-
## Data-Layout and Design choices
|
|
235
|
-
|
|
236
|
-
Details about the file-structure can be found in the [main-project](https://github.com/orgua/shepherd/blob/main/docs/user/data_format.rst).
|
|
237
|
-
|
|
238
|
-
TODO:
|
|
239
|
-
- update design of file
|
|
240
|
-
- data dtype, mode, ...
|
|
241
|
-
|
|
242
|
-
### Modes and Datatypes
|
|
243
|
-
|
|
244
|
-
- Mode `harvester` recorded a harvesting-source like solar with one of various algorithms
|
|
245
|
-
- Datatype `ivsample` is directly usable by shepherd, input for virtual source / converter
|
|
246
|
-
- Datatype `ivcurve` is directly usable by shepherd, input for a virtual harvester (output are ivsamples)
|
|
247
|
-
- Datatype `isc_voc` is specially for solar-cells and needs to be (at least) transformed into ivcurves later
|
|
248
|
-
- Mode `emulator` replayed a harvester-recording through a virtual converter and supplied a target while recording the power-consumption
|
|
249
|
-
- Datatype `ivsample` is the only output of this mode
|
|
250
|
-
|
|
251
|
-
### Compression & Beaglebone
|
|
252
|
-
|
|
253
|
-
- supported are uncompressed, lzf and gzip with level 1 (order of recommendation)
|
|
254
|
-
- lzf seems better-suited due to lower load, or if space isn't a constraint: uncompressed (None as argument)
|
|
255
|
-
- note: lzf seems to cause trouble with some third party hdf5-tools
|
|
256
|
-
- compression is a heavy load for the beaglebone, but it got more performant with recent python-versions
|
|
257
|
-
- size-experiment A: 24 h of ramping / sawtooth (data is repetitive with 1 minute ramp)
|
|
258
|
-
- gzip-1: 49'646 MiB -> 588 KiB/s
|
|
259
|
-
- lzf: 106'445 MiB -> 1262 KiB/s
|
|
260
|
-
- uncompressed: 131'928 MiB -> 1564 KiB/s
|
|
261
|
-
- cpu-load-experiments (input is 24h sawtooth, python 3.10 with most recent libs as of 2022-04)
|
|
262
|
-
- warning: gpio-traffic and other logging-data can cause lots of load
|
|
263
|
-
|
|
264
|
-
```
|
|
265
|
-
emu_120s_gz1_to_gz1.h5 -> emulator, cpu_util [%] = 65.59, data-rate = 352.0 KiB/s
|
|
266
|
-
emu_120s_gz1_to_lzf.h5 -> emulator, cpu_util [%] = 57.37, data-rate = 686.0 KiB/s
|
|
267
|
-
emu_120s_gz1_to_unc.h5 -> emulator, cpu_util [%] = 53.63, data-rate = 1564.0 KiB/s
|
|
268
|
-
emu_120s_lzf_to_gz1.h5 -> emulator, cpu_util [%] = 63.18, data-rate = 352.0 KiB/s
|
|
269
|
-
emu_120s_lzf_to_lzf.h5 -> emulator, cpu_util [%] = 58.60, data-rate = 686.0 KiB/s
|
|
270
|
-
emu_120s_lzf_to_unc.h5 -> emulator, cpu_util [%] = 55.75, data-rate = 1564.0 KiB/s
|
|
271
|
-
emu_120s_unc_to_gz1.h5 -> emulator, cpu_util [%] = 63.84, data-rate = 351.0 KiB/s
|
|
272
|
-
emu_120s_unc_to_lzf.h5 -> emulator, cpu_util [%] = 57.28, data-rate = 686.0 KiB/s
|
|
273
|
-
emu_120s_unc_to_unc.h5 -> emulator, cpu_util [%] = 51.69, data-rate = 1564.0 KiB/s
|
|
274
|
-
```
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
shepherd_data/__init__.py,sha256=Ys41EYMRuO_55OepmLO6icu4xKGX9I9QJowHWa9jzuo,242
|
|
2
|
-
shepherd_data/cli.py,sha256=u9ImWh-VLVlk1-TJaQs7govRHoiiuF5xIeOCLBzr8k8,15341
|
|
3
|
-
shepherd_data/debug_resampler.py,sha256=Y4uxSXqWKoyyJ1H6TY1vZKe0pWMK_mjQY0xBr7dC_f4,983
|
|
4
|
-
shepherd_data/ivonne.py,sha256=ZGCjctOFYJyMH9AsiF8oaFd_QuvVDbGdPuj3qoICHYg,12017
|
|
5
|
-
shepherd_data/mppt.py,sha256=YWMw3lhReHbjmK1LVgrIo2S_SoBmIXI4a_sqUpHMuGE,3640
|
|
6
|
-
shepherd_data/reader.py,sha256=toX3NLqmifUYfrSp81XrmDJkWm8jdhwUIWy8v530tRI,18440
|
|
7
|
-
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
tests/conftest.py,sha256=BCtm2TbfVf85_AsGOUZLXd6TquicWRMgBq2-q5UA2UM,985
|
|
9
|
-
tests/test_cli.py,sha256=PPbRwgOQbWjNcgR7mIXTnGdboCQ4QDC4Kh74kSaEFhc,181
|
|
10
|
-
tests/test_cli_downsample.py,sha256=dsI04rJrFdNl50uW5DplmA4Uh1S2Y2hCYq-HkM47pG8,1803
|
|
11
|
-
tests/test_cli_extract.py,sha256=XhXNNyiRRuXkDqkgWLaFTZkge6tv2a8jJyAd2zt8QlE,2532
|
|
12
|
-
tests/test_cli_plot.py,sha256=IaN1Z37XYgfkxSzJkfej8Vp9FqSUeYCSog3ratnDePQ,3298
|
|
13
|
-
tests/test_cli_validate.py,sha256=DOfaRq6r05eu8x4SnZOPrPfbbMNu0PTus7lSV8Gkzwo,405
|
|
14
|
-
tests/test_examples.py,sha256=e_uc5h_I4wwJnIWuh-PjpSSTICkYemI831iqhxIy_R0,576
|
|
15
|
-
tests/test_ivonne.py,sha256=cIMtpx7tjRFxL9hpr5OPPgroGLzRZxGC4x9Y7vWgze8,1367
|
|
16
|
-
tests/test_reader.py,sha256=Xt__Khgldv8klZJ3OQcMZevC6_3gPH2_TpOorq2ytNI,90
|
|
17
|
-
shepherd_data-2023.11.1.dist-info/METADATA,sha256=a0Jd5FvKexcOLiRcc46seLGm6vut-kcAHiwyWC2N-3U,9942
|
|
18
|
-
shepherd_data-2023.11.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
|
19
|
-
shepherd_data-2023.11.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
|
|
20
|
-
shepherd_data-2023.11.1.dist-info/top_level.txt,sha256=Bk61YO7iYS43TSWtwdSKb2moGBvJqJIYfGaZsQsv09M,20
|
|
21
|
-
shepherd_data-2023.11.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
22
|
-
shepherd_data-2023.11.1.dist-info/RECORD,,
|
tests/__init__.py
DELETED
|
File without changes
|