shepherd-data 2023.12.1__py3-none-any.whl → 2024.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shepherd_data/__init__.py +8 -5
- shepherd_data/cli.py +29 -29
- shepherd_data/ivonne.py +11 -11
- shepherd_data/mppt.py +22 -15
- shepherd_data/reader.py +31 -23
- shepherd_data-2024.4.2.dist-info/METADATA +88 -0
- shepherd_data-2024.4.2.dist-info/RECORD +11 -0
- {shepherd_data-2023.12.1.dist-info → shepherd_data-2024.4.2.dist-info}/WHEEL +1 -1
- {shepherd_data-2023.12.1.dist-info → shepherd_data-2024.4.2.dist-info}/top_level.txt +0 -1
- shepherd_data/debug_resampler.py +0 -30
- shepherd_data-2023.12.1.dist-info/METADATA +0 -274
- shepherd_data-2023.12.1.dist-info/RECORD +0 -22
- tests/__init__.py +0 -0
- tests/conftest.py +0 -33
- tests/test_cli.py +0 -8
- tests/test_cli_downsample.py +0 -47
- tests/test_cli_extract.py +0 -80
- tests/test_cli_plot.py +0 -120
- tests/test_cli_validate.py +0 -15
- tests/test_examples.py +0 -26
- tests/test_ivonne.py +0 -42
- tests/test_reader.py +0 -3
- {shepherd_data-2023.12.1.dist-info → shepherd_data-2024.4.2.dist-info}/entry_points.txt +0 -0
- {shepherd_data-2023.12.1.dist-info → shepherd_data-2024.4.2.dist-info}/zip-safe +0 -0
shepherd_data/debug_resampler.py
DELETED
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import logging
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
import shepherd_data as shpd
|
|
5
|
-
|
|
6
|
-
file = Path("./hrv_sawtooth_1h.h5")
|
|
7
|
-
samplerate_sps = 1000
|
|
8
|
-
|
|
9
|
-
logger = logging.getLogger("SHPData.debug")
|
|
10
|
-
logger.setLevel(logging.DEBUG)
|
|
11
|
-
|
|
12
|
-
with shpd.Reader(file) as shpr:
|
|
13
|
-
out_file = file.with_suffix(f".fs_{samplerate_sps}.h5")
|
|
14
|
-
logger.info(
|
|
15
|
-
"Resampling '%s' from %d Hz to %d Hz ...",
|
|
16
|
-
file.name,
|
|
17
|
-
shpr.samplerate_sps,
|
|
18
|
-
samplerate_sps,
|
|
19
|
-
)
|
|
20
|
-
with shpd.Writer(
|
|
21
|
-
out_file,
|
|
22
|
-
mode=shpr.get_mode(),
|
|
23
|
-
datatype=shpr.get_datatype(),
|
|
24
|
-
window_samples=shpr.get_window_samples(),
|
|
25
|
-
cal_data=shpr.get_calibration_data(),
|
|
26
|
-
) as shpw:
|
|
27
|
-
shpr.resample(shpr.ds_time, shpw.ds_time, samplerate_dst=samplerate_sps, is_time=True)
|
|
28
|
-
shpr.resample(shpr.ds_voltage, shpw.ds_voltage, samplerate_dst=samplerate_sps)
|
|
29
|
-
shpr.resample(shpr.ds_current, shpw.ds_current, samplerate_dst=samplerate_sps)
|
|
30
|
-
shpw.save_metadata()
|
|
@@ -1,274 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: shepherd-data
|
|
3
|
-
Version: 2023.12.1
|
|
4
|
-
Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
|
|
5
|
-
Home-page: https://pypi.org/project/shepherd-data/
|
|
6
|
-
Author: Ingmar Splitt, Kai Geissdoerfer
|
|
7
|
-
Author-email: ingmar.splitt@tu-dresden.de
|
|
8
|
-
Maintainer-email: ingmar.splitt@tu-dresden.de
|
|
9
|
-
License: MIT
|
|
10
|
-
Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
|
|
11
|
-
Project-URL: Source, https://github.com/orgua/shepherd-datalib
|
|
12
|
-
Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
|
|
13
|
-
Platform: unix
|
|
14
|
-
Platform: linux
|
|
15
|
-
Platform: osx
|
|
16
|
-
Platform: cygwin
|
|
17
|
-
Platform: win32
|
|
18
|
-
Platform: win64
|
|
19
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
20
|
-
Classifier: Intended Audience :: Developers
|
|
21
|
-
Classifier: Intended Audience :: Information Technology
|
|
22
|
-
Classifier: Intended Audience :: Science/Research
|
|
23
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
24
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
25
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
26
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
27
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
28
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
29
|
-
Classifier: Operating System :: OS Independent
|
|
30
|
-
Classifier: Natural Language :: English
|
|
31
|
-
Requires-Python: >=3.8
|
|
32
|
-
Description-Content-Type: text/markdown
|
|
33
|
-
Requires-Dist: h5py
|
|
34
|
-
Requires-Dist: numpy
|
|
35
|
-
Requires-Dist: pyYAML
|
|
36
|
-
Requires-Dist: shepherd-core[inventory] >=2023.12.1
|
|
37
|
-
Requires-Dist: click
|
|
38
|
-
Requires-Dist: matplotlib
|
|
39
|
-
Requires-Dist: pandas
|
|
40
|
-
Requires-Dist: scipy
|
|
41
|
-
Requires-Dist: tqdm
|
|
42
|
-
Provides-Extra: dev
|
|
43
|
-
Requires-Dist: shepherd-core[dev] ; extra == 'dev'
|
|
44
|
-
Requires-Dist: pandas-stubs ; extra == 'dev'
|
|
45
|
-
Provides-Extra: elf
|
|
46
|
-
Requires-Dist: shepherd-core[elf] ; extra == 'elf'
|
|
47
|
-
Provides-Extra: test
|
|
48
|
-
Requires-Dist: shepherd-core[test] ; extra == 'test'
|
|
49
|
-
Requires-Dist: pytest-click ; extra == 'test'
|
|
50
|
-
|
|
51
|
-
# Data Module
|
|
52
|
-
|
|
53
|
-
[](https://pypi.org/project/shepherd_data)
|
|
54
|
-
[](https://pypi.python.org/pypi/shepherd-data)
|
|
55
|
-
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
56
|
-
[](https://github.com/astral-sh/ruff)
|
|
57
|
-
|
|
58
|
-
**Documentation**: <https://orgua.github.io/shepherd/external/shepherd_data.html>
|
|
59
|
-
|
|
60
|
-
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
61
|
-
|
|
62
|
-
**Main Project**: <https://github.com/orgua/shepherd>
|
|
63
|
-
|
|
64
|
-
---
|
|
65
|
-
|
|
66
|
-
This Python Module eases the handling of hdf5-recordings used by the [shepherd](https://github.com/orgua/shepherd)-testbed. Users can read, validate and create files and also extract, down-sample and plot information.
|
|
67
|
-
|
|
68
|
-
## Installation
|
|
69
|
-
|
|
70
|
-
### PIP - Online
|
|
71
|
-
|
|
72
|
-
```shell
|
|
73
|
-
pip3 install shepherd-data -U
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
For bleeding-edge-features or dev-work it is possible to install directly from GitHub-Sources (here `dev`-branch):
|
|
77
|
-
|
|
78
|
-
```Shell
|
|
79
|
-
pip install git+https://github.com/orgua/shepherd-datalib.git@dev#subdirectory=shepherd_data -U
|
|
80
|
-
```
|
|
81
|
-
|
|
82
|
-
## Programming Interface
|
|
83
|
-
|
|
84
|
-
### Basic Usage (recommendation)
|
|
85
|
-
|
|
86
|
-
```python
|
|
87
|
-
import shepherd_data as sd
|
|
88
|
-
|
|
89
|
-
with sd.Reader("./hrv_sawtooth_1h.h5") as db:
|
|
90
|
-
print(f"Mode: {db.get_mode()}")
|
|
91
|
-
print(f"Window: {db.get_window_samples()}")
|
|
92
|
-
print(f"Config: {db.get_config()}")
|
|
93
|
-
```
|
|
94
|
-
|
|
95
|
-
### Available Functionality
|
|
96
|
-
|
|
97
|
-
- `Reader()`
|
|
98
|
-
- file can be checked for plausibility and validity (`is_valid()`)
|
|
99
|
-
- internal structure of h5file (`get_metadata()` or `save_metadata()` ... to yaml) with lots of additional data
|
|
100
|
-
- access data and various converters, calculators
|
|
101
|
-
- `read_buffers()` -> generator that provides one buffer per call, can be configured on first call
|
|
102
|
-
- `get_calibration_data()`
|
|
103
|
-
- `get_windows_samples()`
|
|
104
|
-
- `get_mode()`
|
|
105
|
-
- `get_config()`
|
|
106
|
-
- direct access to root h5-structure via `reader['element']`
|
|
107
|
-
- converters for raw / physical units: `si_to_raw()` & `raw_to_si()`
|
|
108
|
-
- `energy()` sums up recorded power over time
|
|
109
|
-
- `downsample()` (if needed) visualize recording (`plot_to_file()`)
|
|
110
|
-
- `Writer()`
|
|
111
|
-
- inherits all functionality from Reader
|
|
112
|
-
- `append_iv_data_raw()`
|
|
113
|
-
- `append_iv_data_si()`
|
|
114
|
-
- `set_config()`
|
|
115
|
-
- `set_windows_samples()`
|
|
116
|
-
- IVonne Reader
|
|
117
|
-
- `convert_2_ivcurves()` converts ivonne-recording into a shepherd ivcurve
|
|
118
|
-
- `upsample_2_isc_voc()` TODO: for now a upsampled but unusable version of samples of short-circuit-current and open-circuit-voltage
|
|
119
|
-
- `convert_2_ivsamples()` already applies a simple harvesting-algo and creates ivsamples
|
|
120
|
-
- `./examples/`
|
|
121
|
-
- `example_convert_ivonne.py` converts IVonne recording (`jogging_10m.iv`) to shepherd ivcurves, NOTE: slow implementation
|
|
122
|
-
- `example_extract_logs.py` is analyzing all files in directory, saves logging-data and calculates cpu-load and data-rate
|
|
123
|
-
- `example_generate_sawtooth.py` is using Writer to generate a 60s ramp with 1h repetition and uses Reader to dump metadata of that file
|
|
124
|
-
- `example_plot_traces.py` demos some mpl-plots with various zoom levels
|
|
125
|
-
- `example_repair_recordings.py` makes old recordings from shepherd 1.x fit for v2
|
|
126
|
-
- `jogging_10m.iv`
|
|
127
|
-
- 50 Hz measurement with Short-Circuit-Current and two other parameters
|
|
128
|
-
- recorded with "IVonne"
|
|
129
|
-
|
|
130
|
-
### Functionality Update (WIP)
|
|
131
|
-
|
|
132
|
-
- Core.`Reader`
|
|
133
|
-
- `__repr__()`
|
|
134
|
-
- `read_buffers`
|
|
135
|
-
- `get_calibration_data`
|
|
136
|
-
- `get_window_samples`
|
|
137
|
-
- `get_mode`
|
|
138
|
-
- `get_config`
|
|
139
|
-
- `get_hostname`
|
|
140
|
-
- `get_datatype`
|
|
141
|
-
- `get_hrv_config`
|
|
142
|
-
- `is_valid`
|
|
143
|
-
- `energy()`
|
|
144
|
-
- `check_timediffs()`
|
|
145
|
-
- `get_metadata()`
|
|
146
|
-
- `save_metadata()`
|
|
147
|
-
|
|
148
|
-
- `Writer(Reader)` (core, data are the same)
|
|
149
|
-
- `append_iv_data_raw`
|
|
150
|
-
- `append_iv_data_si`
|
|
151
|
-
- `store_config`
|
|
152
|
-
- `store_hostname`
|
|
153
|
-
|
|
154
|
-
- data.`Reader(CoreReader)`
|
|
155
|
-
- `save_csv()`
|
|
156
|
-
- `save_log()`
|
|
157
|
-
- `downsample()`
|
|
158
|
-
- `resample()`
|
|
159
|
-
- `generate_plot_data()`
|
|
160
|
-
- `assemble_plot()`
|
|
161
|
-
- `plot_to_file()`
|
|
162
|
-
- `multiplot_to_file()`
|
|
163
|
-
|
|
164
|
-
## CLI-Interface
|
|
165
|
-
|
|
166
|
-
After installing the module the datalib offers some often needed functionality on the command line:
|
|
167
|
-
|
|
168
|
-
**Validate Recordings**
|
|
169
|
-
|
|
170
|
-
- takes a file or directory as an argument
|
|
171
|
-
|
|
172
|
-
```shell
|
|
173
|
-
shepherd-data validate dir_or_file
|
|
174
|
-
|
|
175
|
-
# examples:
|
|
176
|
-
shepherd-data validate ./
|
|
177
|
-
shepherd-data validate hrv_saw_1h.h5
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
**Extract IV-Samples to csv**
|
|
181
|
-
|
|
182
|
-
- takes a file or directory as an argument
|
|
183
|
-
- can take down-sample-factor as an argument
|
|
184
|
-
|
|
185
|
-
```shell
|
|
186
|
-
shepherd-data extract [-f ds-factor] [-s separator_symbol] dir_or_file
|
|
187
|
-
|
|
188
|
-
# examples:
|
|
189
|
-
shepherd-data extract ./
|
|
190
|
-
shepherd-data extract -f 1000 -s ; hrv_saw_1h.h5
|
|
191
|
-
```
|
|
192
|
-
|
|
193
|
-
**Extract meta-data and sys-logs**
|
|
194
|
-
|
|
195
|
-
- takes a file or directory as an argument
|
|
196
|
-
|
|
197
|
-
```shell
|
|
198
|
-
shepherd-data extract-meta dir_or_file
|
|
199
|
-
|
|
200
|
-
# examples:
|
|
201
|
-
shepherd-data extract-meta ./
|
|
202
|
-
shepherd-data extract-meta hrv_saw_1h.h5
|
|
203
|
-
```
|
|
204
|
-
|
|
205
|
-
**Plot IVSamples**
|
|
206
|
-
|
|
207
|
-
- takes a file or directory as an argument
|
|
208
|
-
- can take start- and end-time as an argument
|
|
209
|
-
- can take image-width and -height as an argument
|
|
210
|
-
|
|
211
|
-
```shell
|
|
212
|
-
shepherd-data plot [-s start_time] [-e end_time] [-w plot_width] [-h plot_height] [--multiplot] dir_or_file
|
|
213
|
-
|
|
214
|
-
# examples:
|
|
215
|
-
shepherd-data plot --multiplot ./
|
|
216
|
-
shepherd-data plot -s10 -e20 hrv_saw_1h.h5
|
|
217
|
-
```
|
|
218
|
-
|
|
219
|
-
**Downsample IVSamples (for later GUI-usage, TODO)**
|
|
220
|
-
|
|
221
|
-
- generates a set of downsamplings (20 kHz to 0.1 Hz in x4 to x5 Steps)
|
|
222
|
-
- takes a file or directory as an argument
|
|
223
|
-
- can take down-sample-factor as an argument
|
|
224
|
-
|
|
225
|
-
```shell
|
|
226
|
-
shepherd-data downsample [-f ds-factor] [-r sample-rate] dir_or_file
|
|
227
|
-
|
|
228
|
-
# examples:
|
|
229
|
-
shepherd-data downsample ./
|
|
230
|
-
shepherd-data downsample -f 1000 hrv_saw_1h.h5
|
|
231
|
-
shepherd-data downsample -r 100 hrv_saw_1h.h5
|
|
232
|
-
```
|
|
233
|
-
|
|
234
|
-
## Data-Layout and Design choices
|
|
235
|
-
|
|
236
|
-
Details about the file-structure can be found in the [main-project](https://github.com/orgua/shepherd/blob/main/docs/user/data_format.rst).
|
|
237
|
-
|
|
238
|
-
TODO:
|
|
239
|
-
- update design of file
|
|
240
|
-
- data dtype, mode, ...
|
|
241
|
-
|
|
242
|
-
### Modes and Datatypes
|
|
243
|
-
|
|
244
|
-
- Mode `harvester` recorded a harvesting-source like solar with one of various algorithms
|
|
245
|
-
- Datatype `ivsample` is directly usable by shepherd, input for virtual source / converter
|
|
246
|
-
- Datatype `ivcurve` is directly usable by shepherd, input for a virtual harvester (output are ivsamples)
|
|
247
|
-
- Datatype `isc_voc` is specially for solar-cells and needs to be (at least) transformed into ivcurves later
|
|
248
|
-
- Mode `emulator` replayed a harvester-recording through a virtual converter and supplied a target while recording the power-consumption
|
|
249
|
-
- Datatype `ivsample` is the only output of this mode
|
|
250
|
-
|
|
251
|
-
### Compression & Beaglebone
|
|
252
|
-
|
|
253
|
-
- supported are uncompressed, lzf and gzip with level 1 (order of recommendation)
|
|
254
|
-
- lzf seems better-suited due to lower load, or if space isn't a constraint: uncompressed (None as argument)
|
|
255
|
-
- note: lzf seems to cause trouble with some third party hdf5-tools
|
|
256
|
-
- compression is a heavy load for the beaglebone, but it got more performant with recent python-versions
|
|
257
|
-
- size-experiment A: 24 h of ramping / sawtooth (data is repetitive with 1 minute ramp)
|
|
258
|
-
- gzip-1: 49'646 MiB -> 588 KiB/s
|
|
259
|
-
- lzf: 106'445 MiB -> 1262 KiB/s
|
|
260
|
-
- uncompressed: 131'928 MiB -> 1564 KiB/s
|
|
261
|
-
- cpu-load-experiments (input is 24h sawtooth, python 3.10 with most recent libs as of 2022-04)
|
|
262
|
-
- warning: gpio-traffic and other logging-data can cause lots of load
|
|
263
|
-
|
|
264
|
-
```
|
|
265
|
-
emu_120s_gz1_to_gz1.h5 -> emulator, cpu_util [%] = 65.59, data-rate = 352.0 KiB/s
|
|
266
|
-
emu_120s_gz1_to_lzf.h5 -> emulator, cpu_util [%] = 57.37, data-rate = 686.0 KiB/s
|
|
267
|
-
emu_120s_gz1_to_unc.h5 -> emulator, cpu_util [%] = 53.63, data-rate = 1564.0 KiB/s
|
|
268
|
-
emu_120s_lzf_to_gz1.h5 -> emulator, cpu_util [%] = 63.18, data-rate = 352.0 KiB/s
|
|
269
|
-
emu_120s_lzf_to_lzf.h5 -> emulator, cpu_util [%] = 58.60, data-rate = 686.0 KiB/s
|
|
270
|
-
emu_120s_lzf_to_unc.h5 -> emulator, cpu_util [%] = 55.75, data-rate = 1564.0 KiB/s
|
|
271
|
-
emu_120s_unc_to_gz1.h5 -> emulator, cpu_util [%] = 63.84, data-rate = 351.0 KiB/s
|
|
272
|
-
emu_120s_unc_to_lzf.h5 -> emulator, cpu_util [%] = 57.28, data-rate = 686.0 KiB/s
|
|
273
|
-
emu_120s_unc_to_unc.h5 -> emulator, cpu_util [%] = 51.69, data-rate = 1564.0 KiB/s
|
|
274
|
-
```
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
shepherd_data/__init__.py,sha256=ysHj9J6TMflWnfscRnZ0Z4I622XD0mmyeVQ_NimE95E,242
|
|
2
|
-
shepherd_data/cli.py,sha256=XnW-SGEUZpFS0IkrhC0wAB_t3eafB7MD2Z1-iJHJW6Y,15013
|
|
3
|
-
shepherd_data/debug_resampler.py,sha256=0VNGuUqOeKii6fvkRJUpv-27uv9p1VFsvShU3tvxwYQ,961
|
|
4
|
-
shepherd_data/ivonne.py,sha256=wkEciyxXuNMFtA6HEm9zR9CIrukwR0CDiuJMtt6_wEA,11601
|
|
5
|
-
shepherd_data/mppt.py,sha256=Ifvx7NdsBoyBJGbc5PEEpHFja20SuouXmRpaCHoODWs,3583
|
|
6
|
-
shepherd_data/reader.py,sha256=hAxvMUt86c3EJif_Q4oYb4zhhCr3nS6Rjk-fWmMrDSY,18035
|
|
7
|
-
tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
-
tests/conftest.py,sha256=BCtm2TbfVf85_AsGOUZLXd6TquicWRMgBq2-q5UA2UM,985
|
|
9
|
-
tests/test_cli.py,sha256=PPbRwgOQbWjNcgR7mIXTnGdboCQ4QDC4Kh74kSaEFhc,181
|
|
10
|
-
tests/test_cli_downsample.py,sha256=i8AKw-doGlw5xtg9YG3kFGShV9we6zv7F48D-t-l8us,1775
|
|
11
|
-
tests/test_cli_extract.py,sha256=-8HdLL1sDF-ygAGpbUeA46fKu1mAP8aZN5xde9tMP3M,2504
|
|
12
|
-
tests/test_cli_plot.py,sha256=jDDoDtvyChQyS4CX2cPq1jDLJNfzDjtJXOftBxL-meM,3242
|
|
13
|
-
tests/test_cli_validate.py,sha256=DOfaRq6r05eu8x4SnZOPrPfbbMNu0PTus7lSV8Gkzwo,405
|
|
14
|
-
tests/test_examples.py,sha256=e_uc5h_I4wwJnIWuh-PjpSSTICkYemI831iqhxIy_R0,576
|
|
15
|
-
tests/test_ivonne.py,sha256=cIMtpx7tjRFxL9hpr5OPPgroGLzRZxGC4x9Y7vWgze8,1367
|
|
16
|
-
tests/test_reader.py,sha256=Xt__Khgldv8klZJ3OQcMZevC6_3gPH2_TpOorq2ytNI,90
|
|
17
|
-
shepherd_data-2023.12.1.dist-info/METADATA,sha256=GnWaKvPEEr5JbBXZIeUiCCeSEIVA845WhBOb1dwLLN4,9999
|
|
18
|
-
shepherd_data-2023.12.1.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
|
19
|
-
shepherd_data-2023.12.1.dist-info/entry_points.txt,sha256=6PBfY36A1xNOdzLiz-Qoukya_UzFZAwOapwmRNnPeZ8,56
|
|
20
|
-
shepherd_data-2023.12.1.dist-info/top_level.txt,sha256=Bk61YO7iYS43TSWtwdSKb2moGBvJqJIYfGaZsQsv09M,20
|
|
21
|
-
shepherd_data-2023.12.1.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
22
|
-
shepherd_data-2023.12.1.dist-info/RECORD,,
|
tests/__init__.py
DELETED
|
File without changes
|
tests/conftest.py
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
import numpy as np
|
|
4
|
-
import pytest
|
|
5
|
-
|
|
6
|
-
from shepherd_core import Compression
|
|
7
|
-
from shepherd_data import Writer
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def generate_h5_file(file_path: Path, file_name: str = "harvest_example.h5") -> Path:
|
|
11
|
-
store_path = file_path / file_name
|
|
12
|
-
|
|
13
|
-
with Writer(store_path, compression=Compression.gzip1) as file:
|
|
14
|
-
file.store_hostname("artificial")
|
|
15
|
-
|
|
16
|
-
duration_s = 2
|
|
17
|
-
repetitions = 5
|
|
18
|
-
timestamp_vector = np.arange(0.0, duration_s, file.sample_interval_ns / 1e9)
|
|
19
|
-
|
|
20
|
-
# values in SI units
|
|
21
|
-
voltages = np.linspace(3.60, 1.90, int(file.samplerate_sps * duration_s))
|
|
22
|
-
currents = np.linspace(100e-6, 2000e-6, int(file.samplerate_sps * duration_s))
|
|
23
|
-
|
|
24
|
-
for idx in range(repetitions):
|
|
25
|
-
timestamps = idx * duration_s + timestamp_vector
|
|
26
|
-
file.append_iv_data_si(timestamps, voltages, currents)
|
|
27
|
-
|
|
28
|
-
return store_path
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@pytest.fixture
|
|
32
|
-
def data_h5(tmp_path: Path) -> Path:
|
|
33
|
-
return generate_h5_file(tmp_path)
|
tests/test_cli.py
DELETED
tests/test_cli_downsample.py
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
from click.testing import CliRunner
|
|
4
|
-
|
|
5
|
-
from shepherd_data.cli import cli
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def test_cli_downsample_file_full(data_h5: Path) -> None:
|
|
9
|
-
res = CliRunner().invoke(cli, ["--verbose", "downsample", "--ds-factor", "10", str(data_h5)])
|
|
10
|
-
assert res.exit_code == 0
|
|
11
|
-
assert data_h5.with_suffix(".downsampled_x10.h5").exists()
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def test_cli_downsample_file_short(data_h5: Path) -> None:
|
|
15
|
-
res = CliRunner().invoke(cli, ["-v", "downsample", "-f", "20", str(data_h5)])
|
|
16
|
-
assert res.exit_code == 0
|
|
17
|
-
assert data_h5.with_suffix(".downsampled_x20.h5").exists()
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def test_cli_downsample_file_min(data_h5: Path) -> None:
|
|
21
|
-
res = CliRunner().invoke(cli, ["--verbose", "downsample", str(data_h5)])
|
|
22
|
-
assert res.exit_code == 0
|
|
23
|
-
assert data_h5.with_suffix(".downsampled_x5.h5").exists()
|
|
24
|
-
assert data_h5.with_suffix(".downsampled_x25.h5").exists()
|
|
25
|
-
assert data_h5.with_suffix(".downsampled_x100.h5").exists()
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
def test_cli_downsample_dir_full(data_h5: Path) -> None:
|
|
29
|
-
print(data_h5.parent)
|
|
30
|
-
print(data_h5.parent.is_dir())
|
|
31
|
-
res = CliRunner().invoke(
|
|
32
|
-
cli, ["--verbose", "downsample", "--ds-factor", "40", str(data_h5.parent)]
|
|
33
|
-
)
|
|
34
|
-
assert res.exit_code == 0
|
|
35
|
-
assert data_h5.with_suffix(".downsampled_x40.h5").exists()
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def test_cli_downsample_rate_file_full(data_h5: Path) -> None:
|
|
39
|
-
res = CliRunner().invoke(cli, ["--verbose", "downsample", "--sample-rate", "100", str(data_h5)])
|
|
40
|
-
assert res.exit_code == 0
|
|
41
|
-
assert data_h5.with_suffix(".downsampled_x1000.h5").exists()
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def test_cli_downsample_rate_file_short(data_h5: Path) -> None:
|
|
45
|
-
res = CliRunner().invoke(cli, ["-v", "downsample", "-r", "200", str(data_h5)])
|
|
46
|
-
assert res.exit_code == 0
|
|
47
|
-
assert data_h5.with_suffix(".downsampled_x500.h5").exists()
|
tests/test_cli_extract.py
DELETED
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
from click.testing import CliRunner
|
|
4
|
-
|
|
5
|
-
from shepherd_data.cli import cli
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def test_cli_extract_file_full(data_h5: Path) -> None:
|
|
9
|
-
res = CliRunner().invoke(
|
|
10
|
-
cli,
|
|
11
|
-
[
|
|
12
|
-
"--verbose",
|
|
13
|
-
"extract",
|
|
14
|
-
"--ds-factor",
|
|
15
|
-
"100",
|
|
16
|
-
"--separator",
|
|
17
|
-
",",
|
|
18
|
-
str(data_h5),
|
|
19
|
-
],
|
|
20
|
-
)
|
|
21
|
-
assert res.exit_code == 0
|
|
22
|
-
assert data_h5.with_suffix(".downsampled_x100.h5").exists()
|
|
23
|
-
assert data_h5.with_suffix(".downsampled_x100.data.csv").exists()
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
def test_cli_extract_file_short(data_h5: Path) -> None:
|
|
27
|
-
res = CliRunner().invoke(cli, ["-v", "extract", "-f", "200", "-s", ";", str(data_h5)])
|
|
28
|
-
assert res.exit_code == 0
|
|
29
|
-
assert data_h5.with_suffix(".downsampled_x200.h5").exists()
|
|
30
|
-
assert data_h5.with_suffix(".downsampled_x200.data.csv").exists()
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def test_cli_extract_file_min(data_h5: Path) -> None:
|
|
34
|
-
res = CliRunner().invoke(cli, ["-v", "extract", str(data_h5)])
|
|
35
|
-
assert res.exit_code == 0
|
|
36
|
-
assert data_h5.with_suffix(".downsampled_x1000.h5").exists()
|
|
37
|
-
assert data_h5.with_suffix(".downsampled_x1000.data.csv").exists()
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
def test_cli_extract_dir_full(data_h5: Path) -> None:
|
|
41
|
-
print(data_h5.parent)
|
|
42
|
-
print(data_h5.parent.is_dir())
|
|
43
|
-
res = CliRunner().invoke(
|
|
44
|
-
cli,
|
|
45
|
-
[
|
|
46
|
-
"--verbose",
|
|
47
|
-
"extract",
|
|
48
|
-
"--ds-factor",
|
|
49
|
-
"2000",
|
|
50
|
-
"--separator",
|
|
51
|
-
";",
|
|
52
|
-
str(data_h5.parent),
|
|
53
|
-
],
|
|
54
|
-
)
|
|
55
|
-
assert res.exit_code == 0
|
|
56
|
-
assert data_h5.with_suffix(".downsampled_x2000.h5").exists()
|
|
57
|
-
assert data_h5.with_suffix(".downsampled_x2000.data.csv").exists()
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
def test_cli_extract_meta_file_full(data_h5: Path) -> None:
|
|
61
|
-
res = CliRunner().invoke(cli, ["--verbose", "extract-meta", "--separator", ";", str(data_h5)])
|
|
62
|
-
assert res.exit_code == 0
|
|
63
|
-
# TODO: nothing to grab here, add in base-file, same for tests below
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def test_cli_extract_meta_file_short(data_h5: Path) -> None:
|
|
67
|
-
res = CliRunner().invoke(cli, ["-v", "extract-meta", "-s", "-", str(data_h5)])
|
|
68
|
-
assert res.exit_code == 0
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
def test_cli_extract_meta_file_min(data_h5: Path) -> None:
|
|
72
|
-
res = CliRunner().invoke(cli, ["-v", "extract-meta", "-s", "-", str(data_h5)])
|
|
73
|
-
assert res.exit_code == 0
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
def test_cli_extract_meta_dir_full(data_h5: Path) -> None:
|
|
77
|
-
res = CliRunner().invoke(
|
|
78
|
-
cli, ["--verbose", "extract-meta", "--separator", ";", str(data_h5.parent)]
|
|
79
|
-
)
|
|
80
|
-
assert res.exit_code == 0
|
tests/test_cli_plot.py
DELETED
|
@@ -1,120 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
from click.testing import CliRunner
|
|
4
|
-
|
|
5
|
-
from shepherd_data.cli import cli
|
|
6
|
-
|
|
7
|
-
from .conftest import generate_h5_file
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def test_cli_plot_file_full(data_h5: Path) -> None:
|
|
11
|
-
res = CliRunner().invoke(
|
|
12
|
-
cli,
|
|
13
|
-
[
|
|
14
|
-
"--verbose",
|
|
15
|
-
"plot",
|
|
16
|
-
"--start",
|
|
17
|
-
"0",
|
|
18
|
-
"--end",
|
|
19
|
-
"8",
|
|
20
|
-
"--width",
|
|
21
|
-
"50",
|
|
22
|
-
"--height",
|
|
23
|
-
"10",
|
|
24
|
-
str(data_h5),
|
|
25
|
-
],
|
|
26
|
-
)
|
|
27
|
-
assert res.exit_code == 0
|
|
28
|
-
assert data_h5.with_suffix(".plot_0s000_to_8s000.png").exists()
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def test_cli_plot_file_short(data_h5: Path) -> None:
|
|
32
|
-
res = CliRunner().invoke(
|
|
33
|
-
cli,
|
|
34
|
-
[
|
|
35
|
-
"-v",
|
|
36
|
-
"plot",
|
|
37
|
-
"-s",
|
|
38
|
-
"2.345",
|
|
39
|
-
"-e",
|
|
40
|
-
"8.765",
|
|
41
|
-
"-w",
|
|
42
|
-
"30",
|
|
43
|
-
"-h",
|
|
44
|
-
"20",
|
|
45
|
-
str(data_h5),
|
|
46
|
-
],
|
|
47
|
-
)
|
|
48
|
-
assert res.exit_code == 0
|
|
49
|
-
assert data_h5.with_suffix(".plot_2s345_to_8s765.png").exists()
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def test_cli_plot_file_min(data_h5: Path) -> None:
|
|
53
|
-
res = CliRunner().invoke(cli, ["-v", "plot", str(data_h5)])
|
|
54
|
-
assert res.exit_code == 0
|
|
55
|
-
assert data_h5.with_suffix(".plot_0s000_to_10s000.png").exists() # full duration of file
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def test_cli_plot_dir_min(tmp_path: Path) -> None:
|
|
59
|
-
file1_path = generate_h5_file(tmp_path, "hrv_file1.h5")
|
|
60
|
-
file2_path = generate_h5_file(tmp_path, "hrv_file2.h5")
|
|
61
|
-
res = CliRunner().invoke(cli, ["-v", "plot", str(tmp_path.resolve())])
|
|
62
|
-
assert res.exit_code == 0
|
|
63
|
-
assert file1_path.with_suffix(".plot_0s000_to_10s000.png").exists() # full duration of file
|
|
64
|
-
assert file2_path.with_suffix(".plot_0s000_to_10s000.png").exists() # full duration of file
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def test_cli_multiplot_dir_full(tmp_path: Path) -> None:
|
|
68
|
-
generate_h5_file(tmp_path, "hrv_file1.h5")
|
|
69
|
-
generate_h5_file(tmp_path, "hrv_file2.h5")
|
|
70
|
-
res = CliRunner().invoke(
|
|
71
|
-
cli,
|
|
72
|
-
[
|
|
73
|
-
"--verbose",
|
|
74
|
-
"plot",
|
|
75
|
-
"--start",
|
|
76
|
-
"1",
|
|
77
|
-
"--end",
|
|
78
|
-
"7",
|
|
79
|
-
"--width",
|
|
80
|
-
"40",
|
|
81
|
-
"--height",
|
|
82
|
-
"10",
|
|
83
|
-
"--multiplot",
|
|
84
|
-
str(tmp_path),
|
|
85
|
-
],
|
|
86
|
-
)
|
|
87
|
-
assert res.exit_code == 0
|
|
88
|
-
assert tmp_path.with_suffix(".multiplot_1s000_to_7s000.png").exists()
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def test_cli_multiplot_dir_short(tmp_path: Path) -> None:
|
|
92
|
-
generate_h5_file(tmp_path, "hrv_file1.h5")
|
|
93
|
-
generate_h5_file(tmp_path, "hrv_file2.h5")
|
|
94
|
-
res = CliRunner().invoke(
|
|
95
|
-
cli,
|
|
96
|
-
[
|
|
97
|
-
"-v",
|
|
98
|
-
"plot",
|
|
99
|
-
"-s",
|
|
100
|
-
"2.345",
|
|
101
|
-
"-e",
|
|
102
|
-
"8.765",
|
|
103
|
-
"-w",
|
|
104
|
-
"30",
|
|
105
|
-
"-h",
|
|
106
|
-
"20",
|
|
107
|
-
"-m",
|
|
108
|
-
str(tmp_path),
|
|
109
|
-
],
|
|
110
|
-
)
|
|
111
|
-
assert res.exit_code == 0
|
|
112
|
-
assert tmp_path.with_suffix(".multiplot_2s345_to_8s765.png").exists()
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def test_cli_multiplot_dir_min(tmp_path: Path) -> None:
|
|
116
|
-
generate_h5_file(tmp_path, "hrv_file1.h5")
|
|
117
|
-
generate_h5_file(tmp_path, "hrv_file2.h5")
|
|
118
|
-
res = CliRunner().invoke(cli, ["-v", "plot", "-m", str(tmp_path)])
|
|
119
|
-
assert res.exit_code == 0
|
|
120
|
-
assert tmp_path.with_suffix(".multiplot_0s000_to_10s000.png").exists() # full duration of file
|
tests/test_cli_validate.py
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
|
-
from click.testing import CliRunner
|
|
4
|
-
|
|
5
|
-
from shepherd_data.cli import cli
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def test_cli_validate_file(data_h5: Path) -> None:
|
|
9
|
-
res = CliRunner().invoke(cli, ["-v", "validate", str(data_h5)])
|
|
10
|
-
assert res.exit_code == 0
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def test_cli_validate_dir(data_h5: Path) -> None:
|
|
14
|
-
res = CliRunner().invoke(cli, ["-v", "validate", str(data_h5.parent)])
|
|
15
|
-
assert res.exit_code == 0
|
tests/test_examples.py
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import subprocess
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
import pytest
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
@pytest.fixture
|
|
9
|
-
def example_path() -> Path:
|
|
10
|
-
path = Path(__file__).resolve().parent.parent / "examples"
|
|
11
|
-
os.chdir(path)
|
|
12
|
-
return path
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
examples = [
|
|
16
|
-
"example_convert_ivonne.py",
|
|
17
|
-
"example_extract_logs.py",
|
|
18
|
-
"example_generate_sawtooth.py",
|
|
19
|
-
"example_plot_traces.py",
|
|
20
|
-
"example_repair_recordings.py",
|
|
21
|
-
]
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
@pytest.mark.parametrize("file", examples)
|
|
25
|
-
def test_example_scripts(example_path: Path, file: str) -> None:
|
|
26
|
-
subprocess.check_call(f"python {example_path / file}", shell=True)
|