shepherd-core 2023.12.1__py3-none-any.whl → 2024.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shepherd_core/__init__.py +5 -4
- shepherd_core/calibration_hw_def.py +9 -1
- shepherd_core/commons.py +2 -0
- shepherd_core/data_models/__init__.py +11 -0
- shepherd_core/data_models/base/__init__.py +4 -1
- shepherd_core/data_models/base/cal_measurement.py +18 -6
- shepherd_core/data_models/base/calibration.py +41 -16
- shepherd_core/data_models/base/content.py +20 -5
- shepherd_core/data_models/base/shepherd.py +23 -12
- shepherd_core/data_models/base/timezone.py +5 -0
- shepherd_core/data_models/base/wrapper.py +3 -3
- shepherd_core/data_models/content/__init__.py +5 -4
- shepherd_core/data_models/content/_external_fixtures.yaml +32 -16
- shepherd_core/data_models/content/energy_environment.py +7 -5
- shepherd_core/data_models/content/energy_environment_fixture.yaml +3 -0
- shepherd_core/data_models/content/firmware.py +12 -5
- shepherd_core/data_models/content/firmware_datatype.py +7 -0
- shepherd_core/data_models/content/virtual_harvester.py +25 -20
- shepherd_core/data_models/content/virtual_harvester_fixture.yaml +1 -0
- shepherd_core/data_models/content/virtual_source.py +40 -23
- shepherd_core/data_models/content/virtual_source_fixture.yaml +1 -0
- shepherd_core/data_models/experiment/__init__.py +5 -4
- shepherd_core/data_models/experiment/experiment.py +16 -15
- shepherd_core/data_models/experiment/observer_features.py +18 -12
- shepherd_core/data_models/experiment/target_config.py +11 -7
- shepherd_core/data_models/readme.md +88 -0
- shepherd_core/data_models/task/__init__.py +10 -3
- shepherd_core/data_models/task/emulation.py +9 -6
- shepherd_core/data_models/task/firmware_mod.py +4 -2
- shepherd_core/data_models/task/harvest.py +5 -4
- shepherd_core/data_models/task/observer_tasks.py +4 -2
- shepherd_core/data_models/task/programming.py +3 -1
- shepherd_core/data_models/task/testbed_tasks.py +10 -4
- shepherd_core/data_models/testbed/__init__.py +5 -2
- shepherd_core/data_models/testbed/cape.py +8 -6
- shepherd_core/data_models/testbed/gpio.py +11 -9
- shepherd_core/data_models/testbed/mcu.py +10 -10
- shepherd_core/data_models/testbed/observer.py +10 -5
- shepherd_core/data_models/testbed/observer_fixture.yaml +23 -22
- shepherd_core/data_models/testbed/target.py +5 -3
- shepherd_core/data_models/testbed/target_fixture.yaml +11 -11
- shepherd_core/data_models/testbed/testbed.py +6 -3
- shepherd_core/decoder_waveform/__init__.py +2 -0
- shepherd_core/decoder_waveform/uart.py +44 -25
- shepherd_core/fw_tools/__init__.py +2 -0
- shepherd_core/fw_tools/converter.py +20 -9
- shepherd_core/fw_tools/converter_elf.py +3 -0
- shepherd_core/fw_tools/patcher.py +16 -4
- shepherd_core/fw_tools/validation.py +25 -5
- shepherd_core/inventory/__init__.py +66 -6
- shepherd_core/inventory/python.py +4 -0
- shepherd_core/inventory/system.py +13 -1
- shepherd_core/inventory/target.py +4 -0
- shepherd_core/logger.py +5 -0
- shepherd_core/reader.py +44 -26
- shepherd_core/testbed_client/__init__.py +2 -0
- shepherd_core/testbed_client/cache_path.py +17 -0
- shepherd_core/testbed_client/client.py +14 -8
- shepherd_core/testbed_client/fixtures.py +30 -11
- shepherd_core/testbed_client/user_model.py +13 -6
- shepherd_core/vsource/__init__.py +2 -0
- shepherd_core/vsource/virtual_converter_model.py +11 -4
- shepherd_core/vsource/virtual_harvester_model.py +8 -1
- shepherd_core/vsource/virtual_source_model.py +10 -5
- shepherd_core/writer.py +28 -20
- {shepherd_core-2023.12.1.dist-info → shepherd_core-2024.4.2.dist-info}/METADATA +50 -34
- shepherd_core-2024.4.2.dist-info/RECORD +75 -0
- {shepherd_core-2023.12.1.dist-info → shepherd_core-2024.4.2.dist-info}/WHEEL +1 -1
- {shepherd_core-2023.12.1.dist-info → shepherd_core-2024.4.2.dist-info}/top_level.txt +0 -1
- shepherd_core-2023.12.1.dist-info/RECORD +0 -117
- tests/__init__.py +0 -0
- tests/conftest.py +0 -64
- tests/data_models/__init__.py +0 -0
- tests/data_models/conftest.py +0 -14
- tests/data_models/example_cal_data.yaml +0 -31
- tests/data_models/example_cal_data_faulty.yaml +0 -29
- tests/data_models/example_cal_meas.yaml +0 -178
- tests/data_models/example_cal_meas_faulty1.yaml +0 -142
- tests/data_models/example_cal_meas_faulty2.yaml +0 -136
- tests/data_models/example_config_emulator.yaml +0 -41
- tests/data_models/example_config_experiment.yaml +0 -16
- tests/data_models/example_config_experiment_alternative.yaml +0 -14
- tests/data_models/example_config_harvester.yaml +0 -15
- tests/data_models/example_config_testbed.yaml +0 -26
- tests/data_models/example_config_virtsource.yaml +0 -78
- tests/data_models/test_base_models.py +0 -205
- tests/data_models/test_content_fixtures.py +0 -41
- tests/data_models/test_content_models.py +0 -282
- tests/data_models/test_examples.py +0 -48
- tests/data_models/test_experiment_models.py +0 -277
- tests/data_models/test_task_generation.py +0 -52
- tests/data_models/test_task_models.py +0 -131
- tests/data_models/test_testbed_fixtures.py +0 -47
- tests/data_models/test_testbed_models.py +0 -187
- tests/decoder_waveform/__init__.py +0 -0
- tests/decoder_waveform/test_decoder.py +0 -34
- tests/fw_tools/__init__.py +0 -0
- tests/fw_tools/conftest.py +0 -5
- tests/fw_tools/test_converter.py +0 -76
- tests/fw_tools/test_patcher.py +0 -66
- tests/fw_tools/test_validation.py +0 -56
- tests/inventory/__init__.py +0 -0
- tests/inventory/test_inventory.py +0 -20
- tests/test_cal_hw.py +0 -34
- tests/test_examples.py +0 -40
- tests/test_logger.py +0 -15
- tests/test_reader.py +0 -283
- tests/test_writer.py +0 -169
- tests/testbed_client/__init__.py +0 -0
- tests/vsource/__init__.py +0 -0
- tests/vsource/conftest.py +0 -49
- tests/vsource/test_converter.py +0 -161
- tests/vsource/test_harvester.py +0 -73
- tests/vsource/test_z.py +0 -5
- /shepherd_core/data_models/{doc_virtual_source.py → virtual_source_doc.txt} +0 -0
- {shepherd_core-2023.12.1.dist-info → shepherd_core-2024.4.2.dist-info}/zip-safe +0 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
"""Current implementation of a file-based database."""
|
|
2
|
+
|
|
1
3
|
import copy
|
|
2
4
|
import os
|
|
3
5
|
import pickle
|
|
@@ -18,6 +20,7 @@ from ..data_models.base.timezone import local_now
|
|
|
18
20
|
from ..data_models.base.timezone import local_tz
|
|
19
21
|
from ..data_models.base.wrapper import Wrapper
|
|
20
22
|
from ..logger import logger
|
|
23
|
+
from .cache_path import cache_user_path
|
|
21
24
|
|
|
22
25
|
# Proposed field-name:
|
|
23
26
|
# - inheritance
|
|
@@ -28,7 +31,7 @@ from ..logger import logger
|
|
|
28
31
|
|
|
29
32
|
|
|
30
33
|
class Fixture:
|
|
31
|
-
"""Current implementation of a file-based database"""
|
|
34
|
+
"""Current implementation of a file-based database."""
|
|
32
35
|
|
|
33
36
|
def __init__(self, model_type: str) -> None:
|
|
34
37
|
self.model_type: str = model_type.lower()
|
|
@@ -62,7 +65,8 @@ class Fixture:
|
|
|
62
65
|
key = int(key) if key.isdigit() else None
|
|
63
66
|
if key in self.elements_by_id:
|
|
64
67
|
return self.elements_by_id[int(key)]
|
|
65
|
-
|
|
68
|
+
msg = f"{self.model_type} '{key}' not found!"
|
|
69
|
+
raise ValueError(msg)
|
|
66
70
|
|
|
67
71
|
def __iter__(self) -> Self:
|
|
68
72
|
self._iter_index = 0
|
|
@@ -94,12 +98,14 @@ class Fixture:
|
|
|
94
98
|
if "name" in values and len(chain) < 1:
|
|
95
99
|
base_name = values.get("name")
|
|
96
100
|
if base_name in chain:
|
|
101
|
+
msg = f"Inheritance-Circle detected ({base_name} already in {chain})"
|
|
97
102
|
raise ValueError(
|
|
98
|
-
|
|
103
|
+
msg,
|
|
99
104
|
)
|
|
100
105
|
if base_name == fixture_name:
|
|
106
|
+
msg = f"Inheritance-Circle detected ({base_name} == {fixture_name})"
|
|
101
107
|
raise ValueError(
|
|
102
|
-
|
|
108
|
+
msg,
|
|
103
109
|
)
|
|
104
110
|
chain.append(base_name)
|
|
105
111
|
fixture_base = copy.copy(self[fixture_name])
|
|
@@ -148,15 +154,18 @@ class Fixture:
|
|
|
148
154
|
def query_id(self, _id: int) -> dict:
|
|
149
155
|
if isinstance(_id, int) and _id in self.elements_by_id:
|
|
150
156
|
return self.elements_by_id[_id]
|
|
151
|
-
|
|
157
|
+
msg = f"Initialization of {self.model_type} by ID failed - {_id} is unknown!"
|
|
158
|
+
raise ValueError(msg)
|
|
152
159
|
|
|
153
160
|
def query_name(self, name: str) -> dict:
|
|
154
161
|
if isinstance(name, str) and name.lower() in self.elements_by_name:
|
|
155
162
|
return self.elements_by_name[name.lower()]
|
|
156
|
-
|
|
163
|
+
msg = f"Initialization of {self.model_type} by name failed - {name} is unknown!"
|
|
164
|
+
raise ValueError(msg)
|
|
157
165
|
|
|
158
166
|
|
|
159
167
|
def file_older_than(file: Path, delta: timedelta) -> bool:
|
|
168
|
+
"""Decide if file is older than a specific duration of time."""
|
|
160
169
|
cutoff = local_now() - delta
|
|
161
170
|
mtime = datetime.fromtimestamp(file.stat().st_mtime, tz=local_tz())
|
|
162
171
|
if mtime < cutoff:
|
|
@@ -165,6 +174,8 @@ def file_older_than(file: Path, delta: timedelta) -> bool:
|
|
|
165
174
|
|
|
166
175
|
|
|
167
176
|
class Fixtures:
|
|
177
|
+
"""A collection of individual fixture-elements."""
|
|
178
|
+
|
|
168
179
|
suffix = ".yaml"
|
|
169
180
|
|
|
170
181
|
@validate_call
|
|
@@ -174,10 +185,11 @@ class Fixtures:
|
|
|
174
185
|
else:
|
|
175
186
|
self.file_path = file_path
|
|
176
187
|
self.components: Dict[str, Fixture] = {}
|
|
177
|
-
save_path =
|
|
188
|
+
save_path = cache_user_path / "fixtures.pickle"
|
|
178
189
|
|
|
179
190
|
if save_path.exists() and not file_older_than(save_path, timedelta(hours=24)) and not reset:
|
|
180
191
|
# speedup
|
|
192
|
+
# TODO: also add version as criterion
|
|
181
193
|
with save_path.open("rb", buffering=-1) as fd:
|
|
182
194
|
self.components = pickle.load(fd) # noqa: S301
|
|
183
195
|
logger.debug(" -> found & used pickled fixtures")
|
|
@@ -192,8 +204,12 @@ class Fixtures:
|
|
|
192
204
|
for file in files:
|
|
193
205
|
self.insert_file(file)
|
|
194
206
|
|
|
195
|
-
|
|
196
|
-
|
|
207
|
+
if len(self.components) < 1:
|
|
208
|
+
logger.error(f"No fixture-components found at {self.file_path.as_posix()}")
|
|
209
|
+
else:
|
|
210
|
+
save_path.parent.mkdir(parents=True, exist_ok=True)
|
|
211
|
+
with save_path.open("wb", buffering=-1) as fd:
|
|
212
|
+
pickle.dump(self.components, fd)
|
|
197
213
|
|
|
198
214
|
@validate_call
|
|
199
215
|
def insert_file(self, file: Path) -> None:
|
|
@@ -215,17 +231,20 @@ class Fixtures:
|
|
|
215
231
|
key = key.lower()
|
|
216
232
|
if key in self.components:
|
|
217
233
|
return self.components[key]
|
|
218
|
-
|
|
234
|
+
msg = f"Component '{key}' not found!"
|
|
235
|
+
raise ValueError(msg)
|
|
219
236
|
|
|
220
237
|
def keys(self): # noqa: ANN201
|
|
221
238
|
return self.components.keys()
|
|
222
239
|
|
|
223
240
|
@staticmethod
|
|
224
241
|
def to_file(file: Path) -> None:
|
|
225
|
-
|
|
242
|
+
msg = f"TODO (val={file})"
|
|
243
|
+
raise NotImplementedError(msg)
|
|
226
244
|
|
|
227
245
|
|
|
228
246
|
def get_files(start_path: Path, suffix: str, recursion_depth: int = 0) -> List[Path]:
|
|
247
|
+
"""Generate a recursive list of all files in a directory."""
|
|
229
248
|
if recursion_depth == 0:
|
|
230
249
|
suffix = suffix.lower().split(".")[-1]
|
|
231
250
|
dir_items = os.scandir(start_path)
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
"""Model for user-management."""
|
|
2
|
+
|
|
1
3
|
import secrets
|
|
2
4
|
from hashlib import pbkdf2_hmac
|
|
3
5
|
from typing import Optional
|
|
6
|
+
from typing import Union
|
|
7
|
+
from uuid import uuid4
|
|
4
8
|
|
|
9
|
+
from pydantic import UUID4
|
|
5
10
|
from pydantic import EmailStr
|
|
6
11
|
from pydantic import Field
|
|
7
12
|
from pydantic import SecretBytes
|
|
@@ -11,17 +16,18 @@ from pydantic import model_validator
|
|
|
11
16
|
from pydantic import validate_call
|
|
12
17
|
from typing_extensions import Annotated
|
|
13
18
|
|
|
14
|
-
from ..data_models.base.content import IdInt
|
|
15
19
|
from ..data_models.base.content import NameStr
|
|
16
20
|
from ..data_models.base.content import SafeStr
|
|
17
|
-
from ..data_models.base.content import id_default
|
|
18
21
|
from ..data_models.base.shepherd import ShpModel
|
|
19
22
|
|
|
20
23
|
|
|
21
24
|
@validate_call
|
|
22
25
|
def hash_password(pw: Annotated[str, StringConstraints(min_length=20, max_length=100)]) -> bytes:
|
|
23
|
-
|
|
26
|
+
"""Generate a hash of a string.
|
|
27
|
+
|
|
24
28
|
# NOTE: 1M Iterations need 25s on beaglebone
|
|
29
|
+
# TODO: add salt of testbed -> this fn should be part of Testbed-Object
|
|
30
|
+
"""
|
|
25
31
|
return pbkdf2_hmac(
|
|
26
32
|
"sha512",
|
|
27
33
|
password=pw.encode("utf-8"),
|
|
@@ -32,11 +38,12 @@ def hash_password(pw: Annotated[str, StringConstraints(min_length=20, max_length
|
|
|
32
38
|
|
|
33
39
|
|
|
34
40
|
class User(ShpModel):
|
|
35
|
-
"""meta-data representation of a testbed-component (physical object)"""
|
|
41
|
+
"""meta-data representation of a testbed-component (physical object)."""
|
|
36
42
|
|
|
37
|
-
id:
|
|
43
|
+
# id: UUID4 = Field( # TODO: db-migration - temp fix for documentation
|
|
44
|
+
id: Union[UUID4, int] = Field(
|
|
38
45
|
description="Unique ID",
|
|
39
|
-
default_factory=
|
|
46
|
+
default_factory=uuid4,
|
|
40
47
|
)
|
|
41
48
|
name: NameStr
|
|
42
49
|
description: Optional[SafeStr] = None
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""This is ported py-version of the pru-code.
|
|
2
|
+
|
|
3
|
+
Goals:
|
|
4
|
+
|
|
2
5
|
- stay close to original code-base
|
|
3
6
|
- offer a comparison for the tests
|
|
4
7
|
- step 1 to a virtualization of emulation
|
|
@@ -6,7 +9,8 @@
|
|
|
6
9
|
NOTE: DO NOT OPTIMIZE -> stay close to original code-base
|
|
7
10
|
|
|
8
11
|
Compromises:
|
|
9
|
-
|
|
12
|
+
|
|
13
|
+
- bitshifted values (i.e. _n28) are converted to float without shift
|
|
10
14
|
|
|
11
15
|
"""
|
|
12
16
|
|
|
@@ -19,7 +23,7 @@ from ..data_models.content.virtual_source import ConverterPRUConfig
|
|
|
19
23
|
|
|
20
24
|
|
|
21
25
|
class PruCalibration:
|
|
22
|
-
"""part of calibration.h"""
|
|
26
|
+
"""part of calibration.h."""
|
|
23
27
|
|
|
24
28
|
def __init__(self, cal_emu: Optional[CalibrationEmulator] = None) -> None:
|
|
25
29
|
self.cal = cal_emu if cal_emu else CalibrationEmulator()
|
|
@@ -30,7 +34,8 @@ class PruCalibration:
|
|
|
30
34
|
|
|
31
35
|
@staticmethod
|
|
32
36
|
def conv_adc_raw_to_uV(voltage_raw: int) -> float:
|
|
33
|
-
|
|
37
|
+
msg = f"This Fn should not been used (val={voltage_raw})"
|
|
38
|
+
raise RuntimeError(msg)
|
|
34
39
|
|
|
35
40
|
def conv_uV_to_dac_raw(self, voltage_uV: float) -> int:
|
|
36
41
|
dac_raw = self.cal.dac_V_A.si_to_raw(float(voltage_uV) / (10**6))
|
|
@@ -40,6 +45,8 @@ class PruCalibration:
|
|
|
40
45
|
|
|
41
46
|
|
|
42
47
|
class VirtualConverterModel:
|
|
48
|
+
"""Ported python version of the pru vCnv."""
|
|
49
|
+
|
|
43
50
|
def __init__(self, cfg: ConverterPRUConfig, cal: PruCalibration) -> None:
|
|
44
51
|
self._cal: PruCalibration = cal
|
|
45
52
|
self._cfg: ConverterPRUConfig = cfg
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
"""this is ported py-version of the pru-code
|
|
1
|
+
"""this is ported py-version of the pru-code.
|
|
2
|
+
|
|
3
|
+
Goals:
|
|
4
|
+
|
|
2
5
|
- stay close to original code-base
|
|
3
6
|
- offer a comparison for the tests
|
|
4
7
|
- step 1 to a virtualization of emulation
|
|
@@ -7,16 +10,20 @@ NOTE1: DO NOT OPTIMIZE -> stay close to original c-code-base
|
|
|
7
10
|
NOTE2: adc-harvest-routines are not part of this model (virtual_harvester lines 66:289)
|
|
8
11
|
|
|
9
12
|
Compromises:
|
|
13
|
+
|
|
10
14
|
- Py has to map the settings-list to internal vars -> is kernel-task
|
|
11
15
|
- Python has no static vars -> FName_reset is handling the class-vars
|
|
12
16
|
|
|
13
17
|
"""
|
|
18
|
+
|
|
14
19
|
from typing import Tuple
|
|
15
20
|
|
|
16
21
|
from ..data_models.content.virtual_harvester import HarvesterPRUConfig
|
|
17
22
|
|
|
18
23
|
|
|
19
24
|
class VirtualHarvesterModel:
|
|
25
|
+
"""Ported python version of the pru vHrv."""
|
|
26
|
+
|
|
20
27
|
HRV_IVCURVE: int = 2**4
|
|
21
28
|
HRV_CV: int = 2**8
|
|
22
29
|
HRV_MPPT_VOC: int = 2**12
|
|
@@ -1,11 +1,15 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""This is ported py-version of the pru-code.
|
|
2
|
+
|
|
3
|
+
Goals:
|
|
4
|
+
|
|
2
5
|
- stay close to original code-base
|
|
3
6
|
- offer a comparison for the tests
|
|
4
|
-
- step 1 to a virtualization of emulation
|
|
7
|
+
- step 1 to a virtualization of emulation.
|
|
5
8
|
|
|
6
9
|
NOTE: DO NOT OPTIMIZE -> stay close to original code-base
|
|
7
10
|
|
|
8
11
|
"""
|
|
12
|
+
|
|
9
13
|
from typing import Optional
|
|
10
14
|
|
|
11
15
|
from ..data_models import CalibrationEmulator
|
|
@@ -19,7 +23,7 @@ from .virtual_harvester_model import VirtualHarvesterModel
|
|
|
19
23
|
|
|
20
24
|
|
|
21
25
|
class VirtualSourceModel:
|
|
22
|
-
"""part of sampling.c"""
|
|
26
|
+
"""part of sampling.c."""
|
|
23
27
|
|
|
24
28
|
def __init__(
|
|
25
29
|
self,
|
|
@@ -52,8 +56,9 @@ class VirtualSourceModel:
|
|
|
52
56
|
self.W_out_fWs: float = 0.0
|
|
53
57
|
|
|
54
58
|
def iterate_sampling(self, V_inp_uV: int = 0, I_inp_nA: int = 0, I_out_nA: int = 0) -> int:
|
|
55
|
-
"""TEST-SIMPLIFICATION - code below is not part of pru-code
|
|
56
|
-
|
|
59
|
+
"""TEST-SIMPLIFICATION - code below is not part of pru-code.
|
|
60
|
+
|
|
61
|
+
It originates from sample_emulator() in sampling.c.
|
|
57
62
|
|
|
58
63
|
:param V_inp_uV:
|
|
59
64
|
:param I_inp_nA:
|
shepherd_core/writer.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
"""Writer that inherits from Reader-Baseclass
|
|
2
|
-
|
|
1
|
+
"""Writer that inherits from Reader-Baseclass."""
|
|
2
|
+
|
|
3
3
|
import logging
|
|
4
4
|
import math
|
|
5
5
|
import pathlib
|
|
@@ -35,10 +35,12 @@ from .reader import Reader
|
|
|
35
35
|
def path2str(
|
|
36
36
|
dumper: Dumper, data: Union[pathlib.Path, pathlib.WindowsPath, pathlib.PosixPath]
|
|
37
37
|
) -> ScalarNode:
|
|
38
|
+
"""Add a yaml-representation for a specific datatype."""
|
|
38
39
|
return dumper.represent_scalar("tag:yaml.org,2002:str", str(data.as_posix()))
|
|
39
40
|
|
|
40
41
|
|
|
41
42
|
def time2int(dumper: Dumper, data: timedelta) -> ScalarNode:
|
|
43
|
+
"""Add a yaml-representation for a specific datatype."""
|
|
42
44
|
return dumper.represent_scalar("tag:yaml.org,2002:int", str(int(data.total_seconds())))
|
|
43
45
|
|
|
44
46
|
|
|
@@ -49,7 +51,7 @@ yaml.add_representer(timedelta, time2int, SafeDumper)
|
|
|
49
51
|
|
|
50
52
|
|
|
51
53
|
def unique_path(base_path: Union[str, Path], suffix: str) -> Path:
|
|
52
|
-
"""
|
|
54
|
+
"""Find an unused filename in case it already exists.
|
|
53
55
|
|
|
54
56
|
:param base_path: file-path to test
|
|
55
57
|
:param suffix: file-suffix
|
|
@@ -64,7 +66,7 @@ def unique_path(base_path: Union[str, Path], suffix: str) -> Path:
|
|
|
64
66
|
|
|
65
67
|
|
|
66
68
|
class Writer(Reader):
|
|
67
|
-
"""Stores data for Shepherd in HDF5 format
|
|
69
|
+
"""Stores data for Shepherd in HDF5 format.
|
|
68
70
|
|
|
69
71
|
Choose lossless compression filter
|
|
70
72
|
- lzf: low to moderate compression, VERY fast, no options
|
|
@@ -87,6 +89,7 @@ class Writer(Reader):
|
|
|
87
89
|
otherwise a unique name will be found
|
|
88
90
|
compression: (str) use either None, lzf or "1" (gzips compression level)
|
|
89
91
|
verbose: (bool) provides more debug-info
|
|
92
|
+
|
|
90
93
|
"""
|
|
91
94
|
|
|
92
95
|
comp_default: int = 1
|
|
@@ -123,7 +126,8 @@ class Writer(Reader):
|
|
|
123
126
|
self.file_path: Path = file_path.resolve()
|
|
124
127
|
self._logger.info("Storing data to '%s'", self.file_path)
|
|
125
128
|
elif file_path.exists() and not file_path.is_file():
|
|
126
|
-
|
|
129
|
+
msg = f"Path is not a file ({file_path})"
|
|
130
|
+
raise TypeError(msg)
|
|
127
131
|
else:
|
|
128
132
|
base_dir = file_path.resolve().parents[0]
|
|
129
133
|
self.file_path = unique_path(base_dir / file_path.stem, file_path.suffix)
|
|
@@ -134,15 +138,15 @@ class Writer(Reader):
|
|
|
134
138
|
)
|
|
135
139
|
|
|
136
140
|
if isinstance(mode, str) and mode not in self.mode_dtype_dict:
|
|
137
|
-
|
|
141
|
+
msg = f"Can't handle mode '{mode}' (choose one of {self.mode_dtype_dict})"
|
|
142
|
+
raise ValueError(msg)
|
|
138
143
|
|
|
139
|
-
_dtypes = self.mode_dtype_dict[mode
|
|
144
|
+
_dtypes = self.mode_dtype_dict[mode or self.mode_default]
|
|
140
145
|
if isinstance(datatype, str):
|
|
141
146
|
datatype = EnergyDType[datatype]
|
|
142
147
|
if isinstance(datatype, EnergyDType) and datatype not in _dtypes:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
)
|
|
148
|
+
msg = f"Can't handle value '{datatype}' of datatype (choose one of {_dtypes})"
|
|
149
|
+
raise ValueError(msg)
|
|
146
150
|
|
|
147
151
|
if self._modify:
|
|
148
152
|
self._mode = mode
|
|
@@ -222,7 +226,7 @@ class Writer(Reader):
|
|
|
222
226
|
self.h5file.close()
|
|
223
227
|
|
|
224
228
|
def _create_skeleton(self) -> None:
|
|
225
|
-
"""
|
|
229
|
+
"""Initialize the structure of the HDF5 file.
|
|
226
230
|
|
|
227
231
|
HDF5 is hierarchically structured and before writing data, we have to
|
|
228
232
|
setup this structure, i.e. creating the right groups with corresponding
|
|
@@ -278,13 +282,14 @@ class Writer(Reader):
|
|
|
278
282
|
voltage: np.ndarray,
|
|
279
283
|
current: np.ndarray,
|
|
280
284
|
) -> None:
|
|
281
|
-
"""
|
|
285
|
+
"""Write raw data to database.
|
|
282
286
|
|
|
283
287
|
Args:
|
|
284
288
|
----
|
|
285
289
|
timestamp: just start of buffer or whole ndarray
|
|
286
290
|
voltage: ndarray as raw unsigned integers
|
|
287
291
|
current: ndarray as raw unsigned integers
|
|
292
|
+
|
|
288
293
|
"""
|
|
289
294
|
len_new = min(voltage.size, current.size)
|
|
290
295
|
|
|
@@ -296,7 +301,7 @@ class Writer(Reader):
|
|
|
296
301
|
if isinstance(timestamp, np.ndarray):
|
|
297
302
|
len_new = min(len_new, timestamp.size)
|
|
298
303
|
else:
|
|
299
|
-
raise
|
|
304
|
+
raise TypeError("timestamp-data was not usable")
|
|
300
305
|
|
|
301
306
|
len_old = self.ds_voltage.shape[0]
|
|
302
307
|
|
|
@@ -316,7 +321,9 @@ class Writer(Reader):
|
|
|
316
321
|
voltage: np.ndarray,
|
|
317
322
|
current: np.ndarray,
|
|
318
323
|
) -> None:
|
|
319
|
-
"""
|
|
324
|
+
"""Write data (provided in SI / physical unit) to file.
|
|
325
|
+
|
|
326
|
+
This will convert it to raw-data first.
|
|
320
327
|
|
|
321
328
|
SI-value [SI-Unit] = raw-value * gain + offset,
|
|
322
329
|
|
|
@@ -326,6 +333,7 @@ class Writer(Reader):
|
|
|
326
333
|
-> provide start of buffer or whole ndarray
|
|
327
334
|
voltage: ndarray in physical-unit V
|
|
328
335
|
current: ndarray in physical-unit A
|
|
336
|
+
|
|
329
337
|
"""
|
|
330
338
|
timestamp = self._cal.time.si_to_raw(timestamp)
|
|
331
339
|
voltage = self._cal.voltage.si_to_raw(voltage)
|
|
@@ -333,7 +341,7 @@ class Writer(Reader):
|
|
|
333
341
|
self.append_iv_data_raw(timestamp, voltage, current)
|
|
334
342
|
|
|
335
343
|
def _align(self) -> None:
|
|
336
|
-
"""Align datasets with buffer-size of shepherd"""
|
|
344
|
+
"""Align datasets with buffer-size of shepherd."""
|
|
337
345
|
self._refresh_file_stats()
|
|
338
346
|
n_buff = self.ds_voltage.size / self.samples_per_buffer
|
|
339
347
|
size_new = int(math.floor(n_buff) * self.samples_per_buffer)
|
|
@@ -350,21 +358,21 @@ class Writer(Reader):
|
|
|
350
358
|
self.ds_current.resize((size_new,))
|
|
351
359
|
|
|
352
360
|
def __setitem__(self, key: str, item: Any) -> None:
|
|
353
|
-
"""
|
|
361
|
+
"""Conveniently store relevant key-value data (attribute) in H5-structure."""
|
|
354
362
|
self.h5file.attrs.__setitem__(key, item)
|
|
355
363
|
|
|
356
364
|
def store_config(self, data: dict) -> None:
|
|
357
|
-
"""
|
|
365
|
+
"""Get a better self-describing Output-File.
|
|
366
|
+
|
|
358
367
|
TODO: use data-model?
|
|
359
|
-
:param data: from virtual harvester or converter / source
|
|
368
|
+
:param data: from virtual harvester or converter / source.
|
|
360
369
|
"""
|
|
361
370
|
self.h5file["data"].attrs["config"] = yaml.safe_dump(
|
|
362
371
|
data, default_flow_style=False, sort_keys=False
|
|
363
372
|
)
|
|
364
373
|
|
|
365
374
|
def store_hostname(self, name: str) -> None:
|
|
366
|
-
"""Option to distinguish the host, target or data-source
|
|
367
|
-
-> perfect for plotting later
|
|
375
|
+
"""Option to distinguish the host, target or data-source -> perfect for plotting later.
|
|
368
376
|
|
|
369
377
|
:param name: something unique, or "artificial" in case of generated content
|
|
370
378
|
"""
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
|
-
Name:
|
|
3
|
-
Version:
|
|
2
|
+
Name: shepherd_core
|
|
3
|
+
Version: 2024.4.2
|
|
4
4
|
Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
|
|
11
|
-
Project-URL: Source, https://github.com/orgua/shepherd-datalib
|
|
5
|
+
Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
6
|
+
Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
7
|
+
Project-URL: Documentation, https://github.com/orgua/shepherd-datalib/blob/main/README.md
|
|
8
|
+
Project-URL: Issues, https://github.com/orgua/shepherd-datalib/issues
|
|
9
|
+
Project-URL: Source, https://pypi.org/project/shepherd-core/
|
|
12
10
|
Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
|
|
13
11
|
Platform: unix
|
|
14
12
|
Platform: linux
|
|
@@ -35,12 +33,13 @@ Requires-Dist: numpy
|
|
|
35
33
|
Requires-Dist: pyYAML
|
|
36
34
|
Requires-Dist: chromalog
|
|
37
35
|
Requires-Dist: pydantic[email] >2.0.0
|
|
38
|
-
Requires-Dist: tqdm
|
|
39
36
|
Requires-Dist: scipy
|
|
37
|
+
Requires-Dist: tqdm
|
|
40
38
|
Requires-Dist: intelhex
|
|
41
39
|
Requires-Dist: requests
|
|
42
40
|
Requires-Dist: pyelftools
|
|
43
41
|
Requires-Dist: zstandard
|
|
42
|
+
Requires-Dist: typing-extensions
|
|
44
43
|
Provides-Extra: dev
|
|
45
44
|
Requires-Dist: twine ; extra == 'dev'
|
|
46
45
|
Requires-Dist: pre-commit ; extra == 'dev'
|
|
@@ -63,7 +62,7 @@ Requires-Dist: coverage ; extra == 'test'
|
|
|
63
62
|
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
64
63
|
[](https://github.com/astral-sh/ruff)
|
|
65
64
|
|
|
66
|
-
**Documentation**: <https://orgua.github.io/shepherd
|
|
65
|
+
**Main Documentation**: <https://orgua.github.io/shepherd>
|
|
67
66
|
|
|
68
67
|
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
69
68
|
|
|
@@ -71,53 +70,42 @@ Requires-Dist: coverage ; extra == 'test'
|
|
|
71
70
|
|
|
72
71
|
---
|
|
73
72
|
|
|
74
|
-
|
|
73
|
+
`shepherd-core` is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
|
|
75
74
|
|
|
76
|
-
For postprocessing shepherds .h5-files
|
|
75
|
+
For postprocessing shepherds .h5-files usage of [shepherd_data](https://pypi.org/project/shepherd_data) is recommended.
|
|
77
76
|
|
|
78
77
|
## Features
|
|
79
78
|
|
|
80
79
|
- read and write shepherds hdf5-files
|
|
81
|
-
- create, read, write and convert experiments for the testbed
|
|
80
|
+
- create, read, write and convert experiments for the testbed
|
|
81
|
+
- all required data-models are included
|
|
82
82
|
- simulate the virtual source, including virtual harvesters (and virtual converter as a whole)
|
|
83
|
-
- connect and query the testbed via a webclient (TestbedClient)
|
|
83
|
+
- connect and query the testbed via a webclient (TestbedClient in alpha-stage)
|
|
84
84
|
- offline usage defaults to static demo-fixtures loaded from yaml-files in the model-directories
|
|
85
85
|
- work with target-firmwares
|
|
86
86
|
- embed, modify, verify, convert
|
|
87
87
|
- **Note**: working with ELF-files requires external dependencies, see ``Installation``-Chapter
|
|
88
88
|
- decode waveforms (gpio-state & timestamp) to UART
|
|
89
|
-
- create an inventory (
|
|
89
|
+
- create an inventory (for deployed versions of software, hardware)
|
|
90
90
|
|
|
91
|
-
See [
|
|
91
|
+
See [official documentation](https://orgua.github.io/shepherd) or [example scripts](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased in both. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a [trafficbench](https://github.com/orgua/TrafficBench)-experiment that's used to derive the link-matrix of the testbed-nodes.
|
|
92
92
|
|
|
93
|
-
|
|
93
|
+
## Config-Models in Detail
|
|
94
94
|
|
|
95
|
-
|
|
96
|
-
|---------|--------------|--------------------------------------------|
|
|
97
|
-
| Ubuntu | 3.8 - 3.12 | |
|
|
98
|
-
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
99
|
-
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
95
|
+
These pydantic data-models are used throughout all shepherd interfaces. Users can create an experiment, include their own content and feed it to the testbed.
|
|
100
96
|
|
|
101
|
-
|
|
102
|
-
- hex-conversion needs a working and accessible objcopy
|
|
103
|
-
- elf-supports needs
|
|
104
|
-
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
105
|
-
- most elf-features also still utilize hex-conversion
|
|
106
|
-
|
|
107
|
-
### Data-Models in Detail
|
|
108
|
-
|
|
109
|
-
- new orchestration ``/data-models`` with focus on remote shepherd-testbed
|
|
97
|
+
- orchestration ``/data-models`` with focus on remote shepherd-testbed
|
|
110
98
|
- classes of sub-models
|
|
111
99
|
- ``/base``: base-classes, configuration and -functionality for all models
|
|
112
100
|
- ``/testbed``: meta-data representation of all testbed-components
|
|
113
|
-
- ``/content``: reusable meta-data for fw, h5 and vsrc-definitions
|
|
101
|
+
- ``/content``: reusable user-defined meta-data for fw, h5 and vsrc-definitions
|
|
114
102
|
- ``/experiment``: configuration-models including sub-systems
|
|
115
103
|
- ``/task``: digestible configs for shepherd-herd or -sheep
|
|
116
104
|
- behavior controlled by ``ShpModel`` and ``content``-model
|
|
117
105
|
- a basic database is available as fixtures through a ``tb_client``
|
|
118
106
|
- fixtures selectable by name & ID
|
|
119
107
|
- fixtures support inheritance
|
|
120
|
-
- models support
|
|
108
|
+
- the models support
|
|
121
109
|
- auto-completion with neutral / sensible values
|
|
122
110
|
- complex and custom datatypes (i.e. PositiveInt, lists-checks on length)
|
|
123
111
|
- checking of inputs and type-casting
|
|
@@ -130,6 +118,20 @@ Notes:
|
|
|
130
118
|
- exposes no internal paths
|
|
131
119
|
- experiments can be transformed to task-sets (``TestbedTasks.from_xp()``)
|
|
132
120
|
|
|
121
|
+
## Compatibility
|
|
122
|
+
|
|
123
|
+
| OS | PyVersion | Comment |
|
|
124
|
+
|---------|--------------|--------------------------------------------|
|
|
125
|
+
| Ubuntu | 3.8 - 3.12 | |
|
|
126
|
+
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
127
|
+
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
128
|
+
|
|
129
|
+
Notes:
|
|
130
|
+
- hex-conversion needs a working and accessible objcopy
|
|
131
|
+
- elf-supports needs
|
|
132
|
+
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
133
|
+
- most elf-features also still utilize hex-conversion
|
|
134
|
+
|
|
133
135
|
## Installation
|
|
134
136
|
|
|
135
137
|
The Library is available via PyPI and can be installed with
|
|
@@ -166,3 +168,17 @@ For creating an inventory of the host-system you should install
|
|
|
166
168
|
```shell
|
|
167
169
|
pip install shepherd-core[inventory]
|
|
168
170
|
```
|
|
171
|
+
|
|
172
|
+
## Unittests
|
|
173
|
+
|
|
174
|
+
To run the testbench, follow these steps:
|
|
175
|
+
|
|
176
|
+
1. Navigate your host-shell into the package-folder and
|
|
177
|
+
2. install dependencies
|
|
178
|
+
3. run the testbench (~ 320 tests):
|
|
179
|
+
|
|
180
|
+
```Shell
|
|
181
|
+
cd shepherd-datalib/shepherd_core
|
|
182
|
+
pip3 install ./[tests]
|
|
183
|
+
pytest
|
|
184
|
+
```
|