shepherd-core 2023.11.1__tar.gz → 2024.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/PKG-INFO +49 -37
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/README.md +42 -25
- shepherd_core-2024.4.1/pyproject.toml +112 -0
- shepherd_core-2024.4.1/setup.cfg +4 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/__init__.py +3 -2
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/calibration.py +3 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/content.py +13 -11
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/shepherd.py +4 -6
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/energy_environment.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/virtual_harvester.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/virtual_source.py +31 -53
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/experiment.py +13 -18
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/observer_features.py +9 -15
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/target_config.py +4 -11
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/__init__.py +2 -6
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/emulation.py +7 -14
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/firmware_mod.py +1 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/harvest.py +1 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/observer_tasks.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/testbed_tasks.py +7 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/cape.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/gpio.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/mcu.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/observer.py +7 -23
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/target.py +1 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/testbed.py +2 -4
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/decoder_waveform/uart.py +9 -26
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/__init__.py +1 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/converter.py +4 -12
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/patcher.py +4 -12
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/validation.py +1 -2
- shepherd_core-2024.4.1/shepherd_core/inventory/__init__.py +105 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/system.py +10 -5
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/logger.py +1 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/reader.py +45 -57
- shepherd_core-2024.4.1/shepherd_core/testbed_client/cache_path.py +15 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/client.py +7 -19
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/fixtures.py +8 -15
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/user_model.py +7 -7
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_converter_model.py +5 -15
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_harvester_model.py +2 -3
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_source_model.py +3 -6
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/writer.py +16 -24
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/PKG-INFO +50 -38
- shepherd_core-2024.4.1/shepherd_core.egg-info/SOURCES.txt +72 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/requires.txt +1 -4
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/top_level.txt +0 -1
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_cal_hw.py +2 -6
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_examples.py +6 -5
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_writer.py +4 -4
- shepherd_core-2023.11.1/pyproject.toml +0 -10
- shepherd_core-2023.11.1/setup.cfg +0 -89
- shepherd_core-2023.11.1/shepherd_core/data_models/content/_external_fixtures.yaml +0 -394
- shepherd_core-2023.11.1/shepherd_core/data_models/content/energy_environment_fixture.yaml +0 -50
- shepherd_core-2023.11.1/shepherd_core/data_models/content/virtual_harvester_fixture.yaml +0 -159
- shepherd_core-2023.11.1/shepherd_core/data_models/content/virtual_source_fixture.yaml +0 -229
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/cape_fixture.yaml +0 -94
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/gpio_fixture.yaml +0 -166
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/mcu_fixture.yaml +0 -19
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/observer_fixture.yaml +0 -220
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/target_fixture.yaml +0 -137
- shepherd_core-2023.11.1/shepherd_core/data_models/testbed/testbed_fixture.yaml +0 -25
- shepherd_core-2023.11.1/shepherd_core/inventory/__init__.py +0 -61
- shepherd_core-2023.11.1/shepherd_core.egg-info/SOURCES.txt +0 -126
- shepherd_core-2023.11.1/tests/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/conftest.py +0 -64
- shepherd_core-2023.11.1/tests/data_models/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/data_models/conftest.py +0 -14
- shepherd_core-2023.11.1/tests/data_models/example_cal_data.yaml +0 -31
- shepherd_core-2023.11.1/tests/data_models/example_cal_data_faulty.yaml +0 -29
- shepherd_core-2023.11.1/tests/data_models/example_cal_meas.yaml +0 -178
- shepherd_core-2023.11.1/tests/data_models/example_cal_meas_faulty1.yaml +0 -142
- shepherd_core-2023.11.1/tests/data_models/example_cal_meas_faulty2.yaml +0 -136
- shepherd_core-2023.11.1/tests/data_models/example_config_emulator.yaml +0 -41
- shepherd_core-2023.11.1/tests/data_models/example_config_experiment.yaml +0 -16
- shepherd_core-2023.11.1/tests/data_models/example_config_experiment_alternative.yaml +0 -14
- shepherd_core-2023.11.1/tests/data_models/example_config_harvester.yaml +0 -15
- shepherd_core-2023.11.1/tests/data_models/example_config_testbed.yaml +0 -26
- shepherd_core-2023.11.1/tests/data_models/example_config_virtsource.yaml +0 -78
- shepherd_core-2023.11.1/tests/data_models/test_base_models.py +0 -205
- shepherd_core-2023.11.1/tests/data_models/test_content_fixtures.py +0 -41
- shepherd_core-2023.11.1/tests/data_models/test_content_models.py +0 -288
- shepherd_core-2023.11.1/tests/data_models/test_examples.py +0 -48
- shepherd_core-2023.11.1/tests/data_models/test_experiment_models.py +0 -279
- shepherd_core-2023.11.1/tests/data_models/test_task_generation.py +0 -52
- shepherd_core-2023.11.1/tests/data_models/test_task_models.py +0 -131
- shepherd_core-2023.11.1/tests/data_models/test_testbed_fixtures.py +0 -47
- shepherd_core-2023.11.1/tests/data_models/test_testbed_models.py +0 -187
- shepherd_core-2023.11.1/tests/decoder_waveform/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/decoder_waveform/test_decoder.py +0 -34
- shepherd_core-2023.11.1/tests/fw_tools/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/fw_tools/conftest.py +0 -5
- shepherd_core-2023.11.1/tests/fw_tools/test_converter.py +0 -76
- shepherd_core-2023.11.1/tests/fw_tools/test_patcher.py +0 -66
- shepherd_core-2023.11.1/tests/fw_tools/test_validation.py +0 -56
- shepherd_core-2023.11.1/tests/inventory/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/inventory/test_inventory.py +0 -22
- shepherd_core-2023.11.1/tests/testbed_client/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/vsource/__init__.py +0 -0
- shepherd_core-2023.11.1/tests/vsource/conftest.py +0 -51
- shepherd_core-2023.11.1/tests/vsource/test_converter.py +0 -165
- shepherd_core-2023.11.1/tests/vsource/test_harvester.py +0 -79
- shepherd_core-2023.11.1/tests/vsource/test_z.py +0 -5
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/calibration_hw_def.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/commons.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/cal_measurement.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/timezone.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/wrapper.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/firmware.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/firmware_datatype.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/doc_virtual_source.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/programming.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/decoder_waveform/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/converter_elf.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/python.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/target.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/__init__.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/dependency_links.txt +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/zip-safe +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_logger.py +0 -0
- {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_reader.py +0 -0
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: shepherd_core
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2024.4.1
|
|
4
4
|
Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
|
|
11
|
-
Project-URL: Source, https://github.com/orgua/shepherd-datalib
|
|
5
|
+
Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
6
|
+
Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
|
|
7
|
+
Project-URL: Documentation, https://github.com/orgua/shepherd-datalib/blob/main/README.md
|
|
8
|
+
Project-URL: Issues, https://github.com/orgua/shepherd-datalib/issues
|
|
9
|
+
Project-URL: Source, https://pypi.org/project/shepherd-core/
|
|
12
10
|
Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
|
|
13
11
|
Platform: unix
|
|
14
12
|
Platform: linux
|
|
@@ -35,8 +33,8 @@ Requires-Dist: numpy
|
|
|
35
33
|
Requires-Dist: pyYAML
|
|
36
34
|
Requires-Dist: chromalog
|
|
37
35
|
Requires-Dist: pydantic[email]>2.0.0
|
|
38
|
-
Requires-Dist: tqdm
|
|
39
36
|
Requires-Dist: scipy
|
|
37
|
+
Requires-Dist: tqdm
|
|
40
38
|
Requires-Dist: intelhex
|
|
41
39
|
Requires-Dist: requests
|
|
42
40
|
Requires-Dist: pyelftools
|
|
@@ -46,9 +44,6 @@ Requires-Dist: pwntools-elf-only; extra == "elf"
|
|
|
46
44
|
Provides-Extra: inventory
|
|
47
45
|
Requires-Dist: psutil; extra == "inventory"
|
|
48
46
|
Provides-Extra: dev
|
|
49
|
-
Requires-Dist: black; extra == "dev"
|
|
50
|
-
Requires-Dist: pylint; extra == "dev"
|
|
51
|
-
Requires-Dist: flake8; extra == "dev"
|
|
52
47
|
Requires-Dist: twine; extra == "dev"
|
|
53
48
|
Requires-Dist: pre-commit; extra == "dev"
|
|
54
49
|
Requires-Dist: pyright; extra == "dev"
|
|
@@ -64,9 +59,9 @@ Requires-Dist: coverage; extra == "test"
|
|
|
64
59
|
[](https://pypi.org/project/shepherd_core)
|
|
65
60
|
[](https://pypi.python.org/pypi/shepherd-core)
|
|
66
61
|
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
67
|
-
[](https://github.com/astral-sh/ruff)
|
|
68
63
|
|
|
69
|
-
**Documentation**: <https://orgua.github.io/shepherd
|
|
64
|
+
**Main Documentation**: <https://orgua.github.io/shepherd>
|
|
70
65
|
|
|
71
66
|
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
72
67
|
|
|
@@ -74,53 +69,42 @@ Requires-Dist: coverage; extra == "test"
|
|
|
74
69
|
|
|
75
70
|
---
|
|
76
71
|
|
|
77
|
-
|
|
72
|
+
`shepherd-core` is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
|
|
78
73
|
|
|
79
|
-
For postprocessing shepherds .h5-files
|
|
74
|
+
For postprocessing shepherds .h5-files usage of [shepherd_data](https://pypi.org/project/shepherd_data) is recommended.
|
|
80
75
|
|
|
81
76
|
## Features
|
|
82
77
|
|
|
83
78
|
- read and write shepherds hdf5-files
|
|
84
|
-
- create, read, write and convert experiments for the testbed
|
|
79
|
+
- create, read, write and convert experiments for the testbed
|
|
80
|
+
- all required data-models are included
|
|
85
81
|
- simulate the virtual source, including virtual harvesters (and virtual converter as a whole)
|
|
86
|
-
- connect and query the testbed via a webclient (TestbedClient)
|
|
82
|
+
- connect and query the testbed via a webclient (TestbedClient in alpha-stage)
|
|
87
83
|
- offline usage defaults to static demo-fixtures loaded from yaml-files in the model-directories
|
|
88
84
|
- work with target-firmwares
|
|
89
85
|
- embed, modify, verify, convert
|
|
90
86
|
- **Note**: working with ELF-files requires external dependencies, see ``Installation``-Chapter
|
|
91
87
|
- decode waveforms (gpio-state & timestamp) to UART
|
|
92
|
-
- create an inventory (
|
|
88
|
+
- create an inventory (for deployed versions of software, hardware)
|
|
93
89
|
|
|
94
|
-
See [
|
|
90
|
+
See [official documentation](https://orgua.github.io/shepherd) or [example scripts](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased in both. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a [trafficbench](https://github.com/orgua/TrafficBench)-experiment that's used to derive the link-matrix of the testbed-nodes.
|
|
95
91
|
|
|
96
|
-
|
|
92
|
+
## Config-Models in Detail
|
|
97
93
|
|
|
98
|
-
|
|
99
|
-
|---------|--------------|--------------------------------------------|
|
|
100
|
-
| Ubuntu | 3.8 - 3.12 | |
|
|
101
|
-
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
102
|
-
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
94
|
+
These pydantic data-models are used throughout all shepherd interfaces. Users can create an experiment, include their own content and feed it to the testbed.
|
|
103
95
|
|
|
104
|
-
|
|
105
|
-
- hex-conversion needs a working and accessible objcopy
|
|
106
|
-
- elf-supports needs
|
|
107
|
-
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
108
|
-
- most elf-features also still utilize hex-conversion
|
|
109
|
-
|
|
110
|
-
### Data-Models in Detail
|
|
111
|
-
|
|
112
|
-
- new orchestration ``/data-models`` with focus on remote shepherd-testbed
|
|
96
|
+
- orchestration ``/data-models`` with focus on remote shepherd-testbed
|
|
113
97
|
- classes of sub-models
|
|
114
98
|
- ``/base``: base-classes, configuration and -functionality for all models
|
|
115
99
|
- ``/testbed``: meta-data representation of all testbed-components
|
|
116
|
-
- ``/content``: reusable meta-data for fw, h5 and vsrc-definitions
|
|
100
|
+
- ``/content``: reusable user-defined meta-data for fw, h5 and vsrc-definitions
|
|
117
101
|
- ``/experiment``: configuration-models including sub-systems
|
|
118
102
|
- ``/task``: digestible configs for shepherd-herd or -sheep
|
|
119
103
|
- behavior controlled by ``ShpModel`` and ``content``-model
|
|
120
104
|
- a basic database is available as fixtures through a ``tb_client``
|
|
121
105
|
- fixtures selectable by name & ID
|
|
122
106
|
- fixtures support inheritance
|
|
123
|
-
- models support
|
|
107
|
+
- the models support
|
|
124
108
|
- auto-completion with neutral / sensible values
|
|
125
109
|
- complex and custom datatypes (i.e. PositiveInt, lists-checks on length)
|
|
126
110
|
- checking of inputs and type-casting
|
|
@@ -133,6 +117,20 @@ Notes:
|
|
|
133
117
|
- exposes no internal paths
|
|
134
118
|
- experiments can be transformed to task-sets (``TestbedTasks.from_xp()``)
|
|
135
119
|
|
|
120
|
+
## Compatibility
|
|
121
|
+
|
|
122
|
+
| OS | PyVersion | Comment |
|
|
123
|
+
|---------|--------------|--------------------------------------------|
|
|
124
|
+
| Ubuntu | 3.8 - 3.12 | |
|
|
125
|
+
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
126
|
+
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
127
|
+
|
|
128
|
+
Notes:
|
|
129
|
+
- hex-conversion needs a working and accessible objcopy
|
|
130
|
+
- elf-supports needs
|
|
131
|
+
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
132
|
+
- most elf-features also still utilize hex-conversion
|
|
133
|
+
|
|
136
134
|
## Installation
|
|
137
135
|
|
|
138
136
|
The Library is available via PyPI and can be installed with
|
|
@@ -169,3 +167,17 @@ For creating an inventory of the host-system you should install
|
|
|
169
167
|
```shell
|
|
170
168
|
pip install shepherd-core[inventory]
|
|
171
169
|
```
|
|
170
|
+
|
|
171
|
+
## Unittests
|
|
172
|
+
|
|
173
|
+
To run the testbench, follow these steps:
|
|
174
|
+
|
|
175
|
+
1. Navigate your host-shell into the package-folder and
|
|
176
|
+
2. install dependencies
|
|
177
|
+
3. run the testbench (~ 320 tests):
|
|
178
|
+
|
|
179
|
+
```Shell
|
|
180
|
+
cd shepherd-datalib/shepherd_core
|
|
181
|
+
pip3 install ./[tests]
|
|
182
|
+
pytest
|
|
183
|
+
```
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
[](https://pypi.org/project/shepherd_core)
|
|
4
4
|
[](https://pypi.python.org/pypi/shepherd-core)
|
|
5
5
|
[](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
|
|
6
|
-
[](https://github.com/astral-sh/ruff)
|
|
7
7
|
|
|
8
|
-
**Documentation**: <https://orgua.github.io/shepherd
|
|
8
|
+
**Main Documentation**: <https://orgua.github.io/shepherd>
|
|
9
9
|
|
|
10
10
|
**Source Code**: <https://github.com/orgua/shepherd-datalib>
|
|
11
11
|
|
|
@@ -13,53 +13,42 @@
|
|
|
13
13
|
|
|
14
14
|
---
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
`shepherd-core` is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
|
|
17
17
|
|
|
18
|
-
For postprocessing shepherds .h5-files
|
|
18
|
+
For postprocessing shepherds .h5-files usage of [shepherd_data](https://pypi.org/project/shepherd_data) is recommended.
|
|
19
19
|
|
|
20
20
|
## Features
|
|
21
21
|
|
|
22
22
|
- read and write shepherds hdf5-files
|
|
23
|
-
- create, read, write and convert experiments for the testbed
|
|
23
|
+
- create, read, write and convert experiments for the testbed
|
|
24
|
+
- all required data-models are included
|
|
24
25
|
- simulate the virtual source, including virtual harvesters (and virtual converter as a whole)
|
|
25
|
-
- connect and query the testbed via a webclient (TestbedClient)
|
|
26
|
+
- connect and query the testbed via a webclient (TestbedClient in alpha-stage)
|
|
26
27
|
- offline usage defaults to static demo-fixtures loaded from yaml-files in the model-directories
|
|
27
28
|
- work with target-firmwares
|
|
28
29
|
- embed, modify, verify, convert
|
|
29
30
|
- **Note**: working with ELF-files requires external dependencies, see ``Installation``-Chapter
|
|
30
31
|
- decode waveforms (gpio-state & timestamp) to UART
|
|
31
|
-
- create an inventory (
|
|
32
|
+
- create an inventory (for deployed versions of software, hardware)
|
|
32
33
|
|
|
33
|
-
See [
|
|
34
|
+
See [official documentation](https://orgua.github.io/shepherd) or [example scripts](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased in both. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a [trafficbench](https://github.com/orgua/TrafficBench)-experiment that's used to derive the link-matrix of the testbed-nodes.
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
## Config-Models in Detail
|
|
36
37
|
|
|
37
|
-
|
|
38
|
-
|---------|--------------|--------------------------------------------|
|
|
39
|
-
| Ubuntu | 3.8 - 3.12 | |
|
|
40
|
-
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
41
|
-
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
42
|
-
|
|
43
|
-
Notes:
|
|
44
|
-
- hex-conversion needs a working and accessible objcopy
|
|
45
|
-
- elf-supports needs
|
|
46
|
-
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
47
|
-
- most elf-features also still utilize hex-conversion
|
|
48
|
-
|
|
49
|
-
### Data-Models in Detail
|
|
38
|
+
These pydantic data-models are used throughout all shepherd interfaces. Users can create an experiment, include their own content and feed it to the testbed.
|
|
50
39
|
|
|
51
|
-
-
|
|
40
|
+
- orchestration ``/data-models`` with focus on remote shepherd-testbed
|
|
52
41
|
- classes of sub-models
|
|
53
42
|
- ``/base``: base-classes, configuration and -functionality for all models
|
|
54
43
|
- ``/testbed``: meta-data representation of all testbed-components
|
|
55
|
-
- ``/content``: reusable meta-data for fw, h5 and vsrc-definitions
|
|
44
|
+
- ``/content``: reusable user-defined meta-data for fw, h5 and vsrc-definitions
|
|
56
45
|
- ``/experiment``: configuration-models including sub-systems
|
|
57
46
|
- ``/task``: digestible configs for shepherd-herd or -sheep
|
|
58
47
|
- behavior controlled by ``ShpModel`` and ``content``-model
|
|
59
48
|
- a basic database is available as fixtures through a ``tb_client``
|
|
60
49
|
- fixtures selectable by name & ID
|
|
61
50
|
- fixtures support inheritance
|
|
62
|
-
- models support
|
|
51
|
+
- the models support
|
|
63
52
|
- auto-completion with neutral / sensible values
|
|
64
53
|
- complex and custom datatypes (i.e. PositiveInt, lists-checks on length)
|
|
65
54
|
- checking of inputs and type-casting
|
|
@@ -72,6 +61,20 @@ Notes:
|
|
|
72
61
|
- exposes no internal paths
|
|
73
62
|
- experiments can be transformed to task-sets (``TestbedTasks.from_xp()``)
|
|
74
63
|
|
|
64
|
+
## Compatibility
|
|
65
|
+
|
|
66
|
+
| OS | PyVersion | Comment |
|
|
67
|
+
|---------|--------------|--------------------------------------------|
|
|
68
|
+
| Ubuntu | 3.8 - 3.12 | |
|
|
69
|
+
| Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
|
|
70
|
+
| MacOS | 3.8 - 3.12 | hex-conversion missing |
|
|
71
|
+
|
|
72
|
+
Notes:
|
|
73
|
+
- hex-conversion needs a working and accessible objcopy
|
|
74
|
+
- elf-supports needs
|
|
75
|
+
- ``shepherd-core[elf]`` installs ``pwntools-elf-only``
|
|
76
|
+
- most elf-features also still utilize hex-conversion
|
|
77
|
+
|
|
75
78
|
## Installation
|
|
76
79
|
|
|
77
80
|
The Library is available via PyPI and can be installed with
|
|
@@ -108,3 +111,17 @@ For creating an inventory of the host-system you should install
|
|
|
108
111
|
```shell
|
|
109
112
|
pip install shepherd-core[inventory]
|
|
110
113
|
```
|
|
114
|
+
|
|
115
|
+
## Unittests
|
|
116
|
+
|
|
117
|
+
To run the testbench, follow these steps:
|
|
118
|
+
|
|
119
|
+
1. Navigate your host-shell into the package-folder and
|
|
120
|
+
2. install dependencies
|
|
121
|
+
3. run the testbench (~ 320 tests):
|
|
122
|
+
|
|
123
|
+
```Shell
|
|
124
|
+
cd shepherd-datalib/shepherd_core
|
|
125
|
+
pip3 install ./[tests]
|
|
126
|
+
pytest
|
|
127
|
+
```
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "shepherd_core"
|
|
3
|
+
description = "Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed"
|
|
4
|
+
keywords = ["testbed", "beaglebone", "pru", "batteryless", "energyharvesting", "solar"]
|
|
5
|
+
|
|
6
|
+
authors = [{name = "Ingmar Splitt", email = "ingmar.splitt@tu-dresden.de"},]
|
|
7
|
+
maintainers = [{name = "Ingmar Splitt", email = "ingmar.splitt@tu-dresden.de"},]
|
|
8
|
+
|
|
9
|
+
readme = {file = "README.md", content-type = "text/markdown"}
|
|
10
|
+
license = {file = "LICENSE"}
|
|
11
|
+
dynamic = ["version"]
|
|
12
|
+
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Development Status :: 5 - Production/Stable",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Intended Audience :: Information Technology",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3.8",
|
|
19
|
+
"Programming Language :: Python :: 3.9",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"License :: OSI Approved :: MIT License",
|
|
24
|
+
"Operating System :: OS Independent",
|
|
25
|
+
"Natural Language :: English",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
requires-python = ">=3.8"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"h5py",
|
|
31
|
+
"numpy",
|
|
32
|
+
"pyYAML",
|
|
33
|
+
"chromalog",
|
|
34
|
+
"pydantic[email]>2.0.0",
|
|
35
|
+
"scipy",
|
|
36
|
+
"tqdm",
|
|
37
|
+
"intelhex",
|
|
38
|
+
"requests",
|
|
39
|
+
"pyelftools",
|
|
40
|
+
"zstandard",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[project.optional-dependencies]
|
|
44
|
+
elf = [
|
|
45
|
+
"pwntools-elf-only",
|
|
46
|
+
# TODO: still separate because of missing windows-compat
|
|
47
|
+
# modified package for reading ELF-files with fw_tools
|
|
48
|
+
# original can be problematic to install on windows & beaglebone
|
|
49
|
+
# probably also needed: https://docs.pwntools.com/en/stable/install.html + binutils-$ARCH
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
inventory = [
|
|
53
|
+
"psutil",
|
|
54
|
+
]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
dev = [
|
|
58
|
+
"twine",
|
|
59
|
+
"pre-commit",
|
|
60
|
+
"pyright",
|
|
61
|
+
"ruff",
|
|
62
|
+
"mypy",
|
|
63
|
+
"types-PyYAML",
|
|
64
|
+
]
|
|
65
|
+
|
|
66
|
+
test = [
|
|
67
|
+
"pytest",
|
|
68
|
+
"coverage",
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
[project.urls]
|
|
72
|
+
Documentation = "https://github.com/orgua/shepherd-datalib/blob/main/README.md"
|
|
73
|
+
Issues = "https://github.com/orgua/shepherd-datalib/issues"
|
|
74
|
+
Source = "https://pypi.org/project/shepherd-core/"
|
|
75
|
+
|
|
76
|
+
[build-system]
|
|
77
|
+
requires = ["setuptools"]
|
|
78
|
+
build-backend = "setuptools.build_meta"
|
|
79
|
+
|
|
80
|
+
[tool.setuptools]
|
|
81
|
+
platforms = ["unix", "linux", "osx", "cygwin", "win32", "win64"]
|
|
82
|
+
zip-safe = true
|
|
83
|
+
include-package-data = true
|
|
84
|
+
|
|
85
|
+
[tool.setuptools.package-dir]
|
|
86
|
+
shepherd_core = "shepherd_core"
|
|
87
|
+
|
|
88
|
+
[tool.setuptools.package-data]
|
|
89
|
+
shepherd_core = [
|
|
90
|
+
"README.md",
|
|
91
|
+
"*.yaml",
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
[tool.setuptools.dynamic]
|
|
95
|
+
version = {attr = "shepherd_core.__version__"}
|
|
96
|
+
|
|
97
|
+
[tool.pytest.ini_options]
|
|
98
|
+
markers = [
|
|
99
|
+
"converter: automatically added when no objcopy is found (on ubuntu install build-essential)",
|
|
100
|
+
"elf: automatically added when no elf-support is found (on windows or missing shepherd-core[elf])"
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
addopts = "-vvv --stepwise" # opts: verbose result for each tests
|
|
104
|
+
# TODO: add something like "--cov --cov-report html --cov-report term-missing --cov-fail-under 95"
|
|
105
|
+
|
|
106
|
+
[tool.coverage.run]
|
|
107
|
+
source = ["shepherd_core"]
|
|
108
|
+
omit = ["*/shepherd_data/*"]
|
|
109
|
+
|
|
110
|
+
[tool.mypy]
|
|
111
|
+
python_version = 3.8
|
|
112
|
+
ignore_missing_imports = true
|
|
@@ -4,6 +4,7 @@ Provides classes for storing and retrieving sampled IV data to/from
|
|
|
4
4
|
HDF5 files.
|
|
5
5
|
|
|
6
6
|
"""
|
|
7
|
+
|
|
7
8
|
from .data_models.base.calibration import Calc_t
|
|
8
9
|
from .data_models.base.calibration import CalibrationCape
|
|
9
10
|
from .data_models.base.calibration import CalibrationEmulator
|
|
@@ -12,7 +13,7 @@ from .data_models.base.calibration import CalibrationPair
|
|
|
12
13
|
from .data_models.base.calibration import CalibrationSeries
|
|
13
14
|
from .data_models.base.timezone import local_now
|
|
14
15
|
from .data_models.base.timezone import local_tz
|
|
15
|
-
from .data_models.task import Compression
|
|
16
|
+
from .data_models.task.emulation import Compression
|
|
16
17
|
from .inventory import Inventory
|
|
17
18
|
from .logger import get_verbose_level
|
|
18
19
|
from .logger import increase_verbose_level
|
|
@@ -22,7 +23,7 @@ from .testbed_client.client import TestbedClient
|
|
|
22
23
|
from .testbed_client.client import tb_client
|
|
23
24
|
from .writer import Writer
|
|
24
25
|
|
|
25
|
-
__version__ = "
|
|
26
|
+
__version__ = "2024.04.1"
|
|
26
27
|
|
|
27
28
|
__all__ = [
|
|
28
29
|
"Reader",
|
{shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/calibration.py
RENAMED
|
@@ -165,7 +165,7 @@ class CapeData(ShpModel):
|
|
|
165
165
|
`See<https://github.com/beagleboard/beaglebone-black/wiki/System-Reference-Manual#824_EEPROM_Data_Format>`_
|
|
166
166
|
"""
|
|
167
167
|
|
|
168
|
-
header: conbytes(max_length=4) = b"\
|
|
168
|
+
header: conbytes(max_length=4) = b"\xaa\x55\x33\xee"
|
|
169
169
|
eeprom_revision: constr(max_length=2) = "A2"
|
|
170
170
|
board_name: constr(max_length=32) = "BeagleBone SHEPHERD2 Cape"
|
|
171
171
|
version: constr(max_length=4) = "24B0"
|
|
@@ -207,6 +207,7 @@ class CalibrationCape(ShpModel):
|
|
|
207
207
|
cape: data can be supplied
|
|
208
208
|
Returns:
|
|
209
209
|
CalibrationCape object with extracted calibration data.
|
|
210
|
+
|
|
210
211
|
"""
|
|
211
212
|
dv = cls().model_dump(include={"harvester", "emulator"})
|
|
212
213
|
lw = list(dict_generator(dv))
|
|
@@ -225,6 +226,7 @@ class CalibrationCape(ShpModel):
|
|
|
225
226
|
Returns
|
|
226
227
|
-------
|
|
227
228
|
Byte string representation of calibration values.
|
|
229
|
+
|
|
228
230
|
"""
|
|
229
231
|
lw = list(dict_generator(self.model_dump(include={"harvester", "emulator"})))
|
|
230
232
|
values = [walk[-1] for walk in lw]
|
{shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/content.py
RENAMED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
from datetime import datetime
|
|
3
3
|
from typing import Optional
|
|
4
|
+
from typing import Union
|
|
5
|
+
from uuid import uuid4
|
|
4
6
|
|
|
7
|
+
from pydantic import UUID4
|
|
5
8
|
from pydantic import Field
|
|
6
9
|
from pydantic import StringConstraints
|
|
7
10
|
from pydantic import model_validator
|
|
@@ -14,34 +17,34 @@ from .timezone import local_now
|
|
|
14
17
|
# constr -> to_lower=True, max_length=16, regex=r"^[\w]+$"
|
|
15
18
|
# ⤷ Regex = AlphaNum
|
|
16
19
|
IdInt = Annotated[int, Field(ge=0, lt=2**128)]
|
|
17
|
-
NameStr = Annotated[
|
|
18
|
-
str, StringConstraints(max_length=32, pattern=r'^[^<>:;,?"*|\/\\]+$')
|
|
19
|
-
]
|
|
20
|
+
NameStr = Annotated[str, StringConstraints(max_length=32, pattern=r"^[^<>:;,?\"\*|\/\\]+$")]
|
|
20
21
|
# ⤷ Regex = FileSystem-Compatible ASCII
|
|
21
22
|
SafeStr = Annotated[str, StringConstraints(pattern=r"^[ -~]+$")]
|
|
22
23
|
# ⤷ Regex = All Printable ASCII-Characters with Space
|
|
23
24
|
|
|
24
25
|
|
|
25
26
|
def id_default() -> int:
|
|
27
|
+
# note: IdInt has space for 128 bit, so 128/4 = 32 hex-chars
|
|
26
28
|
time_stamp = str(local_now()).encode("utf-8")
|
|
27
|
-
time_hash = hashlib.sha3_224(time_stamp).hexdigest()[-
|
|
29
|
+
time_hash = hashlib.sha3_224(time_stamp).hexdigest()[-32:]
|
|
28
30
|
return int(time_hash, 16)
|
|
29
31
|
|
|
30
32
|
|
|
31
33
|
class ContentModel(ShpModel):
|
|
32
34
|
# General Properties
|
|
33
|
-
id:
|
|
35
|
+
# id: UUID4 = Field( # TODO: db-migration - temp fix for documentation
|
|
36
|
+
id: Union[UUID4, int] = Field(
|
|
34
37
|
description="Unique ID",
|
|
35
|
-
default_factory=
|
|
38
|
+
default_factory=uuid4,
|
|
36
39
|
)
|
|
37
40
|
name: NameStr
|
|
38
|
-
description: Annotated[
|
|
39
|
-
Optional[SafeStr], Field(description="Required when public")
|
|
40
|
-
] = None
|
|
41
|
+
description: Annotated[Optional[SafeStr], Field(description="Required when public")] = None
|
|
41
42
|
comment: Optional[SafeStr] = None
|
|
42
43
|
created: datetime = Field(default_factory=datetime.now)
|
|
44
|
+
updated_last: datetime = Field(default_factory=datetime.now)
|
|
43
45
|
|
|
44
46
|
# Ownership & Access
|
|
47
|
+
# TODO: remove owner & group, only needed for DB
|
|
45
48
|
owner: NameStr
|
|
46
49
|
group: Annotated[NameStr, Field(description="University or Subgroup")]
|
|
47
50
|
visible2group: bool = False
|
|
@@ -57,7 +60,6 @@ class ContentModel(ShpModel):
|
|
|
57
60
|
is_visible = self.visible2group or self.visible2all
|
|
58
61
|
if is_visible and self.description is None:
|
|
59
62
|
raise ValueError(
|
|
60
|
-
"Public instances require a description "
|
|
61
|
-
"(check visible2*- and description-field)"
|
|
63
|
+
"Public instances require a description (check visible2*- and description-field)"
|
|
62
64
|
)
|
|
63
65
|
return self
|
{shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/shepherd.py
RENAMED
|
@@ -7,6 +7,7 @@ from typing import Any
|
|
|
7
7
|
from typing import Generator
|
|
8
8
|
from typing import Optional
|
|
9
9
|
from typing import Union
|
|
10
|
+
from uuid import UUID
|
|
10
11
|
|
|
11
12
|
import yaml
|
|
12
13
|
from pydantic import BaseModel
|
|
@@ -27,9 +28,7 @@ def path2str(
|
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
def time2int(dumper: Dumper, data: timedelta) -> ScalarNode:
|
|
30
|
-
return dumper.represent_scalar(
|
|
31
|
-
"tag:yaml.org,2002:int", str(int(data.total_seconds()))
|
|
32
|
-
)
|
|
31
|
+
return dumper.represent_scalar("tag:yaml.org,2002:int", str(int(data.total_seconds())))
|
|
33
32
|
|
|
34
33
|
|
|
35
34
|
def generic2str(dumper: Dumper, data: Any) -> ScalarNode:
|
|
@@ -41,6 +40,7 @@ yaml.add_representer(pathlib.WindowsPath, path2str, SafeDumper)
|
|
|
41
40
|
yaml.add_representer(pathlib.Path, path2str, SafeDumper)
|
|
42
41
|
yaml.add_representer(timedelta, time2int, SafeDumper)
|
|
43
42
|
yaml.add_representer(IPv4Address, generic2str, SafeDumper)
|
|
43
|
+
yaml.add_representer(UUID, generic2str, SafeDumper)
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
class ShpModel(BaseModel):
|
|
@@ -109,9 +109,7 @@ class ShpModel(BaseModel):
|
|
|
109
109
|
def schema_to_file(cls, path: Union[str, Path]) -> None:
|
|
110
110
|
"""Store schema to yaml (for frontend-generators)"""
|
|
111
111
|
model_dict = cls.model_json_schema()
|
|
112
|
-
model_yaml = yaml.safe_dump(
|
|
113
|
-
model_dict, default_flow_style=False, sort_keys=False
|
|
114
|
-
)
|
|
112
|
+
model_yaml = yaml.safe_dump(model_dict, default_flow_style=False, sort_keys=False)
|
|
115
113
|
with Path(path).resolve().with_suffix(".yaml").open("w") as f:
|
|
116
114
|
f.write(model_yaml)
|
|
117
115
|
|
|
@@ -76,7 +76,7 @@ class VirtualHarvesterConfig(ContentModel, title="Config for the Harvester"):
|
|
|
76
76
|
logger.debug("VHrv-Inheritances: %s", chain)
|
|
77
77
|
|
|
78
78
|
# post corrections -> should be in separate validator
|
|
79
|
-
cal = CalibrationHarvester() #
|
|
79
|
+
cal = CalibrationHarvester() # TODO: as argument?
|
|
80
80
|
c_limit = values.get("current_limit_uA", 50_000) # cls.current_limit_uA)
|
|
81
81
|
values["current_limit_uA"] = max(10**6 * cal.adc_C_Hrv.raw_to_si(4), c_limit)
|
|
82
82
|
|