shepherd-core 2023.11.1__tar.gz → 2024.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/PKG-INFO +49 -37
  2. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/README.md +42 -25
  3. shepherd_core-2024.4.1/pyproject.toml +112 -0
  4. shepherd_core-2024.4.1/setup.cfg +4 -0
  5. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/__init__.py +3 -2
  6. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/calibration.py +3 -1
  7. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/content.py +13 -11
  8. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/shepherd.py +4 -6
  9. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/energy_environment.py +1 -1
  10. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/virtual_harvester.py +1 -1
  11. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/virtual_source.py +31 -53
  12. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/experiment.py +13 -18
  13. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/observer_features.py +9 -15
  14. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/target_config.py +4 -11
  15. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/__init__.py +2 -6
  16. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/emulation.py +7 -14
  17. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/firmware_mod.py +1 -3
  18. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/harvest.py +1 -3
  19. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/observer_tasks.py +1 -1
  20. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/testbed_tasks.py +7 -3
  21. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/cape.py +1 -1
  22. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/gpio.py +1 -1
  23. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/mcu.py +1 -1
  24. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/observer.py +7 -23
  25. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/target.py +1 -1
  26. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/testbed.py +2 -4
  27. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/decoder_waveform/uart.py +9 -26
  28. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/__init__.py +1 -3
  29. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/converter.py +4 -12
  30. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/patcher.py +4 -12
  31. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/validation.py +1 -2
  32. shepherd_core-2024.4.1/shepherd_core/inventory/__init__.py +105 -0
  33. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/system.py +10 -5
  34. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/logger.py +1 -3
  35. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/reader.py +45 -57
  36. shepherd_core-2024.4.1/shepherd_core/testbed_client/cache_path.py +15 -0
  37. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/client.py +7 -19
  38. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/fixtures.py +8 -15
  39. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/user_model.py +7 -7
  40. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_converter_model.py +5 -15
  41. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_harvester_model.py +2 -3
  42. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/virtual_source_model.py +3 -6
  43. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/writer.py +16 -24
  44. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/PKG-INFO +50 -38
  45. shepherd_core-2024.4.1/shepherd_core.egg-info/SOURCES.txt +72 -0
  46. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/requires.txt +1 -4
  47. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/top_level.txt +0 -1
  48. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_cal_hw.py +2 -6
  49. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_examples.py +6 -5
  50. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_writer.py +4 -4
  51. shepherd_core-2023.11.1/pyproject.toml +0 -10
  52. shepherd_core-2023.11.1/setup.cfg +0 -89
  53. shepherd_core-2023.11.1/shepherd_core/data_models/content/_external_fixtures.yaml +0 -394
  54. shepherd_core-2023.11.1/shepherd_core/data_models/content/energy_environment_fixture.yaml +0 -50
  55. shepherd_core-2023.11.1/shepherd_core/data_models/content/virtual_harvester_fixture.yaml +0 -159
  56. shepherd_core-2023.11.1/shepherd_core/data_models/content/virtual_source_fixture.yaml +0 -229
  57. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/cape_fixture.yaml +0 -94
  58. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/gpio_fixture.yaml +0 -166
  59. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/mcu_fixture.yaml +0 -19
  60. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/observer_fixture.yaml +0 -220
  61. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/target_fixture.yaml +0 -137
  62. shepherd_core-2023.11.1/shepherd_core/data_models/testbed/testbed_fixture.yaml +0 -25
  63. shepherd_core-2023.11.1/shepherd_core/inventory/__init__.py +0 -61
  64. shepherd_core-2023.11.1/shepherd_core.egg-info/SOURCES.txt +0 -126
  65. shepherd_core-2023.11.1/tests/__init__.py +0 -0
  66. shepherd_core-2023.11.1/tests/conftest.py +0 -64
  67. shepherd_core-2023.11.1/tests/data_models/__init__.py +0 -0
  68. shepherd_core-2023.11.1/tests/data_models/conftest.py +0 -14
  69. shepherd_core-2023.11.1/tests/data_models/example_cal_data.yaml +0 -31
  70. shepherd_core-2023.11.1/tests/data_models/example_cal_data_faulty.yaml +0 -29
  71. shepherd_core-2023.11.1/tests/data_models/example_cal_meas.yaml +0 -178
  72. shepherd_core-2023.11.1/tests/data_models/example_cal_meas_faulty1.yaml +0 -142
  73. shepherd_core-2023.11.1/tests/data_models/example_cal_meas_faulty2.yaml +0 -136
  74. shepherd_core-2023.11.1/tests/data_models/example_config_emulator.yaml +0 -41
  75. shepherd_core-2023.11.1/tests/data_models/example_config_experiment.yaml +0 -16
  76. shepherd_core-2023.11.1/tests/data_models/example_config_experiment_alternative.yaml +0 -14
  77. shepherd_core-2023.11.1/tests/data_models/example_config_harvester.yaml +0 -15
  78. shepherd_core-2023.11.1/tests/data_models/example_config_testbed.yaml +0 -26
  79. shepherd_core-2023.11.1/tests/data_models/example_config_virtsource.yaml +0 -78
  80. shepherd_core-2023.11.1/tests/data_models/test_base_models.py +0 -205
  81. shepherd_core-2023.11.1/tests/data_models/test_content_fixtures.py +0 -41
  82. shepherd_core-2023.11.1/tests/data_models/test_content_models.py +0 -288
  83. shepherd_core-2023.11.1/tests/data_models/test_examples.py +0 -48
  84. shepherd_core-2023.11.1/tests/data_models/test_experiment_models.py +0 -279
  85. shepherd_core-2023.11.1/tests/data_models/test_task_generation.py +0 -52
  86. shepherd_core-2023.11.1/tests/data_models/test_task_models.py +0 -131
  87. shepherd_core-2023.11.1/tests/data_models/test_testbed_fixtures.py +0 -47
  88. shepherd_core-2023.11.1/tests/data_models/test_testbed_models.py +0 -187
  89. shepherd_core-2023.11.1/tests/decoder_waveform/__init__.py +0 -0
  90. shepherd_core-2023.11.1/tests/decoder_waveform/test_decoder.py +0 -34
  91. shepherd_core-2023.11.1/tests/fw_tools/__init__.py +0 -0
  92. shepherd_core-2023.11.1/tests/fw_tools/conftest.py +0 -5
  93. shepherd_core-2023.11.1/tests/fw_tools/test_converter.py +0 -76
  94. shepherd_core-2023.11.1/tests/fw_tools/test_patcher.py +0 -66
  95. shepherd_core-2023.11.1/tests/fw_tools/test_validation.py +0 -56
  96. shepherd_core-2023.11.1/tests/inventory/__init__.py +0 -0
  97. shepherd_core-2023.11.1/tests/inventory/test_inventory.py +0 -22
  98. shepherd_core-2023.11.1/tests/testbed_client/__init__.py +0 -0
  99. shepherd_core-2023.11.1/tests/vsource/__init__.py +0 -0
  100. shepherd_core-2023.11.1/tests/vsource/conftest.py +0 -51
  101. shepherd_core-2023.11.1/tests/vsource/test_converter.py +0 -165
  102. shepherd_core-2023.11.1/tests/vsource/test_harvester.py +0 -79
  103. shepherd_core-2023.11.1/tests/vsource/test_z.py +0 -5
  104. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/calibration_hw_def.py +0 -0
  105. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/commons.py +0 -0
  106. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/__init__.py +0 -0
  107. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/__init__.py +0 -0
  108. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/cal_measurement.py +0 -0
  109. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/timezone.py +0 -0
  110. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/base/wrapper.py +0 -0
  111. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/__init__.py +0 -0
  112. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/firmware.py +0 -0
  113. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/content/firmware_datatype.py +0 -0
  114. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/doc_virtual_source.py +0 -0
  115. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/experiment/__init__.py +0 -0
  116. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/task/programming.py +0 -0
  117. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/data_models/testbed/__init__.py +0 -0
  118. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/decoder_waveform/__init__.py +0 -0
  119. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/fw_tools/converter_elf.py +0 -0
  120. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/python.py +0 -0
  121. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/inventory/target.py +0 -0
  122. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/testbed_client/__init__.py +0 -0
  123. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core/vsource/__init__.py +0 -0
  124. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/dependency_links.txt +0 -0
  125. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/shepherd_core.egg-info/zip-safe +0 -0
  126. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_logger.py +0 -0
  127. {shepherd_core-2023.11.1 → shepherd_core-2024.4.1}/tests/test_reader.py +0 -0
@@ -1,14 +1,12 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: shepherd_core
3
- Version: 2023.11.1
3
+ Version: 2024.4.1
4
4
  Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
5
- Home-page: https://pypi.org/project/shepherd-core/
6
- Author: Ingmar Splitt, Kai Geissdoerfer
7
- Author-email: ingmar.splitt@tu-dresden.de
8
- Maintainer-email: ingmar.splitt@tu-dresden.de
9
- License: MIT
10
- Project-URL: Tracker, https://github.com/orgua/shepherd-datalib/issues
11
- Project-URL: Source, https://github.com/orgua/shepherd-datalib
5
+ Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
6
+ Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
7
+ Project-URL: Documentation, https://github.com/orgua/shepherd-datalib/blob/main/README.md
8
+ Project-URL: Issues, https://github.com/orgua/shepherd-datalib/issues
9
+ Project-URL: Source, https://pypi.org/project/shepherd-core/
12
10
  Keywords: testbed,beaglebone,pru,batteryless,energyharvesting,solar
13
11
  Platform: unix
14
12
  Platform: linux
@@ -35,8 +33,8 @@ Requires-Dist: numpy
35
33
  Requires-Dist: pyYAML
36
34
  Requires-Dist: chromalog
37
35
  Requires-Dist: pydantic[email]>2.0.0
38
- Requires-Dist: tqdm
39
36
  Requires-Dist: scipy
37
+ Requires-Dist: tqdm
40
38
  Requires-Dist: intelhex
41
39
  Requires-Dist: requests
42
40
  Requires-Dist: pyelftools
@@ -46,9 +44,6 @@ Requires-Dist: pwntools-elf-only; extra == "elf"
46
44
  Provides-Extra: inventory
47
45
  Requires-Dist: psutil; extra == "inventory"
48
46
  Provides-Extra: dev
49
- Requires-Dist: black; extra == "dev"
50
- Requires-Dist: pylint; extra == "dev"
51
- Requires-Dist: flake8; extra == "dev"
52
47
  Requires-Dist: twine; extra == "dev"
53
48
  Requires-Dist: pre-commit; extra == "dev"
54
49
  Requires-Dist: pyright; extra == "dev"
@@ -64,9 +59,9 @@ Requires-Dist: coverage; extra == "test"
64
59
  [![PyPiVersion](https://img.shields.io/pypi/v/shepherd_core.svg)](https://pypi.org/project/shepherd_core)
65
60
  [![image](https://img.shields.io/pypi/pyversions/shepherd_core.svg)](https://pypi.python.org/pypi/shepherd-core)
66
61
  [![Pytest](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml/badge.svg)](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
67
- [![CodeStyle](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
62
+ [![CodeStyle](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
68
63
 
69
- **Documentation**: <https://orgua.github.io/shepherd/external/shepherd_core.html>
64
+ **Main Documentation**: <https://orgua.github.io/shepherd>
70
65
 
71
66
  **Source Code**: <https://github.com/orgua/shepherd-datalib>
72
67
 
@@ -74,53 +69,42 @@ Requires-Dist: coverage; extra == "test"
74
69
 
75
70
  ---
76
71
 
77
- This Python Module is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
72
+ `shepherd-core` is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
78
73
 
79
- For postprocessing shepherds .h5-files users want to use [shepherd_data](https://pypi.org/project/shepherd_data).
74
+ For postprocessing shepherds .h5-files usage of [shepherd_data](https://pypi.org/project/shepherd_data) is recommended.
80
75
 
81
76
  ## Features
82
77
 
83
78
  - read and write shepherds hdf5-files
84
- - create, read, write and convert experiments for the testbed (all data-models included)
79
+ - create, read, write and convert experiments for the testbed
80
+ - all required data-models are included
85
81
  - simulate the virtual source, including virtual harvesters (and virtual converter as a whole)
86
- - connect and query the testbed via a webclient (TestbedClient)
82
+ - connect and query the testbed via a webclient (TestbedClient in alpha-stage)
87
83
  - offline usage defaults to static demo-fixtures loaded from yaml-files in the model-directories
88
84
  - work with target-firmwares
89
85
  - embed, modify, verify, convert
90
86
  - **Note**: working with ELF-files requires external dependencies, see ``Installation``-Chapter
91
87
  - decode waveforms (gpio-state & timestamp) to UART
92
- - create an inventory (used versions of software, hardware)
88
+ - create an inventory (for deployed versions of software, hardware)
93
89
 
94
- See [examples](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased there. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a trafficbench-experiment that's used to derive the link-matrix.
90
+ See [official documentation](https://orgua.github.io/shepherd) or [example scripts](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased in both. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a [trafficbench](https://github.com/orgua/TrafficBench)-experiment that's used to derive the link-matrix of the testbed-nodes.
95
91
 
96
- ### Compatibility
92
+ ## Config-Models in Detail
97
93
 
98
- | OS | PyVersion | Comment |
99
- |---------|--------------|--------------------------------------------|
100
- | Ubuntu | 3.8 - 3.12 | |
101
- | Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
102
- | MacOS | 3.8 - 3.12 | hex-conversion missing |
94
+ These pydantic data-models are used throughout all shepherd interfaces. Users can create an experiment, include their own content and feed it to the testbed.
103
95
 
104
- Notes:
105
- - hex-conversion needs a working and accessible objcopy
106
- - elf-supports needs
107
- - ``shepherd-core[elf]`` installs ``pwntools-elf-only``
108
- - most elf-features also still utilize hex-conversion
109
-
110
- ### Data-Models in Detail
111
-
112
- - new orchestration ``/data-models`` with focus on remote shepherd-testbed
96
+ - orchestration ``/data-models`` with focus on remote shepherd-testbed
113
97
  - classes of sub-models
114
98
  - ``/base``: base-classes, configuration and -functionality for all models
115
99
  - ``/testbed``: meta-data representation of all testbed-components
116
- - ``/content``: reusable meta-data for fw, h5 and vsrc-definitions
100
+ - ``/content``: reusable user-defined meta-data for fw, h5 and vsrc-definitions
117
101
  - ``/experiment``: configuration-models including sub-systems
118
102
  - ``/task``: digestible configs for shepherd-herd or -sheep
119
103
  - behavior controlled by ``ShpModel`` and ``content``-model
120
104
  - a basic database is available as fixtures through a ``tb_client``
121
105
  - fixtures selectable by name & ID
122
106
  - fixtures support inheritance
123
- - models support
107
+ - the models support
124
108
  - auto-completion with neutral / sensible values
125
109
  - complex and custom datatypes (i.e. PositiveInt, lists-checks on length)
126
110
  - checking of inputs and type-casting
@@ -133,6 +117,20 @@ Notes:
133
117
  - exposes no internal paths
134
118
  - experiments can be transformed to task-sets (``TestbedTasks.from_xp()``)
135
119
 
120
+ ## Compatibility
121
+
122
+ | OS | PyVersion | Comment |
123
+ |---------|--------------|--------------------------------------------|
124
+ | Ubuntu | 3.8 - 3.12 | |
125
+ | Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
126
+ | MacOS | 3.8 - 3.12 | hex-conversion missing |
127
+
128
+ Notes:
129
+ - hex-conversion needs a working and accessible objcopy
130
+ - elf-supports needs
131
+ - ``shepherd-core[elf]`` installs ``pwntools-elf-only``
132
+ - most elf-features also still utilize hex-conversion
133
+
136
134
  ## Installation
137
135
 
138
136
  The Library is available via PyPI and can be installed with
@@ -169,3 +167,17 @@ For creating an inventory of the host-system you should install
169
167
  ```shell
170
168
  pip install shepherd-core[inventory]
171
169
  ```
170
+
171
+ ## Unittests
172
+
173
+ To run the testbench, follow these steps:
174
+
175
+ 1. Navigate your host-shell into the package-folder and
176
+ 2. install dependencies
177
+ 3. run the testbench (~ 320 tests):
178
+
179
+ ```Shell
180
+ cd shepherd-datalib/shepherd_core
181
+ pip3 install ./[tests]
182
+ pytest
183
+ ```
@@ -3,9 +3,9 @@
3
3
  [![PyPiVersion](https://img.shields.io/pypi/v/shepherd_core.svg)](https://pypi.org/project/shepherd_core)
4
4
  [![image](https://img.shields.io/pypi/pyversions/shepherd_core.svg)](https://pypi.python.org/pypi/shepherd-core)
5
5
  [![Pytest](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml/badge.svg)](https://github.com/orgua/shepherd-datalib/actions/workflows/py_unittest.yml)
6
- [![CodeStyle](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
6
+ [![CodeStyle](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
7
7
 
8
- **Documentation**: <https://orgua.github.io/shepherd/external/shepherd_core.html>
8
+ **Main Documentation**: <https://orgua.github.io/shepherd>
9
9
 
10
10
  **Source Code**: <https://github.com/orgua/shepherd-datalib>
11
11
 
@@ -13,53 +13,42 @@
13
13
 
14
14
  ---
15
15
 
16
- This Python Module is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
16
+ `shepherd-core` is designed as a library and bundles data-models and file-access-routines for the shepherd-testbed, that are used by several codebases.
17
17
 
18
- For postprocessing shepherds .h5-files users want to use [shepherd_data](https://pypi.org/project/shepherd_data).
18
+ For postprocessing shepherds .h5-files usage of [shepherd_data](https://pypi.org/project/shepherd_data) is recommended.
19
19
 
20
20
  ## Features
21
21
 
22
22
  - read and write shepherds hdf5-files
23
- - create, read, write and convert experiments for the testbed (all data-models included)
23
+ - create, read, write and convert experiments for the testbed
24
+ - all required data-models are included
24
25
  - simulate the virtual source, including virtual harvesters (and virtual converter as a whole)
25
- - connect and query the testbed via a webclient (TestbedClient)
26
+ - connect and query the testbed via a webclient (TestbedClient in alpha-stage)
26
27
  - offline usage defaults to static demo-fixtures loaded from yaml-files in the model-directories
27
28
  - work with target-firmwares
28
29
  - embed, modify, verify, convert
29
30
  - **Note**: working with ELF-files requires external dependencies, see ``Installation``-Chapter
30
31
  - decode waveforms (gpio-state & timestamp) to UART
31
- - create an inventory (used versions of software, hardware)
32
+ - create an inventory (for deployed versions of software, hardware)
32
33
 
33
- See [examples](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased there. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a trafficbench-experiment that's used to derive the link-matrix.
34
+ See [official documentation](https://orgua.github.io/shepherd) or [example scripts](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/examples) for more details and usage. Most functionality is showcased in both. The [extra](https://github.com/orgua/shepherd-datalib/tree/main/shepherd_core/extra)-directory holds data-generators relevant for the testbed. Notably is a [trafficbench](https://github.com/orgua/TrafficBench)-experiment that's used to derive the link-matrix of the testbed-nodes.
34
35
 
35
- ### Compatibility
36
+ ## Config-Models in Detail
36
37
 
37
- | OS | PyVersion | Comment |
38
- |---------|--------------|--------------------------------------------|
39
- | Ubuntu | 3.8 - 3.12 | |
40
- | Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
41
- | MacOS | 3.8 - 3.12 | hex-conversion missing |
42
-
43
- Notes:
44
- - hex-conversion needs a working and accessible objcopy
45
- - elf-supports needs
46
- - ``shepherd-core[elf]`` installs ``pwntools-elf-only``
47
- - most elf-features also still utilize hex-conversion
48
-
49
- ### Data-Models in Detail
38
+ These pydantic data-models are used throughout all shepherd interfaces. Users can create an experiment, include their own content and feed it to the testbed.
50
39
 
51
- - new orchestration ``/data-models`` with focus on remote shepherd-testbed
40
+ - orchestration ``/data-models`` with focus on remote shepherd-testbed
52
41
  - classes of sub-models
53
42
  - ``/base``: base-classes, configuration and -functionality for all models
54
43
  - ``/testbed``: meta-data representation of all testbed-components
55
- - ``/content``: reusable meta-data for fw, h5 and vsrc-definitions
44
+ - ``/content``: reusable user-defined meta-data for fw, h5 and vsrc-definitions
56
45
  - ``/experiment``: configuration-models including sub-systems
57
46
  - ``/task``: digestible configs for shepherd-herd or -sheep
58
47
  - behavior controlled by ``ShpModel`` and ``content``-model
59
48
  - a basic database is available as fixtures through a ``tb_client``
60
49
  - fixtures selectable by name & ID
61
50
  - fixtures support inheritance
62
- - models support
51
+ - the models support
63
52
  - auto-completion with neutral / sensible values
64
53
  - complex and custom datatypes (i.e. PositiveInt, lists-checks on length)
65
54
  - checking of inputs and type-casting
@@ -72,6 +61,20 @@ Notes:
72
61
  - exposes no internal paths
73
62
  - experiments can be transformed to task-sets (``TestbedTasks.from_xp()``)
74
63
 
64
+ ## Compatibility
65
+
66
+ | OS | PyVersion | Comment |
67
+ |---------|--------------|--------------------------------------------|
68
+ | Ubuntu | 3.8 - 3.12 | |
69
+ | Windows | 3.8 - 3.12 | no support for elf and hex-conversions yet |
70
+ | MacOS | 3.8 - 3.12 | hex-conversion missing |
71
+
72
+ Notes:
73
+ - hex-conversion needs a working and accessible objcopy
74
+ - elf-supports needs
75
+ - ``shepherd-core[elf]`` installs ``pwntools-elf-only``
76
+ - most elf-features also still utilize hex-conversion
77
+
75
78
  ## Installation
76
79
 
77
80
  The Library is available via PyPI and can be installed with
@@ -108,3 +111,17 @@ For creating an inventory of the host-system you should install
108
111
  ```shell
109
112
  pip install shepherd-core[inventory]
110
113
  ```
114
+
115
+ ## Unittests
116
+
117
+ To run the testbench, follow these steps:
118
+
119
+ 1. Navigate your host-shell into the package-folder and
120
+ 2. install dependencies
121
+ 3. run the testbench (~ 320 tests):
122
+
123
+ ```Shell
124
+ cd shepherd-datalib/shepherd_core
125
+ pip3 install ./[tests]
126
+ pytest
127
+ ```
@@ -0,0 +1,112 @@
1
+ [project]
2
+ name = "shepherd_core"
3
+ description = "Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed"
4
+ keywords = ["testbed", "beaglebone", "pru", "batteryless", "energyharvesting", "solar"]
5
+
6
+ authors = [{name = "Ingmar Splitt", email = "ingmar.splitt@tu-dresden.de"},]
7
+ maintainers = [{name = "Ingmar Splitt", email = "ingmar.splitt@tu-dresden.de"},]
8
+
9
+ readme = {file = "README.md", content-type = "text/markdown"}
10
+ license = {file = "LICENSE"}
11
+ dynamic = ["version"]
12
+
13
+ classifiers = [
14
+ "Development Status :: 5 - Production/Stable",
15
+ "Intended Audience :: Developers",
16
+ "Intended Audience :: Information Technology",
17
+ "Intended Audience :: Science/Research",
18
+ "Programming Language :: Python :: 3.8",
19
+ "Programming Language :: Python :: 3.9",
20
+ "Programming Language :: Python :: 3.10",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "License :: OSI Approved :: MIT License",
24
+ "Operating System :: OS Independent",
25
+ "Natural Language :: English",
26
+ ]
27
+
28
+ requires-python = ">=3.8"
29
+ dependencies = [
30
+ "h5py",
31
+ "numpy",
32
+ "pyYAML",
33
+ "chromalog",
34
+ "pydantic[email]>2.0.0",
35
+ "scipy",
36
+ "tqdm",
37
+ "intelhex",
38
+ "requests",
39
+ "pyelftools",
40
+ "zstandard",
41
+ ]
42
+
43
+ [project.optional-dependencies]
44
+ elf = [
45
+ "pwntools-elf-only",
46
+ # TODO: still separate because of missing windows-compat
47
+ # modified package for reading ELF-files with fw_tools
48
+ # original can be problematic to install on windows & beaglebone
49
+ # probably also needed: https://docs.pwntools.com/en/stable/install.html + binutils-$ARCH
50
+ ]
51
+
52
+ inventory = [
53
+ "psutil",
54
+ ]
55
+
56
+
57
+ dev = [
58
+ "twine",
59
+ "pre-commit",
60
+ "pyright",
61
+ "ruff",
62
+ "mypy",
63
+ "types-PyYAML",
64
+ ]
65
+
66
+ test = [
67
+ "pytest",
68
+ "coverage",
69
+ ]
70
+
71
+ [project.urls]
72
+ Documentation = "https://github.com/orgua/shepherd-datalib/blob/main/README.md"
73
+ Issues = "https://github.com/orgua/shepherd-datalib/issues"
74
+ Source = "https://pypi.org/project/shepherd-core/"
75
+
76
+ [build-system]
77
+ requires = ["setuptools"]
78
+ build-backend = "setuptools.build_meta"
79
+
80
+ [tool.setuptools]
81
+ platforms = ["unix", "linux", "osx", "cygwin", "win32", "win64"]
82
+ zip-safe = true
83
+ include-package-data = true
84
+
85
+ [tool.setuptools.package-dir]
86
+ shepherd_core = "shepherd_core"
87
+
88
+ [tool.setuptools.package-data]
89
+ shepherd_core = [
90
+ "README.md",
91
+ "*.yaml",
92
+ ]
93
+
94
+ [tool.setuptools.dynamic]
95
+ version = {attr = "shepherd_core.__version__"}
96
+
97
+ [tool.pytest.ini_options]
98
+ markers = [
99
+ "converter: automatically added when no objcopy is found (on ubuntu install build-essential)",
100
+ "elf: automatically added when no elf-support is found (on windows or missing shepherd-core[elf])"
101
+ ]
102
+
103
+ addopts = "-vvv --stepwise" # opts: verbose result for each tests
104
+ # TODO: add something like "--cov --cov-report html --cov-report term-missing --cov-fail-under 95"
105
+
106
+ [tool.coverage.run]
107
+ source = ["shepherd_core"]
108
+ omit = ["*/shepherd_data/*"]
109
+
110
+ [tool.mypy]
111
+ python_version = 3.8
112
+ ignore_missing_imports = true
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -4,6 +4,7 @@ Provides classes for storing and retrieving sampled IV data to/from
4
4
  HDF5 files.
5
5
 
6
6
  """
7
+
7
8
  from .data_models.base.calibration import Calc_t
8
9
  from .data_models.base.calibration import CalibrationCape
9
10
  from .data_models.base.calibration import CalibrationEmulator
@@ -12,7 +13,7 @@ from .data_models.base.calibration import CalibrationPair
12
13
  from .data_models.base.calibration import CalibrationSeries
13
14
  from .data_models.base.timezone import local_now
14
15
  from .data_models.base.timezone import local_tz
15
- from .data_models.task import Compression
16
+ from .data_models.task.emulation import Compression
16
17
  from .inventory import Inventory
17
18
  from .logger import get_verbose_level
18
19
  from .logger import increase_verbose_level
@@ -22,7 +23,7 @@ from .testbed_client.client import TestbedClient
22
23
  from .testbed_client.client import tb_client
23
24
  from .writer import Writer
24
25
 
25
- __version__ = "2023.11.1"
26
+ __version__ = "2024.04.1"
26
27
 
27
28
  __all__ = [
28
29
  "Reader",
@@ -165,7 +165,7 @@ class CapeData(ShpModel):
165
165
  `See<https://github.com/beagleboard/beaglebone-black/wiki/System-Reference-Manual#824_EEPROM_Data_Format>`_
166
166
  """
167
167
 
168
- header: conbytes(max_length=4) = b"\xAA\x55\x33\xEE"
168
+ header: conbytes(max_length=4) = b"\xaa\x55\x33\xee"
169
169
  eeprom_revision: constr(max_length=2) = "A2"
170
170
  board_name: constr(max_length=32) = "BeagleBone SHEPHERD2 Cape"
171
171
  version: constr(max_length=4) = "24B0"
@@ -207,6 +207,7 @@ class CalibrationCape(ShpModel):
207
207
  cape: data can be supplied
208
208
  Returns:
209
209
  CalibrationCape object with extracted calibration data.
210
+
210
211
  """
211
212
  dv = cls().model_dump(include={"harvester", "emulator"})
212
213
  lw = list(dict_generator(dv))
@@ -225,6 +226,7 @@ class CalibrationCape(ShpModel):
225
226
  Returns
226
227
  -------
227
228
  Byte string representation of calibration values.
229
+
228
230
  """
229
231
  lw = list(dict_generator(self.model_dump(include={"harvester", "emulator"})))
230
232
  values = [walk[-1] for walk in lw]
@@ -1,7 +1,10 @@
1
1
  import hashlib
2
2
  from datetime import datetime
3
3
  from typing import Optional
4
+ from typing import Union
5
+ from uuid import uuid4
4
6
 
7
+ from pydantic import UUID4
5
8
  from pydantic import Field
6
9
  from pydantic import StringConstraints
7
10
  from pydantic import model_validator
@@ -14,34 +17,34 @@ from .timezone import local_now
14
17
  # constr -> to_lower=True, max_length=16, regex=r"^[\w]+$"
15
18
  # ⤷ Regex = AlphaNum
16
19
  IdInt = Annotated[int, Field(ge=0, lt=2**128)]
17
- NameStr = Annotated[
18
- str, StringConstraints(max_length=32, pattern=r'^[^<>:;,?"*|\/\\]+$')
19
- ]
20
+ NameStr = Annotated[str, StringConstraints(max_length=32, pattern=r"^[^<>:;,?\"\*|\/\\]+$")]
20
21
  # ⤷ Regex = FileSystem-Compatible ASCII
21
22
  SafeStr = Annotated[str, StringConstraints(pattern=r"^[ -~]+$")]
22
23
  # ⤷ Regex = All Printable ASCII-Characters with Space
23
24
 
24
25
 
25
26
  def id_default() -> int:
27
+ # note: IdInt has space for 128 bit, so 128/4 = 32 hex-chars
26
28
  time_stamp = str(local_now()).encode("utf-8")
27
- time_hash = hashlib.sha3_224(time_stamp).hexdigest()[-16:]
29
+ time_hash = hashlib.sha3_224(time_stamp).hexdigest()[-32:]
28
30
  return int(time_hash, 16)
29
31
 
30
32
 
31
33
  class ContentModel(ShpModel):
32
34
  # General Properties
33
- id: IdInt = Field( # noqa: A003
35
+ # id: UUID4 = Field( # TODO: db-migration - temp fix for documentation
36
+ id: Union[UUID4, int] = Field(
34
37
  description="Unique ID",
35
- default_factory=id_default,
38
+ default_factory=uuid4,
36
39
  )
37
40
  name: NameStr
38
- description: Annotated[
39
- Optional[SafeStr], Field(description="Required when public")
40
- ] = None
41
+ description: Annotated[Optional[SafeStr], Field(description="Required when public")] = None
41
42
  comment: Optional[SafeStr] = None
42
43
  created: datetime = Field(default_factory=datetime.now)
44
+ updated_last: datetime = Field(default_factory=datetime.now)
43
45
 
44
46
  # Ownership & Access
47
+ # TODO: remove owner & group, only needed for DB
45
48
  owner: NameStr
46
49
  group: Annotated[NameStr, Field(description="University or Subgroup")]
47
50
  visible2group: bool = False
@@ -57,7 +60,6 @@ class ContentModel(ShpModel):
57
60
  is_visible = self.visible2group or self.visible2all
58
61
  if is_visible and self.description is None:
59
62
  raise ValueError(
60
- "Public instances require a description "
61
- "(check visible2*- and description-field)"
63
+ "Public instances require a description (check visible2*- and description-field)"
62
64
  )
63
65
  return self
@@ -7,6 +7,7 @@ from typing import Any
7
7
  from typing import Generator
8
8
  from typing import Optional
9
9
  from typing import Union
10
+ from uuid import UUID
10
11
 
11
12
  import yaml
12
13
  from pydantic import BaseModel
@@ -27,9 +28,7 @@ def path2str(
27
28
 
28
29
 
29
30
  def time2int(dumper: Dumper, data: timedelta) -> ScalarNode:
30
- return dumper.represent_scalar(
31
- "tag:yaml.org,2002:int", str(int(data.total_seconds()))
32
- )
31
+ return dumper.represent_scalar("tag:yaml.org,2002:int", str(int(data.total_seconds())))
33
32
 
34
33
 
35
34
  def generic2str(dumper: Dumper, data: Any) -> ScalarNode:
@@ -41,6 +40,7 @@ yaml.add_representer(pathlib.WindowsPath, path2str, SafeDumper)
41
40
  yaml.add_representer(pathlib.Path, path2str, SafeDumper)
42
41
  yaml.add_representer(timedelta, time2int, SafeDumper)
43
42
  yaml.add_representer(IPv4Address, generic2str, SafeDumper)
43
+ yaml.add_representer(UUID, generic2str, SafeDumper)
44
44
 
45
45
 
46
46
  class ShpModel(BaseModel):
@@ -109,9 +109,7 @@ class ShpModel(BaseModel):
109
109
  def schema_to_file(cls, path: Union[str, Path]) -> None:
110
110
  """Store schema to yaml (for frontend-generators)"""
111
111
  model_dict = cls.model_json_schema()
112
- model_yaml = yaml.safe_dump(
113
- model_dict, default_flow_style=False, sort_keys=False
114
- )
112
+ model_yaml = yaml.safe_dump(model_dict, default_flow_style=False, sort_keys=False)
115
113
  with Path(path).resolve().with_suffix(".yaml").open("w") as f:
116
114
  f.write(model_yaml)
117
115
 
@@ -13,7 +13,7 @@ class EnergyDType(str, Enum):
13
13
  ivsample = "ivsample"
14
14
  ivsamples = "ivsample"
15
15
  ivcurve = "ivcurve"
16
- ivcurves = "ivcurve"
16
+ ivcurves = "ivcurve" # a stream could also be called iv-surface
17
17
  isc_voc = "isc_voc"
18
18
 
19
19
 
@@ -76,7 +76,7 @@ class VirtualHarvesterConfig(ContentModel, title="Config for the Harvester"):
76
76
  logger.debug("VHrv-Inheritances: %s", chain)
77
77
 
78
78
  # post corrections -> should be in separate validator
79
- cal = CalibrationHarvester() # todo: as argument?
79
+ cal = CalibrationHarvester() # TODO: as argument?
80
80
  c_limit = values.get("current_limit_uA", 50_000) # cls.current_limit_uA)
81
81
  values["current_limit_uA"] = max(10**6 * cal.adc_C_Hrv.raw_to_si(4), c_limit)
82
82