shepherd-core 2024.4.1__tar.gz → 2024.4.2__tar.gz

{shepherd_core-2024.4.1 → shepherd_core-2024.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shepherd_core
-Version: 2024.4.1
+Version: 2024.4.2
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
@@ -39,6 +39,7 @@ Requires-Dist: intelhex
 Requires-Dist: requests
 Requires-Dist: pyelftools
 Requires-Dist: zstandard
+Requires-Dist: typing-extensions
 Provides-Extra: elf
 Requires-Dist: pwntools-elf-only; extra == "elf"
 Provides-Extra: inventory

shepherd_core-2024.4.2/examples/experiment_from_yaml.yaml

@@ -0,0 +1,30 @@
+datatype: Experiment
+parameters:
+  id: 4567
+  name: meaningful Test-Name
+  time_start: 2033-03-13 14:15:16
+  target_configs:
+  - target_IDs: [9, 10, 11]
+    custom_IDs: [0, 1, 2]
+    energy_env:
+      name: SolarSunny
+    virtual_source:
+      name: diode+capacitor
+    firmware1:
+      name: nrf52_demo_rf
+  - target_IDs:
+      - 1
+      - 2
+      - 3
+      - 4
+    custom_IDs: [7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17]
+    energy_env:
+      name: ThermoelectricWashingMachine
+    virtual_source:
+      name: BQ25570-Schmitt
+      harvester:
+        name: mppt_bq_thermoelectric
+    firmware1:
+      name: nrf52_demo_rf
+    firmware2:
+      name: msp430_deep_sleep

shepherd_core-2024.4.2/examples/experiment_generic_var1.py

@@ -0,0 +1,74 @@
+"""How-to for defining an experiment - variant 1.
+
+What the code does:
+- recommended approach for missing testbed-client
+- variants
+- var1 - references a server-path for the firmware
+- var2 - embeds local firmware in yaml (elf-support is linux-only)
+- assumption:
+- start ASAP,
+- no custom IDs,
+- static Power-Supply
+- no power-tracing
+
+"""
+
+from pathlib import Path
+
+from shepherd_core import TestbedClient
+from shepherd_core.data_models import FirmwareDType
+from shepherd_core.data_models import GpioTracing
+from shepherd_core.data_models.content import EnergyEnvironment
+from shepherd_core.data_models.content import Firmware
+from shepherd_core.data_models.experiment import Experiment
+from shepherd_core.data_models.experiment import TargetConfig
+from shepherd_core.data_models.task import TestbedTasks
+from shepherd_core.data_models.testbed import MCU
+
+# for online-queries the lib can be connected to the testbed-server
+# NOTE: there are 3 states:
+#    - unconnected -> demo-fixture is queried (locally)
+#    - connected -> publicly available data is queried online
+#    - logged in with token -> also private data is queried online
+tb_client = TestbedClient()
+do_connect = False
+
+if do_connect:
+    tb_client.connect()
+
+xp = Experiment(
+    id="4567",
+    name="meaningful_TestName",
+    # time_start could be "2033-03-13 14:15:16" or "datetime.now() + timedelta(minutes=30)"
+    duration=30,
+    target_configs=[
+        TargetConfig(
+            target_IDs=range(7, 13),
+            custom_IDs=range(1, 100),  # note: longer list is OK
+            energy_env=EnergyEnvironment(name="eenv_static_3000mV_50mA_3600s"),
+            firmware1=Firmware(
+                name="FW_TestXYZ",
+                data=Path("/var/shepherd/content/fw/nes_lab/nrf52_demo_rf/build.elf"),
+                data_type=FirmwareDType.path_elf,
+                data_local=False,
+                mcu=MCU(name="nRF52"),
+            ),
+            power_tracing=None,
+            gpio_tracing=GpioTracing(
+                uart_decode=True,  # enables logging uart from userspace
+                uart_baudrate=115_200,
+            ),
+        ),
+    ],
+)
+xp.to_file("experiment_generic_var1.yaml")
+
+# Create a tasks-list for the testbed
+tb_tasks = TestbedTasks.from_xp(xp)
+tb_tasks.to_file("experiment_generic_var1_tbt.yaml")
+
+# next steps:
+# - copy to server:
+#   scp ./experiment_generic_varX_tbt.yaml user@shepherd.cfaed.tu-dresden.de:/var/shepherd/content/
+# - run with herd-tool:
+#   shepherd-herd --verbose run --attach /var/shepherd/content/experiment_generic_varX_tbt.yaml

shepherd_core-2024.4.2/examples/experiment_generic_var2.py

@@ -0,0 +1,67 @@
+"""How-to for defining an experiment.
+
+- recommended approach for missing testbed-client
+- variants
+- var1 - references a server-path for the firmware
+- var2 - embeds local firmware in yaml (elf-support is linux-only)
+- assumption:
+- start ASAP,
+- no custom IDs,
+- static Power-Supply
+- no power-tracing
+
+"""
+
+from pathlib import Path
+
+from shepherd_core import TestbedClient
+from shepherd_core.data_models import GpioTracing
+from shepherd_core.data_models.content import EnergyEnvironment
+from shepherd_core.data_models.content import Firmware
+from shepherd_core.data_models.experiment import Experiment
+from shepherd_core.data_models.experiment import TargetConfig
+from shepherd_core.data_models.task import TestbedTasks
+
+# for online-queries the lib can be connected to the testbed-server
+# NOTE: there are 3 states:
+#    - unconnected -> demo-fixture is queried (locally)
+#    - connected -> publicly available data is queried online
+#    - logged in with token -> also private data is queried online
+tb_client = TestbedClient()
+do_connect = False
+
+if do_connect:
+    tb_client.connect()
+
+xp = Experiment(
+    id="4567",
+    name="meaningful_TestName",
+    # time_start could be "2033-03-13 14:15:16" or "datetime.now() + timedelta(minutes=30)"
+    duration=30,
+    target_configs=[
+        TargetConfig(
+            target_IDs=range(7, 13),
+            custom_IDs=range(1, 100),  # note: longer list is OK
+            energy_env=EnergyEnvironment(name="eenv_static_3000mV_50mA_3600s"),
+            firmware1=Firmware.from_firmware(
+                file=Path("./firmware_nrf.elf").absolute(),
+            ),
+            power_tracing=None,
+            gpio_tracing=GpioTracing(
+                uart_decode=True,  # enables logging uart from userspace
+                uart_baudrate=115_200,
+            ),
+        ),
+    ],
+)
+xp.to_file("experiment_generic_var2.yaml")
+
+# Create a tasks-list for the testbed
+tb_tasks = TestbedTasks.from_xp(xp)
+tb_tasks.to_file("experiment_generic_var2_tbt.yaml")
+
+# next steps:
+# - copy to server:
+#   scp ./experiment_generic_varX_tbt.yaml user@shepherd.cfaed.tu-dresden.de:/var/shepherd/content/
+# - run with herd-tool:
+#   shepherd-herd --verbose run --attach /var/shepherd/content/experiment_generic_varX_tbt.yaml

shepherd_core-2024.4.2/examples/experiment_models.py

@@ -0,0 +1,91 @@
+"""Shows different ways to define an experiment.
+
+How to define an experiment:
+
+- within python (shown in this example)
+    - object-oriented data-models of
+        - experiment
+        - TargetConfig -> shared for group of targets
+        - virtualSource -> defines energy environment and converters
+    - sub-elements reusable
+    - scriptable for range of experiments
+    - check for plausibility right away
+- as yaml (shown in experiment_from_yaml.yaml)
+    - default file-format for storing meta-data (for shepherd)
+    - minimal writing
+    - easy to copy parts
+    - submittable through web-interface
+
+"""
+
+from shepherd_core import TestbedClient
+from shepherd_core.data_models.content import EnergyEnvironment
+from shepherd_core.data_models.content import Firmware
+from shepherd_core.data_models.content import VirtualHarvesterConfig
+from shepherd_core.data_models.content import VirtualSourceConfig
+from shepherd_core.data_models.experiment import Experiment
+from shepherd_core.data_models.experiment import TargetConfig
+from shepherd_core.data_models.task import TestbedTasks
+
+# generate description for all parameters -> base for web-forms
+Experiment.schema_to_file("experiment_schema.yaml")
+
+# for online-queries the lib can be connected to the testbed-server
+# NOTE: there are 3 states:
+#    - unconnected -> demo-fixture is queried (locally)
+#    - connected -> publicly available data is queried online
+#    - logged in with token -> also private data is queried online
+tb_client = TestbedClient()
+do_connect = False
+
+if do_connect:
+    tb_client.connect()
+
+# Defining an Experiment in Python
+hrv = VirtualHarvesterConfig(name="mppt_bq_thermoelectric")
+
+target_cfgs = [
+    # first Instance similar to yaml-syntax
+    TargetConfig(
+        target_IDs=[9, 10, 11],
+        custom_IDs=[0, 1, 2],
+        energy_env={"name": "SolarSunny"},
+        virtual_source={"name": "diode+capacitor"},
+        firmware1={"name": "nrf52_demo_rf"},
+    ),
+    # second Instance fully object-oriented (recommended)
+    TargetConfig(
+        target_IDs=list(range(1, 5)),
+        custom_IDs=list(range(7, 18)),  # note: longer list is OK
+        energy_env=EnergyEnvironment(name="ThermoelectricWashingMachine"),
+        virtual_source=VirtualSourceConfig(name="BQ25570-Schmitt", harvester=hrv),
+        firmware1=Firmware(name="nrf52_demo_rf"),
+        firmware2=Firmware(name="msp430_deep_sleep"),
+    ),
+]
+
+xperi1 = Experiment(
+    id="4567",
+    name="meaningful Test-Name",
+    time_start="2033-03-13 14:15:16",  # or: datetime.now() + timedelta(minutes=30)
+    target_configs=target_cfgs,
+)
+
+# Safe, reload and compare content
+xperi1.to_file("experiment_from_py.yaml", minimal=False)
+xperi2 = Experiment.from_file("experiment_from_py.yaml")
+print(f"xp1 hash: {xperi1.get_hash()}")
+print(f"xp2 hash: {xperi2.get_hash()}")
+
+# comparison to same config (in yaml) fails due to internal variables, BUT:
+xperi3 = Experiment.from_file("experiment_from_yaml.yaml")
+print(f"xp3 hash: {xperi3.get_hash()} (won't match)")
+
+# Create a tasks-list for the testbed
+tb_tasks2 = TestbedTasks.from_xp(xperi2)
+tb_tasks2.to_file("experiment_tb_tasks.yaml")
+
+# Comparison between task-Lists succeed (experiment-comparison failed)
+tb_tasks3 = TestbedTasks.from_xp(xperi3)
+print(f"tasks2 hash: {tb_tasks2.get_hash()}")
+print(f"tasks3 hash: {tb_tasks3.get_hash()}")

shepherd_core-2024.4.2/examples/firmware_model.py

@@ -0,0 +1,49 @@
+"""This example shows ways to embed firmware into the data-model.
+
+Note: the new semi-automatic way to generate a data-model needs pwntools installed
+or shepherd-core[elf].
+"""
+
+from pathlib import Path
+
+from shepherd_core import TestbedClient
+from shepherd_core import fw_tools
+from shepherd_core.data_models import Firmware
+from shepherd_core.data_models import FirmwareDType
+
+path_elf = Path(__file__).parent.parent / "tests/fw_tools/build_msp.elf"
+
+# Option 1 - fully manual
+
+fw1 = Firmware(
+    name="msp_deep_sleep1",
+    data=fw_tools.file_to_base64(path_elf),
+    data_type=FirmwareDType.base64_elf,
+    mcu={"name": "MSP430FR"},
+    owner="example",
+    group="test",
+)
+fw1.to_file(Path(__file__).parent / "firmware.yaml")
+
+# Option 2 - semi-automatic -> MCU and data-type get derived
+
+fw2 = Firmware.from_firmware(
+    file=path_elf,
+    name="msp_deep_sleep2",
+    owner="example",
+    group="test",
+)
+
+# store embedded data
+path_elf2 = fw2.extract_firmware(Path(__file__).parent)
+print(f"stored firmware to '{path_elf2.name}'")
+
+
+# Option 3 - fully automatic (with login) -> owner and group get prefilled
+
+tb_client = TestbedClient()
+do_connect = False
+
+if do_connect:
+    tb_client.connect(token="your_personal_login_token")  # noqa: S106
+    fw3 = Firmware.from_firmware(file=path_elf, name="msp_deep_sleep3")

shepherd_core-2024.4.2/examples/firmware_modification.py

@@ -0,0 +1,43 @@
+"""This example manipulates elf-files and embeds firmware into the data-model.
+
+Note: make sure to have installed
+- shepherd-core[elf] via pip
+- build-essential or binutils-$ARCH installed.
+"""
+
+import shutil
+from pathlib import Path
+
+from shepherd_core import fw_tools
+from shepherd_core.data_models import Firmware
+from shepherd_core.data_models import FirmwareDType
+
+path_src = Path(__file__).parent.parent / "tests/fw_tools/build_msp.elf"
+path_elf = Path(__file__).with_name("firmware_msp.elf")
+
+# make local copy to play with
+shutil.copy(path_src, path_elf)
+
+print(f"UID old = 0x{fw_tools.read_uid(path_elf):X}")
+fw_tools.modify_uid(path_elf, 0xCAFE)
+print(f"UID new = 0x{fw_tools.read_uid(path_elf):X}")
+
+path_hex = fw_tools.elf_to_hex(path_elf)
+
+# just PoC - there is an easier way to generate data-model, see other fw-example
+b64 = fw_tools.file_to_base64(path_elf)
+
+fw = Firmware(
+    name="msp_deep_sleep",
+    data=b64,
+    data_type=FirmwareDType.base64_elf,
+    mcu={"name": "MSP430FR"},
+    owner="example",
+    group="test",
+)
+
+# for completion also generate hex for nrf
+path_src = Path(__file__).parent.parent / "tests/fw_tools/build_nrf.elf"
+path_elf = Path(__file__).with_name("firmware_nrf.elf")
+shutil.copy(path_src, path_elf)
+fw_tools.elf_to_hex(path_elf)

shepherd_core-2024.4.2/examples/inventory.py

@@ -0,0 +1,24 @@
+"""Shows how to use inventory functionality."""
+
+from pathlib import Path
+
+from shepherd_core.inventory import Inventory
+from shepherd_core.inventory import InventoryList
+from shepherd_core.inventory import PythonInventory
+from shepherd_core.inventory import SystemInventory
+from shepherd_core.inventory import TargetInventory
+
+pi = PythonInventory.collect()
+print(f"PyInv: {pi}")
+si = SystemInventory.collect()
+print(f"SysInv: {si}")
+ti = TargetInventory.collect()
+print(f"TgtInv: {ti}")
+
+
+inv = Inventory.collect()
+print(f"Complete Inventory: {inv}")
+inv.to_file("inventory.yaml", minimal=True, comment="just a test")
+
+inl = InventoryList(elements=[inv])
+inl.to_csv(Path(__file__).parent / "inventory.csv")

shepherd_core-2024.4.2/examples/uart_decode_waveform.py

@@ -0,0 +1,32 @@
+"""Demonstrates decoding of a sampled waveform to UART-symbols."""
+
+from pathlib import Path
+from timeit import timeit
+
+from shepherd_core import logger
+from shepherd_core.decoder_waveform import Uart
+
+# file captured with logic analyzer, 15.5k events (2700 symbols, 61 lines)
+trace = Path(__file__).parent / "uart_raw2.csv"
+uwd = Uart(trace)
+
+sym = uwd.get_symbols()
+lne = uwd.get_lines()
+txt = uwd.get_text()
+logger.info(txt)
+
+do_analysis = False
+if do_analysis:
+    l0 = timeit("Uart(trace)", globals=globals(), number=1000)
+    l1 = timeit("uwd.get_symbols(force_redo=True)", globals=globals(), number=100)
+    l2 = timeit("uwd.get_lines(force_redo=True)", globals=globals(), number=1000)
+    l3 = timeit("uwd.get_text(force_redo=True)", globals=globals(), number=1000)
+    print("t_init\t", l0)
+    print("t_symb\t", l1 * 10)
+    print("t_line\t", l2)
+    print("t_text\t", l3)
+    # Results:
+    # t_init  5.8   [ms/run]
+    # t_symb  70.4  [!!!!!]
+    # t_line  3.9
+    # t_text  0.1