shepherd-core 2024.7.1__tar.gz → 2024.7.3__tar.gz

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shepherd_core
-Version: 2024.7.1
+Version: 2024.7.3
 Summary: Programming- and CLI-Interface for the h5-dataformat of the Shepherd-Testbed
 Author-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>
 Maintainer-email: Ingmar Splitt <ingmar.splitt@tu-dresden.de>

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/experiment_generic_var1.py

@@ -15,7 +15,7 @@ What the code does:
 
 from pathlib import Path
 
-from shepherd_core import TestbedClient
+from shepherd_core import WebClient
 from shepherd_core.data_models import FirmwareDType
 from shepherd_core.data_models import GpioTracing
 from shepherd_core.data_models.content import EnergyEnvironment
@@ -25,16 +25,14 @@ from shepherd_core.data_models.experiment import TargetConfig
 from shepherd_core.data_models.task import TestbedTasks
 from shepherd_core.data_models.testbed import MCU
 
-# for online-queries the lib can be connected to the testbed-server
+# For online-queries the lib can be connected to the testbed-server.
 # NOTE: there are 3 states:
-#    - unconnected -> demo-fixture is queried (locally)
-#    - connected -> publicly available data is queried online
-#    - logged in with token -> also private data is queried online
-tb_client = TestbedClient()
+# - unconnected -> demo-fixtures are queried (locally)
+# - connected -> publicly available data is queried online
+# - logged in with valid token -> also private data is queried online
 do_connect = False
-
 if do_connect:
-    tb_client.connect()
+    WebClient()
 
 xp = Experiment(
     id="4567",

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/experiment_generic_var2.py

@@ -14,7 +14,7 @@
 
 from pathlib import Path
 
-from shepherd_core import TestbedClient
+from shepherd_core import WebClient
 from shepherd_core.data_models import GpioTracing
 from shepherd_core.data_models.content import EnergyEnvironment
 from shepherd_core.data_models.content import Firmware
@@ -22,16 +22,14 @@ from shepherd_core.data_models.experiment import Experiment
 from shepherd_core.data_models.experiment import TargetConfig
 from shepherd_core.data_models.task import TestbedTasks
 
-# for online-queries the lib can be connected to the testbed-server
+# For online-queries the lib can be connected to the testbed-server.
 # NOTE: there are 3 states:
-#    - unconnected -> demo-fixture is queried (locally)
-#    - connected -> publicly available data is queried online
-#    - logged in with token -> also private data is queried online
-tb_client = TestbedClient()
+# - unconnected -> demo-fixtures are queried (locally)
+# - connected -> publicly available data is queried online
+# - logged in with valid token -> also private data is queried online
 do_connect = False
-
 if do_connect:
-    tb_client.connect()
+    WebClient()
 
 xp = Experiment(
     id="4567",

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/experiment_models.py

@@ -18,7 +18,7 @@ How to define an experiment:
 
 """
 
-from shepherd_core import TestbedClient
+from shepherd_core import WebClient
 from shepherd_core.data_models.content import EnergyEnvironment
 from shepherd_core.data_models.content import Firmware
 from shepherd_core.data_models.content import VirtualHarvesterConfig
@@ -30,16 +30,14 @@ from shepherd_core.data_models.task import TestbedTasks
 # generate description for all parameters -> base for web-forms
 Experiment.schema_to_file("experiment_schema.yaml")
 
-# for online-queries the lib can be connected to the testbed-server
+# For online-queries the lib can be connected to the testbed-server.
 # NOTE: there are 3 states:
-#    - unconnected -> demo-fixture is queried (locally)
-#    - connected -> publicly available data is queried online
-#    - logged in with token -> also private data is queried online
-tb_client = TestbedClient()
+# - unconnected -> demo-fixtures are queried (locally)
+# - connected -> publicly available data is queried online
+# - logged in with valid token -> also private data is queried online
 do_connect = False
-
 if do_connect:
-    tb_client.connect()
+    WebClient()
 
 # Defining an Experiment in Python
 hrv = VirtualHarvesterConfig(name="mppt_bq_thermoelectric")

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/firmware_model.py

@@ -6,7 +6,7 @@ or shepherd-core[elf].
 
 from pathlib import Path
 
-from shepherd_core import TestbedClient
+from shepherd_core import WebClient
 from shepherd_core import fw_tools
 from shepherd_core.data_models import Firmware
 from shepherd_core.data_models import FirmwareDType
@@ -41,9 +41,7 @@ print(f"stored firmware to '{path_elf2.name}'")
 
 # Option 3 - fully automatic (with login) -> owner and group get prefilled
 
-tb_client = TestbedClient()
 do_connect = False
-
 if do_connect:
-    tb_client.connect(token="your_personal_login_token")  # noqa: S106
+    WebClient(token="your_personal_login_token")  # noqa: S106
     fw3 = Firmware.from_firmware(file=path_elf, name="msp_deep_sleep3")

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/vharvester_simulation.py

@@ -9,7 +9,6 @@
 from pathlib import Path
 
 from shepherd_core import Reader
-from shepherd_core import TestbedClient
 from shepherd_core.data_models import VirtualHarvesterConfig
 from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig
 from shepherd_core.vsource import VirtualHarvesterModel
@@ -33,17 +32,6 @@ hrv_list = [
     "mppt_opt",
 ]
 
-# for online-queries the lib can be connected to the testbed-server
-# NOTE: there are 3 states:
-#    - unconnected -> demo-fixture is queried (locally)
-#    - connected -> publicly available data is queried online
-#    - logged in with token -> also private data is queried online
-tb_client = TestbedClient()
-do_connect = False
-
-if do_connect:
-    tb_client.connect()
-
 # convert IVonne to IVCurve
 if not file_ivcurve.exists():
     with ivonne.Reader(file_ivonne) as db:

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/examples/vsource_simulation.py

@@ -16,7 +16,7 @@ import matplotlib.pyplot as plt
 import numpy as np
 
 from shepherd_core import CalibrationEmulator
-from shepherd_core import TestbedClient
+from shepherd_core import WebClient
 from shepherd_core.data_models import VirtualSourceConfig
 from shepherd_core.vsource import VirtualSourceModel
 
@@ -38,16 +38,14 @@ src_list = [
 I_mcu_sleep_A = 200e-9
 I_mcu_active_A = 1e-3
 
-# for online-queries the lib can be connected to the testbed-server
+# For online-queries the lib can be connected to the testbed-server.
 # NOTE: there are 3 states:
-#    - unconnected -> demo-fixture is queried (locally)
-#    - connected -> publicly available data is queried online
-#    - logged in with token -> also private data is queried online
-tb_client = TestbedClient()
+# - unconnected -> demo-fixtures are queried (locally)
+# - connected -> publicly available data is queried online
+# - logged in with valid token -> also private data is queried online
 do_connect = False
-
 if do_connect:
-    tb_client.connect()
+    WebClient()
 
 for vs_name, v_hrv_mV, samples in product(src_list, v_hrv_mV_list, sample_dur_list):
     # prepare simulation

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/pyproject.toml

@@ -94,7 +94,7 @@ shepherd_core = [
 ]
 
 [tool.setuptools.dynamic]
-version = {attr = "shepherd_core.__version__"}
+version = {attr = "shepherd_core.version.version"}
 
 [tool.pytest.ini_options]
 markers = [

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/__init__.py

@@ -19,11 +19,11 @@ from .logger import get_verbose_level
 from .logger import increase_verbose_level
 from .logger import logger
 from .reader import Reader
-from .testbed_client.client import TestbedClient
-from .testbed_client.client import tb_client
+from .testbed_client.client_web import WebClient
+from .version import version
 from .writer import Writer
 
-__version__ = "2024.7.1"
+__version__ = version
 
 __all__ = [
     "Reader",
@@ -40,7 +40,6 @@ __all__ = [
     "local_now",
     "Calc_t",
     "Compression",
-    "TestbedClient",
-    "tb_client",  # using this (instead of the Class) is the cleaner, but less pythonic way
+    "WebClient",
     "Inventory",
 ]

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/base/wrapper.py

@@ -7,6 +7,8 @@ from pydantic import BaseModel
 from pydantic import StringConstraints
 from typing_extensions import Annotated
 
+from ...version import version
+
 SafeStrClone = Annotated[str, StringConstraints(pattern=r"^[ -~]+$")]
 # ⤷ copy avoids circular import
 
@@ -19,5 +21,7 @@ class Wrapper(BaseModel):
     comment: Optional[SafeStrClone] = None
     created: Optional[datetime] = None
     # ⤷ Optional metadata
+    lib_ver: Optional[str] = version
+    # ⤷ for debug-purposes and later comp-checks
     parameters: dict
     # ⤷ ShpModel

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/content/energy_environment.py

@@ -35,7 +35,7 @@ class EnergyEnvironment(ContentModel):
 
     # TODO: scale up/down voltage/current
 
-    # additional descriptive metadata
+    # additional descriptive metadata, TODO: these are very solar-centered -> generalize
     light_source: Optional[str] = None
     weather_conditions: Optional[str] = None
     indoor: Optional[bool] = None

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/content/firmware.py

@@ -152,6 +152,7 @@ class Firmware(ContentModel, title="Firmware of Target"):
 
         if not match:
             logger.warning("FW-Hash does not match with stored value!")
+            # TODO: it might be more appropriate to raise here
         return match
 
     @validate_call

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/experiment/experiment.py

@@ -13,6 +13,7 @@ from pydantic import model_validator
 from typing_extensions import Annotated
 from typing_extensions import Self
 
+from ...version import version
 from ..base.content import IdInt
 from ..base.content import NameStr
 from ..base.content import SafeStr
@@ -54,18 +55,20 @@ class Experiment(ShpModel, title="Config of an Experiment"):
     # targets
     target_configs: Annotated[List[TargetConfig], Field(min_length=1, max_length=128)]
 
-    # TODO: we probably need to remember the lib-version for content &| experiment
+    # for debug-purposes and later comp-checks
+    lib_ver: Optional[str] = version
 
     @model_validator(mode="after")
     def post_validation(self) -> Self:
-        self.validate_targets(self.target_configs)
-        self.validate_observers(self.target_configs)
+        testbed = Testbed()  # this will query the first (and only) entry of client
+        self._validate_targets(self.target_configs)
+        self._validate_observers(self.target_configs, testbed)
         if self.duration and self.duration.total_seconds() < 0:
             raise ValueError("Duration of experiment can't be negative.")
         return self
 
     @staticmethod
-    def validate_targets(configs: List[TargetConfig]) -> None:
+    def _validate_targets(configs: List[TargetConfig]) -> None:
         target_ids = []
         custom_ids = []
         for _config in configs:
@@ -83,10 +86,8 @@ class Experiment(ShpModel, title="Config of an Experiment"):
             raise ValueError("Custom Target-ID are faulty (some form of id-collisions)!")
 
     @staticmethod
-    def validate_observers(configs: List[TargetConfig]) -> None:
+    def _validate_observers(configs: List[TargetConfig], testbed: Testbed) -> None:
         target_ids = [_id for _config in configs for _id in _config.target_IDs]
-
-        testbed = Testbed(name="shepherd_tud_nes")
         obs_ids = [testbed.get_observer(_id).id for _id in target_ids]
         if len(target_ids) > len(set(obs_ids)):
             raise ValueError(

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/task/testbed_tasks.py

@@ -32,8 +32,9 @@ class TestbedTasks(ShpModel):
     @validate_call
     def from_xp(cls, xp: Experiment, tb: Optional[Testbed] = None) -> Self:
         if tb is None:
-            # TODO: just for testing OK
-            tb = Testbed(name="shepherd_tud_nes")
+            # TODO: is tb-argument really needed? prob. not
+            tb = Testbed()  # this will query the first (and only) entry of client
+
         tgt_ids = xp.get_target_ids()
         obs_tasks = [ObserverTasks.from_xp(xp, tb, _id) for _id in tgt_ids]
         return cls(

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/data_models/testbed/testbed.py

@@ -11,6 +11,7 @@ from pydantic import model_validator
 from typing_extensions import Annotated
 from typing_extensions import Self
 
+from ... import logger
 from ...testbed_client import tb_client
 from ..base.content import IdInt
 from ..base.content import NameStr
@@ -42,6 +43,14 @@ class Testbed(ShpModel):
     @model_validator(mode="before")
     @classmethod
     def query_database(cls, values: dict) -> dict:
+        # allow instantiating an empty Testbed
+        #   -> query the first (and only) entry of client
+        if len(values) == 0:
+            ids = tb_client.query_ids(cls.__name__)
+            if len(ids) > 1:
+                logger.warning("More than one testbed defined?!?")
+            values = {"id": ids[0]}
+
         values, _ = tb_client.try_completing_model(cls.__name__, values)
         return values
 

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/inventory/system.py

@@ -5,6 +5,7 @@ import subprocess
 import time
 from datetime import datetime
 from pathlib import Path
+from typing import List
 from typing import Optional
 
 from typing_extensions import Self
@@ -48,8 +49,8 @@ class SystemInventory(ShpModel):
     #   ip IPvAnyAddress
     #   mac MACStr
 
-    fs_root: Optional[str] = None
-    beagle: Optional[str] = None
+    fs_root: List[str] = None
+    beagle: List[str] = None
 
     model_config = ConfigDict(str_min_length=0)
 
@@ -71,20 +72,20 @@ class SystemInventory(ShpModel):
             uptime = time.time() - psutil.boot_time()
 
         fs_cmd = ["/usr/bin/df", "-h", "/"]
-        fs_out = None
+        fs_out = []
         if Path(fs_cmd[0]).is_file():
             reply = subprocess.run(  # noqa: S603
                 fs_cmd, timeout=30, capture_output=True, check=False
             )
-            fs_out = str(reply.stdout)
+            fs_out = str(reply.stdout).split(r"\n")
 
         beagle_cmd = ["/usr/bin/beagle-version"]
-        beagle_out = None
+        beagle_out = []
         if Path(beagle_cmd[0]).is_file():
             reply = subprocess.run(  # noqa: S603
                 beagle_cmd, timeout=30, capture_output=True, check=False
             )
-            beagle_out = str(reply.stdout)
+            beagle_out = str(reply.stdout).split(r"\n")
 
         ptp_cmd = ["/usr/sbin/ptp4l", "-v"]
         ptp_out = None

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/reader.py

@@ -536,9 +536,9 @@ class Reader:
     def count_errors_in_log(self, group_name: str = "sheep", min_level: int = 40) -> int:
         if group_name not in self.h5file:
             return 0
-        if "level" not in self.h5file["sheep"]:
+        if "level" not in self.h5file[group_name]:
             return 0
-        _lvl = self.h5file["sheep"]["level"]
+        _lvl = self.h5file[group_name]["level"]
         if _lvl.shape[0] < 1:
             return 0
         _items = [1 for _x in _lvl[:] if _x >= min_level]
@@ -688,4 +688,11 @@ class Reader:
         gpio_wf = pin_wf.astype(float)
         gpio_wf[:, 0] = gpio_wf[:, 0] / 1e9
 
-        return Uart(gpio_wf).get_lines()
+        try:
+            return Uart(gpio_wf).get_lines()
+        except TypeError:
+            self._logger.error("TypeError: Extracting UART from GPIO failed - will skip file.")
+            return None
+        except ValueError:
+            self._logger.error("ValueError: Extracting UART from GPIO failed - will skip file.")
+            return None

{shepherd_core-2024.7.1 → shepherd_core-2024.7.3}/shepherd_core/testbed_client/__init__.py

@@ -1,9 +1,11 @@
 """Client to access a testbed-instance for controlling experiments."""
 
-from .client import tb_client
+from .client_abc_fix import tb_client
+from .client_web import WebClient
 from .user_model import User
 
 __all__ = [
     "tb_client",
+    "WebClient",
     "User",
 ]

shepherd_core-2024.7.3/shepherd_core/testbed_client/client_abc_fix.py

@@ -0,0 +1,126 @@
+"""AbstractBase-Class & Client-Class to access the file based fixtures.
+
+Fixtures == OffLineDemoInstances
+offline:	core  - fixtClient
+webDev:	    core  - webClient  <->	webSrv - fixtClient
+webUser:	core  - webClient  <->  webSrv - DbClient
+webInfra:   core  - webClient+ <->  webSrv - DbClient
+
+Users, Sheep and ServerApps should have access to the same DB via WebClient
+
+Note: ABC and FixClient can't be in separate files when tb_client should
+      default to FixClient (circular import)
+
+TODO: Comfort functions missing
+    - fixtures to DB, and vice versa
+"""
+
+from abc import ABC
+from abc import abstractmethod
+from typing import List
+from typing import Optional
+
+from ..data_models.base.shepherd import ShpModel
+from ..data_models.base.wrapper import Wrapper
+from .fixtures import Fixtures
+
+
+class AbcClient(ABC):
+    """AbstractBase-Class to access a testbed instance."""
+
+    def __init__(self) -> None:
+        global tb_client  # noqa: PLW0603
+        tb_client = self
+
+    @abstractmethod
+    def insert(self, data: ShpModel) -> bool:
+        """Insert (and probably replace) entry.
+
+        TODO: fixtures get replaced, but is that wanted for web?
+        """
+
+    @abstractmethod
+    def query_ids(self, model_type: str) -> List[int]:
+        pass
+
+    @abstractmethod
+    def query_names(self, model_type: str) -> List[str]:
+        pass
+
+    @abstractmethod
+    def query_item(
+        self, model_type: str, uid: Optional[int] = None, name: Optional[str] = None
+    ) -> dict:
+        pass
+
+    @abstractmethod
+    def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
+        # TODO: maybe internal? yes
+        pass
+
+    def try_completing_model(self, model_type: str, values: dict) -> (dict, list):
+        """Init by name/id, for none existing instances raise Exception.
+
+        This is the main entry-point for querying a model (used be the core-lib).
+        """
+        if len(values) == 1 and next(iter(values.keys())) in {"id", "name"}:
+            try:
+                values = self.query_item(model_type, name=values.get("name"), uid=values.get("id"))
+            except ValueError as err:
+                raise ValueError(
+                    "Query %s by name / ID failed - %s is unknown!", model_type, values
+                ) from err
+        return self.try_inheritance(model_type, values)
+
+    @abstractmethod
+    def fill_in_user_data(self, values: dict) -> dict:
+        # TODO: is it really helpful and needed?
+        pass
+
+
+class FixturesClient(AbcClient):
+    """Client-Class to access the file based fixtures."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._fixtures: Optional[Fixtures] = Fixtures()
+
+    def insert(self, data: ShpModel) -> bool:
+        wrap = Wrapper(
+            datatype=type(data).__name__,
+            parameters=data.model_dump(),
+        )
+        self._fixtures.insert_model(wrap)
+        return True
+
+    def query_ids(self, model_type: str) -> List[int]:
+        return list(self._fixtures[model_type].elements_by_id.keys())
+
+    def query_names(self, model_type: str) -> List[str]:
+        return list(self._fixtures[model_type].elements_by_name.keys())
+
+    def query_item(
+        self, model_type: str, uid: Optional[int] = None, name: Optional[str] = None
+    ) -> dict:
+        if uid is not None:
+            return self._fixtures[model_type].query_id(uid)
+        if name is not None:
+            return self._fixtures[model_type].query_name(name)
+        raise ValueError("Query needs either uid or name of object")
+
+    def try_inheritance(self, model_type: str, values: dict) -> (dict, list):
+        return self._fixtures[model_type].inheritance(values)
+
+    def fill_in_user_data(self, values: dict) -> dict:
+        """Add fake user-data when offline-client is used.
+
+        Hotfix until WebClient is working.
+        """
+        if values.get("owner") is None:
+            values["owner"] = "unknown"
+        if values.get("group") is None:
+            values["group"] = "unknown"
+        return values
+
+
+tb_client: AbcClient = FixturesClient()