shepherd-core 2023.11.1__py3-none-any.whl → 2024.4.1__py3-none-any.whl

shepherd_core/data_models/task/testbed_tasks.py

@@ -1,12 +1,12 @@
 from typing import List
 from typing import Optional
 
-from pydantic import EmailStr
 from pydantic import Field
 from pydantic import validate_call
 from typing_extensions import Annotated
 from typing_extensions import Self
 
+from ..base.content import IdInt
 from ..base.content import NameStr
 from ..base.shepherd import ShpModel
 from ..experiment.experiment import Experiment
@@ -21,7 +21,10 @@ class TestbedTasks(ShpModel):
     observer_tasks: Annotated[List[ObserverTasks], Field(min_length=1, max_length=128)]
 
     # POST PROCESS
-    email: Optional[EmailStr] = None
+    email_results: bool = False
+    owner_id: Optional[IdInt]
+    # TODO: had real email previously, does it really need these at all?
+    #  DB stores experiment and knows when to email
 
     @classmethod
     @validate_call
@@ -34,7 +37,8 @@ class TestbedTasks(ShpModel):
         return cls(
             name=xp.name,
             observer_tasks=obs_tasks,
-            email=xp.email_results,
+            email_results=xp.email_results,
+            owner_id=xp.owner_id,
         )
 
     def get_observer_tasks(self, observer: str) -> Optional[ObserverTasks]:

shepherd_core/data_models/testbed/cape.py

@@ -24,7 +24,7 @@ class TargetPort(str, Enum):
 class Cape(ShpModel, title="Shepherd-Cape"):
     """meta-data representation of a testbed-component (physical object)"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     version: NameStr
     description: SafeStr

shepherd_core/data_models/testbed/gpio.py

@@ -26,7 +26,7 @@ class Direction(str, Enum):
 class GPIO(ShpModel, title="GPIO of Observer Node"):
     """meta-data representation of a testbed-component"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     description: Optional[SafeStr] = None
     comment: Optional[SafeStr] = None

shepherd_core/data_models/testbed/mcu.py

@@ -26,7 +26,7 @@ class ProgrammerProtocol(str, Enum):
 class MCU(ShpModel, title="Microcontroller of the Target Node"):
     """meta-data representation of a testbed-component (physical object)"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     description: SafeStr
     comment: Optional[SafeStr] = None

shepherd_core/data_models/testbed/observer.py

@@ -19,16 +19,14 @@ from .target import Target
 
 MACStr = Annotated[
     str,
-    StringConstraints(
-        max_length=17, pattern=r"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"
-    ),
+    StringConstraints(max_length=17, pattern=r"^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$"),
 ]
 
 
 class Observer(ShpModel, title="Shepherd-Sheep"):
     """meta-data representation of a testbed-component (physical object)"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     description: SafeStr
     comment: Optional[SafeStr] = None
@@ -65,23 +63,13 @@ class Observer(ShpModel, title="Shepherd-Sheep"):
         has_cape = self.cape is not None
         has_target = (self.target_a is not None) or (self.target_b is not None)
         if not has_cape and has_target:
-            raise ValueError(
-                f"Observer '{self.name}' is faulty " f"-> has targets but no cape"
-            )
+            raise ValueError(f"Observer '{self.name}' is faulty -> has targets but no cape")
         return self
 
     def has_target(self, target_id: int) -> bool:
-        if (
-            self.target_a is not None
-            and target_id == self.target_a.id
-            and self.target_a.active
-        ):
+        if self.target_a is not None and target_id == self.target_a.id and self.target_a.active:
             return True
-        if (
-            self.target_b is not None
-            and target_id == self.target_b.id
-            and self.target_b.active
-        ):
+        if self.target_b is not None and target_id == self.target_b.id and self.target_b.active:
             return True
         return False
 
@@ -91,9 +79,7 @@ class Observer(ShpModel, title="Shepherd-Sheep"):
                 return TargetPort.A
             if self.target_b is not None and target_id == self.target_b.id:
                 return TargetPort.B
-        raise ValueError(
-            f"Target-ID {target_id} was not found in Observer '{self.name}'"
-        )
+        raise ValueError(f"Target-ID {target_id} was not found in Observer '{self.name}'")
 
     def get_target(self, target_id: int) -> Target:
         if self.has_target(target_id):
@@ -101,6 +87,4 @@ class Observer(ShpModel, title="Shepherd-Sheep"):
                 return self.target_a
             if self.target_b is not None and target_id == self.target_b.id:
                 return self.target_b
-        raise ValueError(
-            f"Target-ID {target_id} was not found in Observer '{self.name}'"
-        )
+        raise ValueError(f"Target-ID {target_id} was not found in Observer '{self.name}'")

shepherd_core/data_models/testbed/target.py

@@ -21,7 +21,7 @@ MCUPort = Annotated[int, Field(ge=1, le=2)]
 class Target(ShpModel, title="Target Node (DuT)"):
     """meta-data representation of a testbed-component (physical object)"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     version: NameStr
     description: SafeStr

shepherd_core/data_models/testbed/testbed.py

@@ -20,7 +20,7 @@ from .observer import Observer
 class Testbed(ShpModel):
     """meta-data representation of a testbed-component (physical object)"""
 
-    id: IdInt  # noqa: A003
+    id: IdInt
     name: NameStr
     description: SafeStr
     comment: Optional[SafeStr] = None
@@ -87,6 +87,4 @@ class Testbed(ShpModel):
                 continue
             if _observer.has_target(target_id):
                 return _observer
-        raise ValueError(
-            f"Target-ID {target_id} was not found in Testbed '{self.name}'"
-        )
+        raise ValueError(f"Target-ID {target_id} was not found in Testbed '{self.name}'")

shepherd_core/decoder_waveform/uart.py

@@ -17,6 +17,7 @@ https://en.wikipedia.org/wiki/Universal_asynchronous_receiver-transmitter
 https://sigrok.org/wiki/Protocol_decoder:Uart
 
 """
+
 from enum import Enum
 from pathlib import Path
 from typing import Optional
@@ -56,9 +57,7 @@ class Uart:
           (some detectors still missing)
         """
         if isinstance(content, Path):
-            self.events_sig: np.ndarray = np.loadtxt(
-                content.as_posix(), delimiter=",", skiprows=1
-            )
+            self.events_sig: np.ndarray = np.loadtxt(content.as_posix(), delimiter=",", skiprows=1)
             # TODO: if float fails load as str -
             #  cast first col as np.datetime64 with ns-resolution, convert to delta
         else:
@@ -66,9 +65,7 @@ class Uart:
 
         # verify table
         if self.events_sig.shape[1] != 2:
-            raise TypeError(
-                "Input file should have 2 rows -> (comma-separated) timestamp & value"
-            )
+            raise TypeError("Input file should have 2 rows -> (comma-separated) timestamp & value")
         if self.events_sig.shape[0] < 8:
             raise TypeError("Input file is too short (< state-changes)")
         # verify timestamps
@@ -80,23 +77,17 @@ class Uart:
         self._convert_analog2digital()
         self._filter_redundant_states()
 
-        self.baud_rate: int = (
-            baud_rate if baud_rate is not None else self.detect_baud_rate()
-        )
+        self.baud_rate: int = baud_rate if baud_rate is not None else self.detect_baud_rate()
         self.dur_tick: float = 1.0 / self.baud_rate
 
         self._add_duration()
 
-        self.inversion: bool = (
-            inversion if inversion is not None else self.detect_inversion()
-        )
+        self.inversion: bool = inversion if inversion is not None else self.detect_inversion()
         self.half_stop: bool = self.detect_half_stop()  # not needed ATM
 
         # TODO: add detectors
         self.parity: Parity = parity if parity is not None else Parity.no
-        self.bit_order: BitOrder = (
-            bit_order if bit_order is not None else BitOrder.lsb_first
-        )
+        self.bit_order: BitOrder = bit_order if bit_order is not None else BitOrder.lsb_first
         self.frame_length: int = frame_length if frame_length is not None else 8
 
         if not (0 < self.frame_length <= 64):
@@ -146,14 +137,10 @@ class Uart:
             logger.warning("Tried to add state-duration, but it seems already present")
             return
         if not hasattr(self, "dur_tick"):
-            raise ValueError(
-                "Make sure that baud-rate was calculated before running add_dur()"
-            )
+            raise ValueError("Make sure that baud-rate was calculated before running add_dur()")
         dur_steps = self.events_sig[1:, 0] - self.events_sig[:-1, 0]
         dur_steps = np.reshape(dur_steps, (dur_steps.size, 1))
-        self.events_sig = np.append(
-            self.events_sig[:-1, :], dur_steps / self.dur_tick, axis=1
-        )
+        self.events_sig = np.append(self.events_sig[:-1, :], dur_steps / self.dur_tick, axis=1)
 
     def detect_inversion(self) -> bool:
         """Analyze bit-state during long pauses (unchanged states)
@@ -182,16 +169,12 @@ class Uart:
     def detect_half_stop(self) -> bool:
         """Looks into the spacing between time-steps"""
         events = self.events_sig[:1000, :]  # speedup for large datasets
-        return (
-            np.sum((events > 1.333 * self.dur_tick) & (events < 1.667 * self.dur_tick))
-            > 0
-        )
+        return np.sum((events > 1.333 * self.dur_tick) & (events < 1.667 * self.dur_tick)) > 0
 
     def detect_dataframe_length(self) -> int:
         """Look after longest pauses
         - accumulate steps until a state with uneven step-size is found
         """
-        pass
 
     def get_symbols(self, *, force_redo: bool = False) -> np.ndarray:
         """Ways to detect EOF:

shepherd_core/fw_tools/__init__.py

@@ -25,9 +25,7 @@ except ImportError:
         "cffi",
     ]
     # only update when module is not avail
-    MOCK_MODULES = [
-        mod_name for mod_name in MOCK_MODULES if find_spec(mod_name) is None
-    ]
+    MOCK_MODULES = [mod_name for mod_name in MOCK_MODULES if find_spec(mod_name) is None]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 from .converter import base64_to_file

shepherd_core/fw_tools/converter.py

@@ -22,9 +22,7 @@ def firmware_to_hex(file_path: Path) -> Path:
         return elf_to_hex(file_path)
     if is_hex(file_path):
         return file_path
-    raise ValueError(
-        "FW2Hex: unknown file '%s', it should be ELF or HEX", file_path.name
-    )
+    raise ValueError("FW2Hex: unknown file '%s', it should be ELF or HEX", file_path.name)
 
 
 @validate_call
@@ -72,9 +70,7 @@ def base64_to_hash(content: str) -> str:
 
 
 @validate_call
-def extract_firmware(
-    data: Union[str, Path], data_type: FirmwareDType, file_path: Path
-) -> Path:
+def extract_firmware(data: Union[str, Path], data_type: FirmwareDType, file_path: Path) -> Path:
     """- base64-string will be transformed into file
     - if data is a path the file will be copied to the destination
     """
@@ -90,14 +86,10 @@ def extract_firmware(
         elif data_type == FirmwareDType.path_hex:
             file = file_path.with_suffix(".hex")
         else:
-            raise ValueError(
-                "FW-Extraction failed due to unknown datatype '%s'", data_type
-            )
+            raise ValueError("FW-Extraction failed due to unknown datatype '%s'", data_type)
         if not file.parent.exists():
             file.parent.mkdir(parents=True)
         shutil.copy(data, file)
     else:
-        raise ValueError(
-            "FW-Extraction failed due to unknown data-type '%s'", type(data)
-        )
+        raise ValueError("FW-Extraction failed due to unknown data-type '%s'", type(data))
     return file

shepherd_core/fw_tools/patcher.py

@@ -46,9 +46,7 @@ def find_symbol(file_elf: Path, symbol: str) -> bool:
 
 
 @validate_call
-def read_symbol(
-    file_elf: Path, symbol: str, length: int = uid_len_default
-) -> Optional[int]:
+def read_symbol(file_elf: Path, symbol: str, length: int = uid_len_default) -> Optional[int]:
     """Interpreted as int"""
     if not find_symbol(file_elf, symbol):
         return None
@@ -99,9 +97,7 @@ def modify_symbol_value(
     value_raw = elf.read(address=addr, count=uid_len_default)[-uid_len_default:]
     # ⤷ cutting needed -> msp produces 4b instead of 2
     value_old = int.from_bytes(bytes=value_raw, byteorder=elf.endian, signed=False)
-    value_raw = value.to_bytes(
-        length=uid_len_default, byteorder=elf.endian, signed=False
-    )
+    value_raw = value.to_bytes(length=uid_len_default, byteorder=elf.endian, signed=False)
     try:
         elf.write(address=addr, data=value_raw)
     except AttributeError:
@@ -110,9 +106,7 @@ def modify_symbol_value(
     if overwrite:
         file_new = file_elf
     else:
-        file_new = file_elf.with_name(
-            file_elf.stem + "_" + str(value) + file_elf.suffix
-        )
+        file_new = file_elf.with_name(file_elf.stem + "_" + str(value) + file_elf.suffix)
         # could be simplified, but py3.8-- doesn't know .with_stem()
     elf.save(path=file_new)
     elf.close()
@@ -127,6 +121,4 @@ def modify_symbol_value(
 
 
 def modify_uid(file_elf: Path, value: int) -> Optional[Path]:
-    return modify_symbol_value(
-        file_elf, symbol=uid_str_default, value=value, overwrite=True
-    )
+    return modify_symbol_value(file_elf, symbol=uid_str_default, value=value, overwrite=True)

shepherd_core/fw_tools/validation.py

@@ -1,6 +1,5 @@
-"""TODO: Work in Progress
+"""TODO: Work in Progress"""
 
-"""
 import tempfile
 from pathlib import Path
 

shepherd_core/inventory/__init__.py

@@ -3,6 +3,9 @@
 - system-parameters
 - hardware-config
 """
+
+from datetime import datetime
+from datetime import timedelta
 from pathlib import Path
 from typing import List
 
@@ -26,20 +29,20 @@ __all__ = [
 
 class Inventory(PythonInventory, SystemInventory, TargetInventory):
     # has all child-parameters
+    hostname: str
+    created: datetime
 
     @classmethod
     def collect(cls) -> Self:
         # one by one for more precise error messages
-        pid = PythonInventory.collect().model_dump(
-            exclude_unset=True, exclude_defaults=True
-        )
-        sid = SystemInventory.collect().model_dump(
-            exclude_unset=True, exclude_defaults=True
-        )
-        tid = TargetInventory.collect().model_dump(
-            exclude_unset=True, exclude_defaults=True
-        )
+        # NOTE: system is first, as it must take a precise timestamp
+        sid = SystemInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
+        pid = PythonInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
+        tid = TargetInventory.collect().model_dump(exclude_unset=True, exclude_defaults=True)
         model = {**pid, **sid, **tid}
+        # make important metadata available at root level
+        model["created"] = sid["timestamp"]
+        model["hostname"] = sid["hostname"]
         return cls(**model)
 
 
@@ -59,3 +62,44 @@ class InventoryList(ShpModel):
                 content = list(item.model_dump().values())
                 content = ["" if value is None else str(value) for value in content]
                 fd.write(", ".join(content) + "\r\n")
+
+    def warn(self) -> dict:
+        warnings = {}
+        ts_earl = min([_e.created.timestamp() for _e in self.elements])
+        for _e in self.elements:
+            if _e.uptime > timedelta(hours=30).total_seconds():
+                warnings["uptime"] = f"[{self.hostname}] restart is recommended"
+            if (_e.created.timestamp() - ts_earl) > 10:
+                warnings["time_delta"] = f"[{self.hostname}] time-sync has failed"
+
+        # turn  dict[hostname][type] = val
+        # to    dict[type][val] = list[hostnames]
+        _inp = {
+            _e.hostname: _e.model_dump(exclude_unset=True, exclude_defaults=True)
+            for _e in self.elements
+        }
+        result = {}
+        for _host, _types in _inp.items():
+            for _type, _val in _types.items():
+                if _type not in result:
+                    result[_type] = {}
+                if _val not in result[_type]:
+                    result[_type][_val] = []
+                result[_type][_val].append(_host)
+        rescnt = {_key: len(_val) for _key, _val in result.items()}
+        t_unique = [
+            "h5py",
+            "numpy",
+            "pydantic",
+            "python",
+            "shepherd_core",
+            "shepherd_sheep",
+            "yaml",
+            "zstandard",
+        ]
+        for _key in t_unique:
+            if rescnt[_key] > 1:
+                warnings[_key] = f"[{_key}] VersionMismatch - {result[_key]}"
+
+        # TODO: finish with more potential warnings
+        return warnings

shepherd_core/inventory/system.py

@@ -2,10 +2,12 @@ import platform
 import subprocess
 import time
 from contextlib import suppress
+from datetime import datetime
 from typing import Optional
 
 from typing_extensions import Self
 
+from .. import local_now
 from .. import logger
 
 try:
@@ -22,6 +24,9 @@ from ..data_models import ShpModel
 class SystemInventory(ShpModel):
     uptime: PositiveInt
     # ⤷ seconds
+    timestamp: datetime
+    # time_delta: timedelta = timedelta(0)  # noqa: ERA001
+    # ⤷ lag behind earliest observer, TODO: wrong place
 
     system: str
     release: str
@@ -43,6 +48,8 @@ class SystemInventory(ShpModel):
 
     @classmethod
     def collect(cls) -> Self:
+        ts = local_now()
+
         if psutil is None:
             ifs2 = {}
             uptime = 0
@@ -53,15 +60,12 @@ class SystemInventory(ShpModel):
             )
         else:
             ifs1 = psutil.net_if_addrs().items()
-            ifs2 = {
-                name: (_if[1].address, _if[0].address)
-                for name, _if in ifs1
-                if len(_if) > 1
-            }
+            ifs2 = {name: (_if[1].address, _if[0].address) for name, _if in ifs1 if len(_if) > 1}
             uptime = time.time() - psutil.boot_time()
 
         model_dict = {
             "uptime": round(uptime),
+            "timestamp": ts,
             "system": platform.system(),
             "release": platform.release(),
             "version": platform.version(),
@@ -69,6 +73,7 @@ class SystemInventory(ShpModel):
             "processor": platform.processor(),
             "hostname": platform.node(),
             "interfaces": ifs2,
+            # TODO: add free space on /
         }
 
         with suppress(FileNotFoundError):

shepherd_core/logger.py

@@ -15,9 +15,7 @@ def get_verbose_level() -> int:
     return verbose_level
 
 
-def set_log_verbose_level(
-    log_: Union[logging.Logger, logging.Handler], verbose: int
-) -> None:
+def set_log_verbose_level(log_: Union[logging.Logger, logging.Handler], verbose: int) -> None:
     if verbose == 0:
         log_.setLevel(logging.ERROR)
         logging.basicConfig(level=logging.ERROR)