marilib-pkg 0.6.0__py3-none-any.whl

marilib/latency.py

@@ -0,0 +1,78 @@
+import struct
+import threading
+import time
+from typing import TYPE_CHECKING
+import math
+
+from marilib.mari_protocol import Frame
+
+if TYPE_CHECKING:
+    from marilib.marilib import MariLib
+
+LATENCY_PACKET_MAGIC = b"\x4c\x54"  # "LT" for Latency Test
+
+
+class LatencyTester:
+    """A thread-based class to periodically test latency to all nodes."""
+
+    def __init__(self, marilib: "MariLib", interval: float = 10.0):
+        self.marilib = marilib
+        self.interval = interval
+        self._stop_event = threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+
+    def start(self):
+        """Starts the latency testing thread."""
+        print("[yellow]Latency tester started.[/]")
+        self._thread.start()
+
+    def stop(self):
+        """Stops the latency testing thread."""
+        self._stop_event.set()
+        self._thread.join()
+        print("[yellow]Latency tester stopped.[/]")
+
+    def _run(self):
+        """The main loop for the testing thread."""
+        while not self._stop_event.is_set():
+            if not self.marilib.gateway.nodes:
+                time.sleep(self.interval)
+                continue
+
+            for node in list(self.marilib.gateway.nodes):
+                if self._stop_event.is_set():
+                    break
+                self.send_latency_request(node.address)
+                time.sleep(self.interval / len(self.marilib.gateway.nodes))
+
+    def send_latency_request(self, address: int):
+        """Sends a latency request packet to a specific address."""
+
+        payload = LATENCY_PACKET_MAGIC + struct.pack("<d", time.time())
+        self.marilib.send_frame(address, payload)
+
+    def handle_response(self, frame: Frame):
+        """
+        Processes a latency response frame.
+        This should be called when a LATENCY_DATA event is received.
+        """
+        if not frame.payload.startswith(LATENCY_PACKET_MAGIC):
+            return
+        try:
+            # Unpack the original timestamp from the payload
+            original_ts = struct.unpack("<d", frame.payload[2:10])[0]
+            rtt = time.time() - original_ts
+            if math.isnan(rtt) or math.isinf(rtt):
+                return  # Ignore corrupted/invalid packets
+            if rtt < 0 or rtt > 5.0:
+                return  # Ignore this outlier
+
+            node = self.marilib.gateway.get_node(frame.header.source)
+            if node:
+                # Update statistics for both the specific node and the whole gateway
+                node.latency_stats.add_latency(rtt)
+                self.marilib.gateway.latency_stats.add_latency(rtt)
+
+        except (struct.error, IndexError):
+            # Ignore packets that are too short or malformed
+            pass

marilib/logger.py

@@ -0,0 +1,211 @@
+import csv
+import os
+from dataclasses import dataclass, field
+from datetime import datetime, timedelta
+from typing import IO, List, Dict
+
+from marilib.model import MariGateway, MariNode
+
+
+@dataclass
+class MetricsLogger:
+    """
+    A metrics logger that saves statistics to CSV files with log rotation.
+    """
+
+    log_dir_base: str = "logs"
+    rotation_interval_minutes: int = 1440  # 1 day
+    already_logged_setup_parameters: bool = False
+    log_interval_seconds: float = 1.0
+    last_log_time: Dict[int, datetime] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """
+        Initializes the logger with rotation and setup logging capabilities.
+        """
+        try:
+            self.rotation_interval = timedelta(minutes=self.rotation_interval_minutes)
+
+            self.start_time = datetime.now()
+            self.run_timestamp = self.start_time.strftime("%Y%m%d_%H%M%S")
+            self.log_dir = os.path.join(self.log_dir_base, f"run_{self.run_timestamp}")
+            os.makedirs(self.log_dir, exist_ok=True)
+
+            self._gateway_file: IO[str] | None = None
+            self._nodes_file: IO[str] | None = None
+            self._events_file: IO[str] | None = None
+            self._gateway_writer = None
+            self._nodes_writer = None
+            self._events_writer = None
+            self.segment_start_time: datetime | None = None
+
+            # Open events log file
+            events_path = os.path.join(self.log_dir, "log_events.csv")
+            self._events_file = open(events_path, "w", newline="", encoding="utf-8")
+            self._events_writer = csv.writer(self._events_file)
+            self._events_writer.writerow(
+                ["timestamp", "gateway_address", "node_address", "event_name", "event_tag"]
+            )
+
+            self._open_new_segment()
+            self.active = True
+
+        except (IOError, OSError) as e:
+            print(f"Error: Failed to initialize logger: {e}")
+            self.active = False
+
+    def log_setup_parameters(self, params: Dict[str, any] | None):
+        """Creates and writes test setup parameters to metrics_setup.csv."""
+        if not params or self.already_logged_setup_parameters:
+            return
+        # only log setup parameters once
+        self.already_logged_setup_parameters = True
+
+        setup_path = os.path.join(self.log_dir, "metrics_setup.csv")
+        with open(setup_path, "w", newline="", encoding="utf-8") as f:
+            writer = csv.writer(f)
+            writer.writerow(["param", "value"])
+            writer.writerow(["start_time", self.start_time.isoformat()])
+            for key, value in params.items():
+                writer.writerow([key, value])
+
+    def _open_new_segment(self):
+        self._close_segment_files()
+
+        self.segment_start_time = datetime.now()
+        segment_ts = self.segment_start_time.strftime("%H%M%S")
+
+        gateway_path = os.path.join(self.log_dir, f"gateway_metrics_{segment_ts}.csv")
+        nodes_path = os.path.join(self.log_dir, f"node_metrics_{segment_ts}.csv")
+
+        self._gateway_file = open(gateway_path, "w", newline="", encoding="utf-8")
+        self._gateway_writer = csv.writer(self._gateway_file)
+        gateway_header = [
+            "timestamp",
+            "gateway_address",
+            "schedule_id",
+            "connected_nodes",
+            "tx_total",
+            "rx_total",
+            "tx_rate_1s",
+            "rx_rate_1s",
+            "avg_latency_ms",
+        ]
+        self._gateway_writer.writerow(gateway_header)
+
+        self._nodes_file = open(nodes_path, "w", newline="", encoding="utf-8")
+        self._nodes_writer = csv.writer(self._nodes_file)
+        nodes_header = [
+            "timestamp",
+            "gateway_address",
+            "node_address",
+            "is_alive",
+            "tx_total",
+            "rx_total",
+            "tx_rate_1s",
+            "rx_rate_1s",
+            "success_rate_30s",
+            "success_rate_total",
+            "pdr_downlink",
+            "pdr_uplink",
+            "rssi_dbm_5s",
+            "last_latency_ms",
+            "avg_latency_ms",
+        ]
+        self._nodes_writer.writerow(nodes_header)
+
+    def _check_for_rotation(self):
+        if datetime.now() - self.segment_start_time >= self.rotation_interval:
+            self._open_new_segment()
+
+    def _log_common(self):
+        if not self.active:
+            return False
+        self._check_for_rotation()
+        return True
+
+    def log_periodic_metrics(self, gateway: MariGateway, nodes: List[MariNode]):
+        last_log_time = self.last_log_time.get(gateway.info.address, self.segment_start_time)
+        if datetime.now() - last_log_time >= timedelta(seconds=self.log_interval_seconds):
+            self.log_gateway_metrics(gateway)
+            self.log_all_nodes_metrics(nodes)
+            self.last_log_time[gateway.info.address] = datetime.now()
+
+    def log_gateway_metrics(self, gateway: MariGateway):
+        if not self._log_common() or self._gateway_writer is None:
+            return
+
+        timestamp = datetime.now().isoformat()
+        row = [
+            timestamp,
+            f"0x{gateway.info.address:016X}",
+            gateway.info.schedule_id,
+            len(gateway.nodes),
+            gateway.stats.sent_count(include_test_packets=False),
+            gateway.stats.received_count(include_test_packets=False),
+            gateway.stats.sent_count(1, include_test_packets=False),
+            gateway.stats.received_count(1, include_test_packets=False),
+            f"{gateway.latency_stats.avg_ms:.2f}",
+        ]
+        self._gateway_writer.writerow(row)
+
+    def log_all_nodes_metrics(self, nodes: List[MariNode]):
+        """Writes metrics for all nodes, handling rotation."""
+        if not self._log_common() or self._nodes_writer is None:
+            return
+
+        timestamp = datetime.now().isoformat()
+        for node in nodes:
+            row = [
+                timestamp,
+                f"0x{node.gateway_address:016X}",
+                f"0x{node.address:016X}",
+                node.is_alive,
+                node.stats.sent_count(include_test_packets=False),
+                node.stats.received_count(include_test_packets=False),
+                node.stats.sent_count(1, include_test_packets=False),
+                node.stats.received_count(1, include_test_packets=False),
+                f"{node.stats.success_rate(30):.2%}",
+                f"{node.stats.success_rate():.2%}",
+                f"{node.pdr_downlink:.2%}",
+                f"{node.pdr_uplink:.2%}",
+                node.stats.received_rssi_dbm(5),
+                f"{node.latency_stats.last_ms:.2f}",
+                f"{node.latency_stats.avg_ms:.2f}",
+            ]
+            self._nodes_writer.writerow(row)
+
+    def log_event(
+        self, gateway_address: int, node_address: int, event_name: str, event_tag: str = ""
+    ):
+        """Logs an event to the events log file."""
+        if not self.active or self._events_writer is None:
+            return
+
+        timestamp = datetime.now().isoformat()
+        row = [
+            timestamp,
+            f"0x{gateway_address:016X}",
+            f"0x{node_address:016X}",
+            event_name,
+            event_tag,
+        ]
+        self._events_writer.writerow(row)
+        if self._events_file:
+            self._events_file.flush()
+
+    def _close_segment_files(self):
+        if self._gateway_file and not self._gateway_file.closed:
+            self._gateway_file.close()
+        if self._nodes_file and not self._nodes_file.closed:
+            self._nodes_file.close()
+
+    def close(self):
+        if not self.active:
+            return
+
+        self._close_segment_files()
+        if self._events_file and not self._events_file.closed:
+            self._events_file.close()
+        print(f"\nMetrics saved to: {self.log_dir}")
+        self.active = False

marilib/mari_protocol.py

@@ -0,0 +1,76 @@
+import dataclasses
+from dataclasses import dataclass
+
+from marilib.protocol import Packet, PacketFieldMetadata, PacketType
+
+MARI_PROTOCOL_VERSION = 2
+MARI_BROADCAST_ADDRESS = 0xFFFFFFFFFFFFFFFF
+MARI_NET_ID_DEFAULT = 0x0001
+
+
+@dataclass
+class HeaderStats(Packet):
+    """Dataclass that holds MAC header stats."""
+
+    metadata: list[PacketFieldMetadata] = dataclasses.field(
+        default_factory=lambda: [
+            PacketFieldMetadata(name="rssi", disp="rssi", length=1),
+        ]
+    )
+    rssi: int = 0
+
+    @property
+    def rssi_dbm(self) -> int:
+        if self.rssi > 127:
+            return self.rssi - 255
+        return self.rssi
+
+
+@dataclass
+class Header(Packet):
+    """Dataclass that holds MAC header fields."""
+
+    metadata: list[PacketFieldMetadata] = dataclasses.field(
+        default_factory=lambda: [
+            PacketFieldMetadata(name="version", disp="ver.", length=1),
+            PacketFieldMetadata(name="type_", disp="type", length=1),
+            PacketFieldMetadata(name="network_id", disp="net", length=2),
+            PacketFieldMetadata(name="destination", disp="dst", length=8),
+            PacketFieldMetadata(name="source", disp="src", length=8),
+        ]
+    )
+    version: int = MARI_PROTOCOL_VERSION
+    type_: int = PacketType.DATA
+    network_id: int = MARI_NET_ID_DEFAULT
+    destination: int = MARI_BROADCAST_ADDRESS
+    source: int = 0x0000000000000000
+
+    def __repr__(self):
+        type_ = PacketType(self.type_).name
+        return f"Header(version={self.version}, type_={type_}, network_id=0x{self.network_id:04x}, destination=0x{self.destination:016x}, source=0x{self.source:016x})"
+
+
+@dataclass
+class Frame:
+    """Data class that holds a payload packet."""
+
+    header: Header = None
+    stats: HeaderStats = dataclasses.field(default_factory=HeaderStats)
+    payload: bytes = b""
+
+    def from_bytes(self, bytes_):
+        self.header = Header().from_bytes(bytes_[0:20])
+        if len(bytes_) > 20:
+            self.stats = HeaderStats().from_bytes(bytes_[20:21])
+            if len(bytes_) > 21:
+                self.payload = bytes_[21:]
+        return self
+
+    def to_bytes(self, byteorder="little") -> bytes:
+        header_bytes = self.header.to_bytes(byteorder)
+        stats_bytes = self.stats.to_bytes(byteorder)
+        return header_bytes + stats_bytes + self.payload
+
+    def __repr__(self):
+        header_no_metadata = dataclasses.replace(self.header, metadata=[])
+        return f"Frame(header={header_no_metadata}, payload={self.payload})"

marilib/marilib.py

@@ -0,0 +1,35 @@
+from abc import ABC, abstractmethod
+
+from marilib.model import MariNode
+
+
+class MarilibBase(ABC):
+    """Base class for Marilib applications."""
+
+    @abstractmethod
+    def update(self):
+        """Recurrent bookkeeping. Don't forget to call this periodically on your main loop."""
+
+    @abstractmethod
+    def nodes(self) -> list[MariNode]:
+        """Returns all nodes in the network."""
+
+    @abstractmethod
+    def add_node(self, address: int, gateway_address: int = None) -> MariNode | None:
+        """Adds a node to the network."""
+
+    @abstractmethod
+    def remove_node(self, address: int) -> MariNode | None:
+        """Removes a node from the network."""
+
+    @abstractmethod
+    def send_frame(self, dst: int, payload: bytes):
+        """Sends a frame to the network."""
+
+    @abstractmethod
+    def render_tui(self):
+        """Renders the TUI."""
+
+    @abstractmethod
+    def close_tui(self):
+        """Closes the TUI."""

marilib/marilib_cloud.py

@@ -0,0 +1,193 @@
+import threading
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Callable
+
+from marilib.latency import LatencyTester
+from marilib.mari_protocol import Frame, Header
+from marilib.model import (
+    EdgeEvent,
+    GatewayInfo,
+    MariGateway,
+    MariNode,
+    NodeInfoCloud,
+)
+from marilib.communication_adapter import MQTTAdapter
+from marilib.marilib import MarilibBase
+from marilib.tui_cloud import MarilibTUICloud
+
+LOAD_PACKET_PAYLOAD = b"L"
+
+
+@dataclass
+class MarilibCloud(MarilibBase):
+    """
+    The MarilibCloud class runs in a computer.
+    It is used to communicate with a Mari radio gateway (nRF5340) via MQTT.
+    """
+
+    cb_application: Callable[[EdgeEvent, MariNode | Frame | GatewayInfo], None]
+    mqtt_interface: MQTTAdapter
+    network_id: int
+    tui: MarilibTUICloud | None = None
+
+    logger: Any | None = None
+    gateways: dict[int, MariGateway] = field(default_factory=dict)
+    lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+    latency_tester: LatencyTester | None = None
+
+    started_ts: datetime = field(default_factory=datetime.now)
+    last_received_mqtt_data_ts: datetime = field(default_factory=datetime.now)
+    main_file: str | None = None
+
+    def __post_init__(self):
+        self.setup_params = {
+            "main_file": self.main_file or "unknown",
+            "mqtt_host": self.mqtt_interface.host,
+            "mqtt_port": self.mqtt_interface.port,
+            "network_id": self.network_id_str,
+        }
+        self.mqtt_interface.set_network_id(self.network_id_str)
+        self.mqtt_interface.set_on_data_received(self.on_mqtt_data_received)
+        self.mqtt_interface.init()
+        if self.logger:
+            self.logger.log_setup_parameters(self.setup_params)
+
+    # ============================ MarilibBase methods =========================
+
+    def update(self):
+        """Recurrent bookkeeping. Don't forget to call this periodically on your main loop."""
+        with self.lock:
+            # remove dead gateways
+            self.gateways = {
+                addr: gateway for addr, gateway in self.gateways.items() if gateway.is_alive
+            }
+            # update each gateway
+            for gateway in self.gateways.values():
+                gateway.update()
+                if self.logger:
+                    self.logger.log_periodic_metrics(gateway, gateway.nodes)
+
+    @property
+    def nodes(self) -> list[MariNode]:
+        return [node for gateway in self.gateways.values() for node in gateway.nodes]
+
+    def add_node(self, address: int, gateway_address: int = None) -> MariNode | None:
+        with self.lock:
+            gateway = self.gateways.get(gateway_address)
+            if gateway:
+                node = gateway.add_node(address)
+                return node
+        return None
+
+    def remove_node(self, address: int, gateway_address: int = None) -> MariNode | None:
+        with self.lock:
+            gateway = self.gateways.get(gateway_address)
+            if gateway:
+                node = gateway.remove_node(address)
+                return node
+        return None
+
+    def send_frame(self, dst: int, payload: bytes):
+        """
+        Sends a frame to a gateway via MQTT.
+        Consists in publishing a message to the /mari/{network_id}/to_edge topic.
+        """
+        mari_frame = Frame(Header(destination=dst), payload=payload)
+
+        self.mqtt_interface.send_data_to_edge(
+            EdgeEvent.to_bytes(EdgeEvent.NODE_DATA) + mari_frame.to_bytes()
+        )
+
+    def render_tui(self):
+        if self.tui:
+            self.tui.render(self)
+
+    def close_tui(self):
+        if self.tui:
+            self.tui.close()
+
+    # ============================ MarilibCloud methods =========================
+
+    @property
+    def network_id_str(self) -> str:
+        return f"{self.network_id:04X}"
+
+    # ============================ Callbacks ===================================
+
+    def handle_mqtt_data(self, data: bytes) -> tuple[bool, EdgeEvent, Any]:
+        """
+        Handles the MQTT data received from the MQTT broker:
+        - parses the event
+        - updates node or gateway information
+        - returns the event type and data (if any)
+        """
+
+        if len(data) < 1:
+            return False, EdgeEvent.UNKNOWN, None
+
+        self.last_received_mqtt_data_ts = datetime.now()
+
+        try:
+            event_type = EdgeEvent(data[0])
+        except ValueError:
+            return False, EdgeEvent.UNKNOWN, None
+
+        try:
+            if event_type == EdgeEvent.NODE_JOINED:
+                node_info = NodeInfoCloud().from_bytes(data[1:])
+                if node := self.add_node(node_info.address, node_info.gateway_address):
+                    return True, EdgeEvent.NODE_JOINED, node_info
+
+            elif event_type == EdgeEvent.NODE_LEFT:
+                node_info = NodeInfoCloud().from_bytes(data[1:])
+                if node := self.remove_node(node_info.address, node_info.gateway_address):
+                    return True, EdgeEvent.NODE_LEFT, node_info
+
+            elif event_type == EdgeEvent.NODE_KEEP_ALIVE:
+                node_info = NodeInfoCloud().from_bytes(data[1:])
+                gateway = self.gateways.get(node_info.gateway_address)
+                if gateway:
+                    gateway.update_node_liveness(node_info.address)
+                    return True, EdgeEvent.NODE_KEEP_ALIVE, node_info
+
+            elif event_type == EdgeEvent.GATEWAY_INFO:
+                gateway_info = GatewayInfo().from_bytes(data[1:])
+                gateway = self.gateways.get(gateway_info.address)
+                if not gateway:
+                    # we are learning about a new gateway, so instantiate it and add it to the list
+                    gateway = MariGateway(info=gateway_info)
+                    self.gateways[gateway.info.address] = gateway
+                else:
+                    gateway.set_info(gateway_info)
+                return True, EdgeEvent.GATEWAY_INFO, gateway_info
+
+            elif event_type == EdgeEvent.NODE_DATA:
+                frame = Frame().from_bytes(data[1:])
+
+                gateway_address = frame.header.destination
+                node_address = frame.header.source
+                gateway = self.gateways.get(gateway_address)
+                node = gateway.get_node(node_address)
+                if not gateway or not node:
+                    return False, EdgeEvent.UNKNOWN, None
+
+                gateway.update_node_liveness(node_address)
+                gateway.register_received_frame(frame, is_test_packet=False)
+                return True, EdgeEvent.NODE_DATA, frame
+
+        except Exception as e:
+            print(f"Error handling MQTT data: {e}")
+
+        # fallback result in case of error
+        return False, EdgeEvent.UNKNOWN, None
+
+    def on_mqtt_data_received(self, data: bytes):
+        res, event_type, event_data = self.handle_mqtt_data(data)
+        if res:
+            if self.logger and event_type in [EdgeEvent.NODE_JOINED, EdgeEvent.NODE_LEFT]:
+                # TODO: update the logging system to also support GATEWAY_INFO events from multiple gateways
+                self.logger.log_event(
+                    event_data.gateway_address, event_data.address, event_type.name
+                )
+            self.cb_application(event_type, event_data)