ReticulumTelemetryHub 0.1.0__py3-none-any.whl → 0.143.0__py3-none-any.whl

reticulum_telemetry_hub/reticulum_server/internal_adapter.py

@@ -0,0 +1,358 @@
+"""Reticulum-facing adapter for the internal API."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+import logging
+import time
+from dataclasses import dataclass
+from typing import Awaitable
+from typing import Callable
+from typing import Optional
+from typing import Union
+from uuid import uuid4
+
+import LXMF
+from msgpack import unpackb
+from datetime import datetime
+from datetime import timezone
+
+from reticulum_telemetry_hub.internal_api.core import InternalApiCore
+from reticulum_telemetry_hub.internal_api.v1.enums import CommandStatus
+from reticulum_telemetry_hub.internal_api.v1.enums import CommandType
+from reticulum_telemetry_hub.internal_api.v1.enums import EventType
+from reticulum_telemetry_hub.internal_api.v1.enums import IssuerType
+from reticulum_telemetry_hub.internal_api.v1.enums import MessageType
+from reticulum_telemetry_hub.internal_api.v1.enums import NodeType
+from reticulum_telemetry_hub.internal_api.v1.enums import Qos
+from reticulum_telemetry_hub.internal_api.v1.schemas import CommandEnvelope
+from reticulum_telemetry_hub.internal_api.v1.schemas import EventEnvelope
+from reticulum_telemetry_hub.internal_api.versioning import select_api_version
+from reticulum_telemetry_hub.reticulum_server.constants import PLUGIN_COMMAND
+
+
+_DEDUP_TTL_SECONDS = 600.0
+_LOGGER = logging.getLogger(__name__)
+
+
+def _maybe_await(result: Union[None, Awaitable[None]]) -> Awaitable[None]:
+    if asyncio.iscoroutine(result):
+        return result
+    return asyncio.sleep(0)
+
+
+class InlineEventBus:
+    """Inline event bus for synchronous Reticulum adapters."""
+
+    def __init__(self) -> None:
+        self._subscribers: list[Callable[[EventEnvelope], Union[None, Awaitable[None]]]] = []
+
+    def subscribe(self, handler):
+        """Subscribe to events and return an unsubscribe callback."""
+
+        self._subscribers.append(handler)
+
+        def _unsubscribe() -> None:
+            if handler in self._subscribers:
+                self._subscribers.remove(handler)
+
+        return _unsubscribe
+
+    async def start(self) -> None:
+        """No-op start."""
+
+        return None
+
+    async def stop(self) -> None:
+        """No-op stop."""
+
+        return None
+
+    async def publish(self, event: EventEnvelope) -> None:
+        """Publish events to subscribers immediately."""
+
+        for handler in list(self._subscribers):
+            await _maybe_await(handler(_copy_event(event)))
+
+
+@dataclass
+class LxmfInbound:
+    """Normalized LXMF inbound payload for adapter processing."""
+
+    message_id: Optional[str]
+    source_id: Optional[str]
+    topic_id: Optional[str]
+    text: Optional[str]
+    fields: dict
+    commands: list[dict]
+
+
+class MessageDeduper:
+    """TTL-based duplicate filter for LXMF message IDs."""
+
+    def __init__(self, *, ttl_seconds: float, clock: Callable[[], float]) -> None:
+        self._ttl_seconds = ttl_seconds
+        self._clock = clock
+        self._entries: dict[str, float] = {}
+        self._order: list[tuple[float, str]] = []
+
+    def is_duplicate(self, message_id: str) -> bool:
+        """Return True when message_id has been seen within the TTL window."""
+
+        now = self._clock()
+        self._prune(now)
+        if message_id in self._entries:
+            return True
+        self._entries[message_id] = now
+        self._order.append((now, message_id))
+        return False
+
+    def _prune(self, now: float) -> None:
+        cutoff = now - self._ttl_seconds
+        while self._order and self._order[0][0] < cutoff:
+            _, message_id = self._order.pop(0)
+            self._entries.pop(message_id, None)
+
+
+class ReticulumInternalAdapter:
+    """Adapter translating LXMF inputs to internal API commands."""
+
+    def __init__(
+        self,
+        *,
+        send_message: Callable[[str, Optional[str]], None],
+        clock: Callable[[], float] = time.time,
+    ) -> None:
+        self.event_bus = InlineEventBus()
+        self.core = InternalApiCore(self.event_bus)
+        self._send_message = send_message
+        self._clock = clock
+        self._seen_nodes: set[str] = set()
+        self._deduper = MessageDeduper(ttl_seconds=_DEDUP_TTL_SECONDS, clock=clock)
+        self._pending_text: dict[str, tuple[str, str]] = {}
+        self.event_bus.subscribe(self._handle_event)
+
+    def handle_inbound(self, inbound: LxmfInbound) -> None:
+        """Handle normalized LXMF inputs."""
+
+        message_id = (inbound.message_id or "").lower()
+        if message_id and self._deduper.is_duplicate(message_id):
+            return
+
+        node_id = inbound.source_id
+        if node_id:
+            if node_id not in self._seen_nodes:
+                self._seen_nodes.add(node_id)
+                self._register_node(node_id)
+            self.core.touch_node(node_id)
+
+        commands = inbound.commands or []
+        for command in commands:
+            name = _command_name(command)
+            if not name:
+                continue
+            normalized = name.lower()
+            if normalized == "join":
+                if node_id and node_id not in self._seen_nodes:
+                    self._seen_nodes.add(node_id)
+                    self._register_node(node_id)
+                continue
+            if normalized == "subscribetopic":
+                topic_id = _extract_topic_id(command)
+                if node_id and topic_id:
+                    self._subscribe_topic(node_id, topic_id)
+                continue
+            if normalized == "createtopic":
+                continue
+
+        if inbound.topic_id and inbound.text:
+            self._publish_text(node_id, inbound.topic_id, inbound.text)
+
+        telemetry_payloads = _extract_telemetry_payloads(inbound.fields)
+        if inbound.topic_id and telemetry_payloads:
+            for payload in telemetry_payloads:
+                self._publish_telemetry(node_id, inbound.topic_id, payload)
+
+    def _register_node(self, node_id: str) -> None:
+        command = _build_command(
+            CommandType.REGISTER_NODE,
+            {
+                "node_id": node_id,
+                "node_type": NodeType.RETICULUM,
+                "metadata": None,
+            },
+            issuer_id=node_id,
+        )
+        _run_command(self.core, command)
+
+    def _subscribe_topic(self, node_id: str, topic_id: str) -> None:
+        command = _build_command(
+            CommandType.SUBSCRIBE_TOPIC,
+            {"subscriber_id": node_id, "topic_path": topic_id},
+            issuer_id=node_id,
+        )
+        _run_command(self.core, command)
+
+    def _publish_text(self, node_id: Optional[str], topic_id: str, text: str) -> None:
+        if not node_id:
+            return
+        command = _build_command(
+            CommandType.PUBLISH_MESSAGE,
+            {
+                "topic_path": topic_id,
+                "message_type": MessageType.TEXT,
+                "content": {
+                    "message_type": MessageType.TEXT,
+                    "text": text,
+                    "encoding": "utf-8",
+                },
+                "qos": Qos.BEST_EFFORT,
+            },
+            issuer_id=node_id,
+        )
+        message_id = command.command_id.hex
+        self._pending_text[message_id] = (topic_id, text)
+        result = _run_command(self.core, command)
+        if result.status != CommandStatus.ACCEPTED:
+            self._pending_text.pop(message_id, None)
+
+    def _publish_telemetry(
+        self, node_id: Optional[str], topic_id: str, payload: dict
+    ) -> None:
+        if not node_id:
+            return
+        telemetry_type = payload.get("telemetry_type")
+        command = _build_command(
+            CommandType.PUBLISH_MESSAGE,
+            {
+                "topic_path": topic_id,
+                "message_type": MessageType.TELEMETRY,
+                "content": {
+                    "message_type": MessageType.TELEMETRY,
+                    "telemetry_type": telemetry_type if isinstance(telemetry_type, str) else None,
+                    "data": payload,
+                },
+                "qos": Qos.BEST_EFFORT,
+            },
+            issuer_id=node_id,
+        )
+        _run_command(self.core, command)
+
+    async def _handle_event(self, event: EventEnvelope) -> None:
+        """Fan out message events to LXMF recipients."""
+
+        if event.event_type != EventType.MESSAGE_PUBLISHED:
+            return
+        payload = event.payload
+        message_id = getattr(payload, "message_id", "")
+        pending = self._pending_text.pop(message_id, None)
+        if not pending:
+            return
+        topic_id, text = pending
+        message_text = f"[topic:{topic_id}]\n{text}"
+        for subscriber_id in self.core.get_subscriber_ids(topic_id):
+            self._send_message(message_text, subscriber_id)
+
+
+def _run_command(core: InternalApiCore, command: CommandEnvelope):
+    return asyncio.run(core.handle_command(command))
+
+
+def _build_command(
+    command_type: CommandType, payload: dict, *, issuer_id: str
+) -> CommandEnvelope:
+    command = CommandEnvelope.model_validate(
+        {
+            "api_version": select_api_version(),
+            "command_id": uuid4(),
+            "command_type": command_type,
+            "issued_at": datetime.now(timezone.utc),
+            "issuer": {"type": IssuerType.RETICULUM, "id": issuer_id},
+            "payload": payload,
+        }
+    )
+    _LOGGER.debug(
+        "Reticulum adapter command built",
+        extra={
+            "command_id": str(command.command_id),
+            "command_type": command.command_type.value,
+            "correlation_id": str(command.command_id),
+        },
+    )
+    return command
+
+
+def _copy_event(event: EventEnvelope) -> EventEnvelope:
+    copy_method = getattr(event, "model_copy", None)
+    if callable(copy_method):
+        return copy_method(deep=True)
+    return copy.deepcopy(event)
+
+
+def _command_name(command: dict) -> Optional[str]:
+    value = command.get("Command") or command.get("command")
+    if isinstance(value, str):
+        return value.strip()
+    plugin_value = command.get(PLUGIN_COMMAND) or command.get(str(PLUGIN_COMMAND))
+    if isinstance(plugin_value, str):
+        return plugin_value.strip()
+    return None
+
+
+def _extract_topic_id(command: dict) -> Optional[str]:
+    for key in ("TopicID", "topic_id", "topic", "Topic"):
+        value = command.get(key)
+        if value:
+            return str(value)
+    return None
+
+
+def _extract_telemetry_payloads(fields: dict) -> list[dict]:
+    payloads: list[dict] = []
+    if not isinstance(fields, dict):
+        return payloads
+    if LXMF.FIELD_TELEMETRY in fields:
+        raw = fields.get(LXMF.FIELD_TELEMETRY)
+        payload = _decode_telemetry_payload(raw)
+        if isinstance(payload, dict):
+            payloads.append(payload)
+    if LXMF.FIELD_TELEMETRY_STREAM in fields:
+        raw_stream = fields.get(LXMF.FIELD_TELEMETRY_STREAM)
+        entries = _decode_telemetry_stream(raw_stream)
+        for payload in entries:
+            if isinstance(payload, dict):
+                payloads.append(payload)
+    return payloads
+
+
+def _decode_telemetry_payload(raw) -> Optional[dict]:
+    if isinstance(raw, (bytes, bytearray)):
+        try:
+            decoded = unpackb(raw, strict_map_key=False)
+        except Exception:
+            return None
+        return decoded if isinstance(decoded, dict) else None
+    if isinstance(raw, dict):
+        return raw
+    return None
+
+
+def _decode_telemetry_stream(raw) -> list[dict]:
+    if isinstance(raw, (bytes, bytearray)):
+        try:
+            raw = unpackb(raw, strict_map_key=False)
+        except Exception:
+            return []
+    if not isinstance(raw, (list, tuple)):
+        return []
+    payloads: list[dict] = []
+    for entry in raw:
+        if not isinstance(entry, (list, tuple)) or len(entry) < 3:
+            continue
+        payload = entry[2]
+        if isinstance(payload, (bytes, bytearray)):
+            payload = _decode_telemetry_payload(payload)
+        if isinstance(payload, dict):
+            payloads.append(payload)
+    return payloads

reticulum_telemetry_hub/reticulum_server/outbound_queue.py

@@ -0,0 +1,312 @@
+import threading
+import time
+from dataclasses import dataclass, field
+from queue import Empty, Full, Queue
+
+import LXMF
+import RNS
+
+
+@dataclass
+class OutboundPayload:
+    """
+    Message payload scheduled for outbound delivery.
+
+    Args:
+        connection (RNS.Destination): Destination to deliver the message to.
+        message_text (str): Plaintext message body to deliver.
+        destination_hash (bytes | None): Raw destination hash for diagnostics.
+        destination_hex (str | None): Hex-encoded destination hash for logging.
+        fields (dict | None): Optional LXMF fields to include with the message.
+        attempts (int): Number of delivery attempts performed.
+        next_attempt_at (float): Monotonic timestamp before the next attempt.
+    """
+
+    connection: RNS.Destination
+    message_text: str
+    destination_hash: bytes | None
+    destination_hex: str | None
+    fields: dict | None = None
+    attempts: int = 0
+    next_attempt_at: float = field(default_factory=time.monotonic)
+
+
+class OutboundMessageQueue:
+    """
+    Threaded dispatcher that delivers LXMF messages without blocking callers.
+    """
+
+    def __init__(
+        self,
+        lxm_router: LXMF.LXMRouter,
+        sender: RNS.Destination,
+        *,
+        queue_size: int = 64,
+        worker_count: int = 2,
+        send_timeout: float = 5.0,
+        backoff_seconds: float = 0.5,
+        max_attempts: int = 3,
+    ):
+        """
+        Initialize a bounded outbound queue.
+
+        Args:
+            lxm_router (LXMF.LXMRouter): Router responsible for delivery.
+            sender (RNS.Destination): Sender identity to attach to messages.
+            queue_size (int): Maximum queued payloads before applying backpressure.
+            worker_count (int): Parallel worker threads processing the queue.
+            send_timeout (float): Seconds to wait for a send before timing out.
+            backoff_seconds (float): Base delay between retry attempts.
+            max_attempts (int): Maximum attempts before dropping a message.
+        """
+        self._lxm_router = lxm_router
+        self._sender = sender
+        self._queue: Queue[OutboundPayload] = Queue(maxsize=max(queue_size, 1))
+        self._worker_count = max(worker_count, 1)
+        self._send_timeout = max(send_timeout, 0.1)
+        self._backoff_seconds = max(backoff_seconds, 0.01)
+        self._max_attempts = max(max_attempts, 1)
+        self._stop_event = threading.Event()
+        self._workers: list[threading.Thread] = []
+        self._inflight = 0
+        self._inflight_lock = threading.Lock()
+
+    def start(self) -> None:
+        """Start background worker threads if they are not already running."""
+
+        if self._workers:
+            return
+
+        self._stop_event.clear()
+        for index in range(self._worker_count):
+            worker = threading.Thread(
+                target=self._worker_loop,
+                name=f"lxmf-outbound-{index}",
+                daemon=True,
+            )
+            worker.start()
+            self._workers.append(worker)
+
+    def stop(self) -> None:
+        """Signal workers to stop and wait for them to exit."""
+
+        if not self._workers:
+            return
+
+        self._stop_event.set()
+        for worker in self._workers:
+            worker.join(timeout=0.5)
+        self._workers.clear()
+
+    def queue_message(
+        self,
+        connection: RNS.Destination,
+        message_text: str,
+        destination_hash: bytes | None,
+        destination_hex: str | None,
+        fields: dict | None = None,
+    ) -> bool:
+        """
+        Enqueue a message for delivery.
+
+        Args:
+            connection (RNS.Destination): Destination to deliver the message to.
+            message_text (str): Plaintext message body to deliver.
+            destination_hash (bytes | None): Raw destination hash for diagnostics.
+            destination_hex (str | None): Hex-encoded destination hash for logging.
+            fields (dict | None): Optional LXMF message fields.
+
+        Returns:
+            bool: ``True`` when the message was queued successfully.
+        """
+
+        payload = OutboundPayload(
+            connection=connection,
+            message_text=message_text,
+            destination_hash=destination_hash,
+            destination_hex=destination_hex,
+            fields=fields,
+        )
+        return self._enqueue_payload(payload)
+
+    def wait_for_flush(self, timeout: float = 1.0) -> bool:
+        """
+        Wait until the queue and inflight sends complete.
+
+        Args:
+            timeout (float): Seconds to wait before giving up.
+
+        Returns:
+            bool: ``True`` when the queue drained before the timeout elapsed.
+        """
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            with self._inflight_lock:
+                inflight = self._inflight
+            if self._queue.empty() and inflight == 0:
+                return True
+            time.sleep(0.01)
+        return False
+
+    def _enqueue_payload(self, payload: OutboundPayload) -> bool:
+        try:
+            self._queue.put_nowait(payload)
+            return True
+        except Full:
+            dropped = self._drop_oldest()
+            if dropped is not None:
+                RNS.log(
+                    (
+                        "Outbound queue is full; dropped oldest payload destined for"
+                        f" {dropped.destination_hex or 'unknown destination'}"
+                    ),
+                    getattr(RNS, "LOG_WARNING", 2),
+                )
+            try:
+                self._queue.put_nowait(payload)
+                return True
+            except Full:
+                RNS.log(
+                    (
+                        "Outbound queue is saturated; dropping payload destined for"
+                        f" {payload.destination_hex or 'unknown destination'}"
+                    ),
+                    getattr(RNS, "LOG_WARNING", 2),
+                )
+                return False
+
+    def _drop_oldest(self) -> OutboundPayload | None:
+        try:
+            oldest = self._queue.get_nowait()
+            self._queue.task_done()
+            return oldest
+        except Empty:
+            return None
+
+    def _worker_loop(self) -> None:
+        while not self._stop_event.is_set() or not self._queue.empty():
+            try:
+                payload = self._queue.get(timeout=0.1)
+            except Empty:
+                continue
+
+            now = time.monotonic()
+            if payload.next_attempt_at > now:
+                delay = min(payload.next_attempt_at - now, 0.1)
+                time.sleep(delay)
+                self._enqueue_payload(payload)
+                self._queue.task_done()
+                continue
+
+            try:
+                self._process_payload(payload)
+            finally:
+                self._queue.task_done()
+
+    def _process_payload(self, payload: OutboundPayload) -> None:
+        with self._inflight_lock:
+            self._inflight += 1
+
+        try:
+            success = self._send_with_timeout(payload)
+            if not success:
+                self._handle_failure(payload)
+        finally:
+            with self._inflight_lock:
+                self._inflight -= 1
+
+    def _send_with_timeout(self, payload: OutboundPayload) -> bool:
+        error: list[Exception | None] = [None]
+
+        def _send() -> None:
+            try:
+                message = LXMF.LXMessage(
+                    payload.connection,
+                    self._sender,
+                    payload.message_text,
+                    fields=payload.fields or {},
+                    desired_method=LXMF.LXMessage.DIRECT,
+                )
+                if payload.destination_hash:
+                    message.destination_hash = payload.destination_hash
+                self._lxm_router.handle_outbound(message)
+            except Exception as exc:  # pragma: no cover - defensive logging
+                error[0] = exc
+
+        sender = threading.Thread(target=_send, name="lxmf-send", daemon=True)
+        sender.start()
+        sender.join(timeout=self._send_timeout)
+        if sender.is_alive():
+            RNS.log(
+                (
+                    "Timed out delivering outbound message to"
+                    f" {payload.destination_hex or 'unknown destination'}"
+                ),
+                getattr(RNS, "LOG_WARNING", 2),
+            )
+            return False
+
+        if error[0] is not None:
+            RNS.log(
+                (
+                    "Failed to deliver outbound message to"
+                    f" {payload.destination_hex or 'unknown destination'}: {error[0]}"
+                ),
+                getattr(RNS, "LOG_WARNING", 2),
+            )
+            return False
+        return True
+
+    def _handle_failure(self, payload: OutboundPayload) -> None:
+        payload.attempts += 1
+        if payload.attempts >= self._max_attempts:
+            RNS.log(
+                (
+                    "Dropping outbound message to"
+                    f" {payload.destination_hex or 'unknown destination'} after"
+                    f" {payload.attempts} attempts"
+                ),
+                getattr(RNS, "LOG_WARNING", 2),
+            )
+            self._propagate_failed_message(payload)
+            return
+
+        payload.next_attempt_at = (
+            time.monotonic() + self._backoff_seconds * payload.attempts
+        )
+        # Reason: requeue with backoff so slower destinations do not halt others.
+        self._enqueue_payload(payload)
+
+    def _propagate_failed_message(self, payload: OutboundPayload) -> None:
+        if not getattr(self._lxm_router, "propagation_node", False):
+            return
+
+        try:
+            message = LXMF.LXMessage(
+                payload.connection,
+                self._sender,
+                payload.message_text,
+                fields=payload.fields or {},
+                desired_method=LXMF.LXMessage.DIRECT,
+            )
+            if payload.destination_hash:
+                message.destination_hash = payload.destination_hash
+            message.pack()
+            if not message.packed:
+                return
+            self._lxm_router.lxmf_propagation(message.packed)
+            RNS.log(
+                (
+                    "Stored failed outbound message for propagation to"
+                    f" {payload.destination_hex or 'unknown destination'}"
+                ),
+                getattr(RNS, "LOG_INFO", 4),
+            )
+        except Exception as exc:  # pragma: no cover - defensive logging
+            RNS.log(
+                (
+                    "Failed to store outbound message for propagation to"
+                    f" {payload.destination_hex or 'unknown destination'}: {exc}"
+                ),
+                getattr(RNS, "LOG_WARNING", 2),
+            )