arthexis 0.1.16__py3-none-any.whl → 0.1.26__py3-none-any.whl

nodes/models.py

@@ -4,6 +4,7 @@ from collections.abc import Iterable
 from copy import deepcopy
 from dataclasses import dataclass
 from django.db import models
+from django.db.models import Q
 from django.db.utils import DatabaseError
 from django.db.models.signals import post_delete
 from django.dispatch import Signal, receiver
@@ -13,17 +14,19 @@ from core.fields import SigilLongAutoField, SigilShortAutoField
 import re
 import json
 import base64
+import ipaddress
 from django.utils import timezone
 from django.utils.text import slugify
 from django.conf import settings
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone as datetime_timezone
 import uuid
 import os
-import shutil
 import socket
 import stat
 import subprocess
+import shutil
 from pathlib import Path
+from urllib.parse import urlparse, urlunsplit
 from utils import revision
 from core.notifications import notify_async
 from django.core.exceptions import ValidationError
@@ -41,6 +44,9 @@ import logging
 logger = logging.getLogger(__name__)
 
 
+ROLE_RENAMES: dict[str, str] = {"Constellation": "Watchtower"}
+
+
 class NodeRoleManager(models.Manager):
     def get_by_natural_key(self, name: str):
         return self.get(name=name)
@@ -143,8 +149,6 @@ class NodeFeature(Entity):
             return False
         if node.features.filter(pk=self.pk).exists():
             return True
-        if self.slug == "gway-runner":
-            return Node._has_gway_runner()
         if self.slug == "gui-toast":
             from core.notifications import supports_gui_toast
 
@@ -188,8 +192,10 @@ class Node(Entity):
 
     DEFAULT_BADGE_COLOR = "#28a745"
     ROLE_BADGE_COLORS = {
-        "Constellation": "#daa520",  # goldenrod
+        "Watchtower": "#daa520",  # goldenrod
+        "Constellation": "#daa520",  # legacy alias
         "Control": "#673ab7",  # deep purple
+        "Interface": "#0dcaf0",  # cyan
     }
 
     class Relation(models.TextChoices):
@@ -199,9 +205,20 @@ class Node(Entity):
         SELF = "SELF", "Self"
 
     hostname = models.CharField(max_length=100)
-    address = models.GenericIPAddressField()
+    network_hostname = models.CharField(max_length=253, blank=True)
+    ipv4_address = models.GenericIPAddressField(
+        protocol="IPv4", blank=True, null=True
+    )
+    ipv6_address = models.GenericIPAddressField(
+        protocol="IPv6", blank=True, null=True
+    )
+    address = models.GenericIPAddressField(blank=True, null=True)
     mac_address = models.CharField(max_length=17, unique=True, null=True, blank=True)
     port = models.PositiveIntegerField(default=8000)
+    message_queue_length = models.PositiveSmallIntegerField(
+        default=10,
+        help_text="Maximum queued NetMessages to retain for this peer.",
+    )
     badge_color = models.CharField(max_length=7, default=DEFAULT_BADGE_COLOR)
     role = models.ForeignKey(NodeRole, on_delete=models.SET_NULL, null=True, blank=True)
     current_relation = models.CharField(
@@ -242,19 +259,244 @@ class Node(Entity):
     RPI_CAMERA_BINARIES = ("rpicam-hello", "rpicam-still", "rpicam-vid")
     AP_ROUTER_SSID = "gelectriic-ap"
     NMCLI_TIMEOUT = 5
-    GWAY_RUNNER_COMMAND = "gway"
-    GWAY_RUNNER_CANDIDATES = ("~/.local/bin/gway", "/usr/local/bin/gway")
     AUTO_MANAGED_FEATURES = set(FEATURE_LOCK_MAP.keys()) | {
         "gui-toast",
         "rpi-camera",
         "ap-router",
-        "gway-runner",
     }
     MANUAL_FEATURE_SLUGS = {"clipboard-poll", "screenshot-poll", "audio-capture"}
 
     def __str__(self) -> str:  # pragma: no cover - simple representation
         return f"{self.hostname}:{self.port}"
 
+    @staticmethod
+    def _ip_preference(ip_value: str) -> tuple[int, str]:
+        """Return a sort key favouring globally routable addresses."""
+
+        try:
+            parsed = ipaddress.ip_address(ip_value)
+        except ValueError:
+            return (3, ip_value)
+
+        if parsed.is_global:
+            return (0, ip_value)
+
+        if parsed.is_loopback or parsed.is_link_local:
+            return (2, ip_value)
+
+        if parsed.is_private:
+            return (2, ip_value)
+
+        return (1, ip_value)
+
+    @classmethod
+    def _select_preferred_ip(cls, addresses: Iterable[str]) -> str | None:
+        """Return the preferred IP from ``addresses`` when available."""
+
+        best: tuple[int, str] | None = None
+        for candidate in addresses:
+            candidate = (candidate or "").strip()
+            if not candidate:
+                continue
+            score = cls._ip_preference(candidate)
+            if best is None or score < best:
+                best = score
+        return best[1] if best else None
+
+    @classmethod
+    def _resolve_ip_addresses(
+        cls, *hosts: str, include_ipv4: bool = True, include_ipv6: bool = True
+    ) -> tuple[list[str], list[str]]:
+        """Resolve ``hosts`` into IPv4 and IPv6 address lists."""
+
+        ipv4: list[str] = []
+        ipv6: list[str] = []
+
+        for host in hosts:
+            host = (host or "").strip()
+            if not host:
+                continue
+            try:
+                info = socket.getaddrinfo(
+                    host,
+                    None,
+                    socket.AF_UNSPEC,
+                    socket.SOCK_STREAM,
+                )
+            except OSError:
+                continue
+            for family, _, _, _, sockaddr in info:
+                if family == socket.AF_INET and include_ipv4:
+                    value = sockaddr[0]
+                    if value not in ipv4:
+                        ipv4.append(value)
+                elif family == socket.AF_INET6 and include_ipv6:
+                    value = sockaddr[0]
+                    if value not in ipv6:
+                        ipv6.append(value)
+
+        return ipv4, ipv6
+
+    def get_remote_host_candidates(self) -> list[str]:
+        """Return host strings that may reach this node."""
+
+        values: list[str] = []
+        for attr in (
+            "network_hostname",
+            "hostname",
+            "ipv6_address",
+            "ipv4_address",
+            "address",
+            "public_endpoint",
+        ):
+            value = getattr(self, attr, "") or ""
+            value = value.strip()
+            if value and value not in values:
+                values.append(value)
+
+        resolved_ipv6: list[str] = []
+        resolved_ipv4: list[str] = []
+        for host in list(values):
+            if host.startswith("http://") or host.startswith("https://"):
+                continue
+            try:
+                ipaddress.ip_address(host)
+            except ValueError:
+                ipv4, ipv6 = self._resolve_ip_addresses(host)
+                for candidate in ipv6:
+                    if candidate not in values and candidate not in resolved_ipv6:
+                        resolved_ipv6.append(candidate)
+                for candidate in ipv4:
+                    if candidate not in values and candidate not in resolved_ipv4:
+                        resolved_ipv4.append(candidate)
+        values.extend(resolved_ipv6)
+        values.extend(resolved_ipv4)
+        return values
+
+    def get_primary_contact(self) -> str:
+        """Return the first reachable host for this node."""
+
+        for host in self.get_remote_host_candidates():
+            if host:
+                return host
+        return ""
+
+    def get_best_ip(self) -> str:
+        """Return the preferred IP address for this node if known."""
+
+        candidates: list[str] = []
+        for value in (
+            getattr(self, "address", "") or "",
+            getattr(self, "ipv4_address", "") or "",
+            getattr(self, "ipv6_address", "") or "",
+        ):
+            value = value.strip()
+            if not value:
+                continue
+            try:
+                ipaddress.ip_address(value)
+            except ValueError:
+                continue
+            candidates.append(value)
+        if not candidates:
+            return ""
+        selected = self._select_preferred_ip(candidates)
+        return selected or ""
+
+    def iter_remote_urls(self, path: str):
+        """Yield potential remote URLs for ``path`` on this node."""
+
+        host_candidates = self.get_remote_host_candidates()
+        default_port = self.port or 8000
+        normalized_path = path if path.startswith("/") else f"/{path}"
+        seen: set[str] = set()
+
+        for host in host_candidates:
+            host = host.strip()
+            if not host:
+                continue
+            base_path = ""
+            formatted_host = host
+            port_override: int | None = None
+
+            if "://" in host:
+                parsed = urlparse(host)
+                netloc = parsed.netloc or parsed.path
+                base_path = (parsed.path or "").rstrip("/")
+                combined_path = (
+                    f"{base_path}{normalized_path}" if base_path else normalized_path
+                )
+                primary = urlunsplit((parsed.scheme, netloc, combined_path, "", ""))
+                if primary not in seen:
+                    seen.add(primary)
+                    yield primary
+                if parsed.scheme == "https":
+                    fallback = urlunsplit(("http", netloc, combined_path, "", ""))
+                    if fallback not in seen:
+                        seen.add(fallback)
+                        yield fallback
+                elif parsed.scheme == "http":
+                    alternate = urlunsplit(("https", netloc, combined_path, "", ""))
+                    if alternate not in seen:
+                        seen.add(alternate)
+                        yield alternate
+                continue
+
+            if host.startswith("[") and "]" in host:
+                end = host.index("]")
+                core_host = host[1:end]
+                remainder = host[end + 1 :]
+                if remainder.startswith(":"):
+                    remainder = remainder[1:]
+                    port_part, sep, path_tail = remainder.partition("/")
+                    if port_part:
+                        try:
+                            port_override = int(port_part)
+                        except ValueError:
+                            port_override = None
+                    if sep:
+                        base_path = f"/{path_tail}".rstrip("/")
+                elif "/" in remainder:
+                    _, _, path_tail = remainder.partition("/")
+                    base_path = f"/{path_tail}".rstrip("/")
+                formatted_host = f"[{core_host}]"
+            else:
+                if "/" in host:
+                    host_only, _, path_tail = host.partition("/")
+                    formatted_host = host_only or host
+                    base_path = f"/{path_tail}".rstrip("/")
+                try:
+                    ip_obj = ipaddress.ip_address(formatted_host)
+                except ValueError:
+                    parts = formatted_host.rsplit(":", 1)
+                    if len(parts) == 2 and parts[1].isdigit():
+                        formatted_host = parts[0]
+                        port_override = int(parts[1])
+                    try:
+                        ip_obj = ipaddress.ip_address(formatted_host)
+                    except ValueError:
+                        ip_obj = None
+                else:
+                    if ip_obj.version == 6 and not formatted_host.startswith("["):
+                        formatted_host = f"[{formatted_host}]"
+
+            effective_port = port_override if port_override is not None else default_port
+            combined_path = f"{base_path}{normalized_path}" if base_path else normalized_path
+
+            for scheme, scheme_default_port in (("https", 443), ("http", 80)):
+                base = f"{scheme}://{formatted_host}"
+                if effective_port and (
+                    port_override is not None or effective_port != scheme_default_port
+                ):
+                    explicit = f"{base}:{effective_port}{combined_path}"
+                    if explicit not in seen:
+                        seen.add(explicit)
+                        yield explicit
+                candidate = f"{base}{combined_path}"
+                if candidate not in seen:
+                    seen.add(candidate)
+                    yield candidate
+
     @staticmethod
     def get_current_mac() -> str:
         """Return the MAC address of the current host."""
@@ -285,7 +527,14 @@ class Node(Entity):
         """Return the node representing the current host if it exists."""
         mac = cls.get_current_mac()
         try:
-            return cls.objects.filter(mac_address=mac).first()
+            node = cls.objects.filter(mac_address__iexact=mac).first()
+            if node:
+                return node
+            return (
+                cls.objects.filter(current_relation=cls.Relation.SELF)
+                .filter(Q(mac_address__isnull=True) | Q(mac_address=""))
+                .first()
+            )
         except DatabaseError:
             logger.debug("nodes.Node.get_local skipped: database unavailable", exc_info=True)
             return None
@@ -293,11 +542,66 @@ class Node(Entity):
     @classmethod
     def register_current(cls):
         """Create or update the :class:`Node` entry for this host."""
-        hostname = socket.gethostname()
+        hostname_override = (
+            os.environ.get("NODE_HOSTNAME")
+            or os.environ.get("HOSTNAME")
+            or ""
+        )
+        hostname_override = hostname_override.strip()
+        hostname = hostname_override or socket.gethostname()
+
+        network_hostname = os.environ.get("NODE_PUBLIC_HOSTNAME", "").strip()
+        if not network_hostname:
+            fqdn = socket.getfqdn(hostname)
+            if fqdn and "." in fqdn:
+                network_hostname = fqdn
+
+        ipv4_override = os.environ.get("NODE_PUBLIC_IPV4", "").strip()
+        ipv6_override = os.environ.get("NODE_PUBLIC_IPV6", "").strip()
+
+        ipv4_candidates: list[str] = []
+        ipv6_candidates: list[str] = []
+
+        for override, version in ((ipv4_override, 4), (ipv6_override, 6)):
+            override = override.strip()
+            if not override:
+                continue
+            try:
+                parsed = ipaddress.ip_address(override)
+            except ValueError:
+                continue
+            if parsed.version == version:
+                if version == 4 and override not in ipv4_candidates:
+                    ipv4_candidates.append(override)
+                elif version == 6 and override not in ipv6_candidates:
+                    ipv6_candidates.append(override)
+
+        resolve_hosts: list[str] = []
+        for value in (network_hostname, hostname_override, hostname):
+            value = (value or "").strip()
+            if value and value not in resolve_hosts:
+                resolve_hosts.append(value)
+
+        resolved_ipv4, resolved_ipv6 = cls._resolve_ip_addresses(*resolve_hosts)
+        for ip_value in resolved_ipv4:
+            if ip_value not in ipv4_candidates:
+                ipv4_candidates.append(ip_value)
+        for ip_value in resolved_ipv6:
+            if ip_value not in ipv6_candidates:
+                ipv6_candidates.append(ip_value)
+
         try:
-            address = socket.gethostbyname(hostname)
+            direct_address = socket.gethostbyname(hostname)
         except OSError:
-            address = "127.0.0.1"
+            direct_address = ""
+
+        if direct_address and direct_address not in ipv4_candidates:
+            ipv4_candidates.append(direct_address)
+
+        ipv4_address = cls._select_preferred_ip(ipv4_candidates)
+        ipv6_address = cls._select_preferred_ip(ipv6_candidates)
+
+        preferred_contact = ipv4_address or ipv6_address or direct_address or "127.0.0.1"
         port = int(os.environ.get("PORT", 8000))
         base_path = str(settings.BASE_DIR)
         ver_path = Path(settings.BASE_DIR) / "VERSION"
@@ -305,13 +609,20 @@ class Node(Entity):
         rev_value = revision.get_revision()
         installed_revision = rev_value if rev_value else ""
         mac = cls.get_current_mac()
-        slug = slugify(hostname)
+        endpoint_override = os.environ.get("NODE_PUBLIC_ENDPOINT", "").strip()
+        slug_source = endpoint_override or hostname
+        slug = slugify(slug_source)
+        if not slug:
+            slug = cls._generate_unique_public_endpoint(hostname or mac)
         node = cls.objects.filter(mac_address=mac).first()
         if not node:
             node = cls.objects.filter(public_endpoint=slug).first()
         defaults = {
             "hostname": hostname,
-            "address": address,
+            "network_hostname": network_hostname,
+            "ipv4_address": ipv4_address,
+            "ipv6_address": ipv6_address,
+            "address": preferred_contact,
             "port": port,
             "base_path": base_path,
             "installed_version": installed_version,
@@ -322,6 +633,7 @@ class Node(Entity):
         }
         role_lock = Path(settings.BASE_DIR) / "locks" / "role.lck"
         role_name = role_lock.read_text().strip() if role_lock.exists() else "Terminal"
+        role_name = ROLE_RENAMES.get(role_name, role_name)
         desired_role = NodeRole.objects.filter(name=role_name).first()
 
         if node:
@@ -388,7 +700,10 @@ class Node(Entity):
 
         payload = {
             "hostname": self.hostname,
+            "network_hostname": self.network_hostname,
             "address": self.address,
+            "ipv4_address": self.ipv4_address,
+            "ipv6_address": self.ipv6_address,
             "port": self.port,
             "mac_address": self.mac_address,
             "public_key": self.public_key,
@@ -405,17 +720,18 @@ class Node(Entity):
 
         peers = Node.objects.exclude(pk=self.pk)
         for peer in peers:
-            host_candidates: list[str] = []
-            if peer.address:
-                host_candidates.append(peer.address)
-            if peer.hostname and peer.hostname not in host_candidates:
-                host_candidates.append(peer.hostname)
+            host_candidates = peer.get_remote_host_candidates()
             port = peer.port or 8000
             urls: list[str] = []
             for host in host_candidates:
                 host = host.strip()
                 if not host:
                     continue
+                if host.startswith("http://") or host.startswith("https://"):
+                    normalized = host.rstrip("/")
+                    if normalized not in urls:
+                        urls.append(normalized)
+                    continue
                 if ":" in host and not host.startswith("["):
                     host = f"[{host}]"
                 http_url = (
@@ -461,7 +777,24 @@ class Node(Entity):
         security_dir.mkdir(parents=True, exist_ok=True)
         priv_path = security_dir / f"{self.public_endpoint}"
         pub_path = security_dir / f"{self.public_endpoint}.pub"
-        if not priv_path.exists() or not pub_path.exists():
+        regenerate = not priv_path.exists() or not pub_path.exists()
+        if not regenerate:
+            key_max_age = getattr(settings, "NODE_KEY_MAX_AGE", timedelta(days=90))
+            if key_max_age is not None:
+                try:
+                    priv_mtime = datetime.fromtimestamp(
+                        priv_path.stat().st_mtime, tz=datetime_timezone.utc
+                    )
+                    pub_mtime = datetime.fromtimestamp(
+                        pub_path.stat().st_mtime, tz=datetime_timezone.utc
+                    )
+                except OSError:
+                    regenerate = True
+                else:
+                    cutoff = timezone.now() - key_max_age
+                    if priv_mtime < cutoff or pub_mtime < cutoff:
+                        regenerate = True
+        if regenerate:
             private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
             private_bytes = private_key.private_bytes(
                 encoding=serialization.Encoding.PEM,
@@ -474,12 +807,35 @@ class Node(Entity):
             )
             priv_path.write_bytes(private_bytes)
             pub_path.write_bytes(public_bytes)
-            self.public_key = public_bytes.decode()
-            self.save(update_fields=["public_key"])
+            public_text = public_bytes.decode()
+            if self.public_key != public_text:
+                self.public_key = public_text
+                self.save(update_fields=["public_key"])
         elif not self.public_key:
             self.public_key = pub_path.read_text()
             self.save(update_fields=["public_key"])
 
+    def get_private_key(self):
+        """Return the private key for this node if available."""
+
+        if not self.public_endpoint:
+            return None
+        try:
+            self.ensure_keys()
+        except Exception:
+            return None
+        priv_path = (
+            Path(self.base_path or settings.BASE_DIR)
+            / "security"
+            / f"{self.public_endpoint}"
+        )
+        try:
+            return serialization.load_pem_private_key(
+                priv_path.read_bytes(), password=None
+            )
+        except Exception:
+            return None
+
     @property
     def is_local(self):
         """Determine if this node represents the current host."""
@@ -671,21 +1027,6 @@ class Node(Entity):
                 return True
         return False
 
-    @classmethod
-    def _find_gway_runner_command(cls) -> str | None:
-        command = shutil.which(cls.GWAY_RUNNER_COMMAND)
-        if command:
-            return command
-        for candidate in cls.GWAY_RUNNER_CANDIDATES:
-            expanded = os.path.expanduser(candidate)
-            if os.path.isfile(expanded) and os.access(expanded, os.X_OK):
-                return expanded
-        return None
-
-    @classmethod
-    def _has_gway_runner(cls) -> bool:
-        return cls._find_gway_runner_command() is not None
-
     def refresh_features(self):
         if not self.pk:
             return
@@ -700,8 +1041,6 @@ class Node(Entity):
                 detected_slugs.add(slug)
         if self._has_rpi_camera():
             detected_slugs.add("rpi-camera")
-        if self._has_gway_runner():
-            detected_slugs.add("gway-runner")
         if self._hosts_gelectriic_ap():
             detected_slugs.add("ap-router")
         try:
@@ -754,6 +1093,7 @@ class Node(Entity):
         self._sync_screenshot_task(screenshot_enabled)
         self._sync_landing_lead_task(celery_enabled)
         self._sync_ocpp_session_report_task(celery_enabled)
+        self._sync_upstream_poll_task(celery_enabled)
 
     def _sync_clipboard_task(self, enabled: bool):
         from django_celery_beat.models import IntervalSchedule, PeriodicTask
@@ -859,6 +1199,28 @@ class Node(Entity):
         except (OperationalError, ProgrammingError):
             logger.debug("Skipping OCPP session report task sync; tables not ready")
 
+    def _sync_upstream_poll_task(self, celery_enabled: bool):
+        if not self.is_local:
+            return
+
+        from django_celery_beat.models import IntervalSchedule, PeriodicTask
+
+        task_name = "nodes_poll_upstream_messages"
+        if celery_enabled:
+            schedule, _ = IntervalSchedule.objects.get_or_create(
+                every=1, period=IntervalSchedule.MINUTES
+            )
+            PeriodicTask.objects.update_or_create(
+                name=task_name,
+                defaults={
+                    "interval": schedule,
+                    "task": "nodes.tasks.poll_unreachable_upstream",
+                    "enabled": True,
+                },
+            )
+        else:
+            PeriodicTask.objects.filter(name=task_name).delete()
+
     def send_mail(
         self,
         subject: str,
@@ -1024,8 +1386,8 @@ class NodeManager(Profile):
     )
 
     class Meta:
-        verbose_name = "Node Manager"
-        verbose_name_plural = "Node Managers"
+        verbose_name = "Node Profile"
+        verbose_name_plural = "Node Profiles"
 
     def __str__(self) -> str:
         owner = self.owner_display()
@@ -1410,7 +1772,6 @@ class NetMessage(Entity):
     propagated_to = models.ManyToManyField(
         Node, blank=True, related_name="received_net_messages"
     )
-    confirmed_peers = models.JSONField(default=dict, blank=True)
     created = models.DateTimeField(auto_now_add=True)
     complete = models.BooleanField(default=False, editable=False)
 
@@ -1479,6 +1840,7 @@ class NetMessage(Entity):
         payload = attachments if attachments is not None else self.attachments or []
         if not payload:
             return
+
         try:
             objects = list(
                 serializers.deserialize(
@@ -1498,6 +1860,193 @@ class NetMessage(Entity):
                     self.pk,
                 )
 
+    def _build_payload(
+        self,
+        *,
+        sender_id: str | None,
+        origin_uuid: str | None,
+        reach_name: str | None,
+        seen: list[str],
+    ) -> dict[str, object]:
+        payload: dict[str, object] = {
+            "uuid": str(self.uuid),
+            "subject": self.subject,
+            "body": self.body,
+            "seen": list(seen),
+            "reach": reach_name,
+            "sender": sender_id,
+            "origin": origin_uuid,
+        }
+        if self.attachments:
+            payload["attachments"] = self.attachments
+        if self.filter_node:
+            payload["filter_node"] = str(self.filter_node.uuid)
+        if self.filter_node_feature:
+            payload["filter_node_feature"] = self.filter_node_feature.slug
+        if self.filter_node_role:
+            payload["filter_node_role"] = self.filter_node_role.name
+        if self.filter_current_relation:
+            payload["filter_current_relation"] = self.filter_current_relation
+        if self.filter_installed_version:
+            payload["filter_installed_version"] = self.filter_installed_version
+        if self.filter_installed_revision:
+            payload["filter_installed_revision"] = self.filter_installed_revision
+        return payload
+
+    @staticmethod
+    def _serialize_payload(payload: dict[str, object]) -> str:
+        return json.dumps(payload, separators=(",", ":"), sort_keys=True)
+
+    @staticmethod
+    def _sign_payload(payload_json: str, private_key) -> str | None:
+        if not private_key:
+            return None
+        try:
+            signature = private_key.sign(
+                payload_json.encode(),
+                padding.PKCS1v15(),
+                hashes.SHA256(),
+            )
+        except Exception:
+            return None
+        return base64.b64encode(signature).decode()
+
+    def queue_for_node(self, node: "Node", seen: list[str]) -> None:
+        """Queue this message for later delivery to ``node``."""
+
+        if node.current_relation != Node.Relation.DOWNSTREAM:
+            return
+
+        now = timezone.now()
+        expires_at = now + timedelta(hours=1)
+        normalized_seen = [str(value) for value in seen]
+        entry, created = PendingNetMessage.objects.get_or_create(
+            node=node,
+            message=self,
+            defaults={
+                "seen": normalized_seen,
+                "stale_at": expires_at,
+            },
+        )
+        if created:
+            entry.queued_at = now
+            entry.save(update_fields=["queued_at"])
+        else:
+            entry.seen = normalized_seen
+            entry.stale_at = expires_at
+            entry.queued_at = now
+            entry.save(update_fields=["seen", "stale_at", "queued_at"])
+        self._trim_queue(node)
+
+    def clear_queue_for_node(self, node: "Node") -> None:
+        PendingNetMessage.objects.filter(node=node, message=self).delete()
+
+    def _trim_queue(self, node: "Node") -> None:
+        limit = max(int(node.message_queue_length or 0), 0)
+        if limit == 0:
+            PendingNetMessage.objects.filter(node=node).delete()
+            return
+        qs = PendingNetMessage.objects.filter(node=node).order_by("-queued_at")
+        keep_ids = list(qs.values_list("pk", flat=True)[:limit])
+        if keep_ids:
+            PendingNetMessage.objects.filter(node=node).exclude(pk__in=keep_ids).delete()
+        else:
+            qs.delete()
+
+    @classmethod
+    def receive_payload(
+        cls,
+        data: dict[str, object],
+        *,
+        sender: "Node",
+    ) -> "NetMessage":
+        msg_uuid = data.get("uuid")
+        if not msg_uuid:
+            raise ValueError("uuid required")
+        subject = (data.get("subject") or "")[:64]
+        body = (data.get("body") or "")[:256]
+        attachments = cls.normalize_attachments(data.get("attachments"))
+        reach_name = data.get("reach")
+        reach_role = None
+        if reach_name:
+            reach_role = NodeRole.objects.filter(name=reach_name).first()
+        filter_node_uuid = data.get("filter_node")
+        filter_node = None
+        if filter_node_uuid:
+            filter_node = Node.objects.filter(uuid=filter_node_uuid).first()
+        filter_feature_slug = data.get("filter_node_feature")
+        filter_feature = None
+        if filter_feature_slug:
+            filter_feature = NodeFeature.objects.filter(slug=filter_feature_slug).first()
+        filter_role_name = data.get("filter_node_role")
+        filter_role = None
+        if filter_role_name:
+            filter_role = NodeRole.objects.filter(name=filter_role_name).first()
+        filter_relation_value = data.get("filter_current_relation")
+        filter_relation = ""
+        if filter_relation_value:
+            relation = Node.normalize_relation(filter_relation_value)
+            filter_relation = relation.value if relation else ""
+        filter_installed_version = (data.get("filter_installed_version") or "")[:20]
+        filter_installed_revision = (data.get("filter_installed_revision") or "")[:40]
+        seen_values = data.get("seen", [])
+        if not isinstance(seen_values, list):
+            seen_values = list(seen_values)  # type: ignore[arg-type]
+        normalized_seen = [str(value) for value in seen_values if value is not None]
+        origin_id = data.get("origin")
+        origin_node = None
+        if origin_id:
+            origin_node = Node.objects.filter(uuid=origin_id).first()
+        if not origin_node:
+            origin_node = sender
+        msg, created = cls.objects.get_or_create(
+            uuid=msg_uuid,
+            defaults={
+                "subject": subject,
+                "body": body,
+                "reach": reach_role,
+                "node_origin": origin_node,
+                "attachments": attachments or None,
+                "filter_node": filter_node,
+                "filter_node_feature": filter_feature,
+                "filter_node_role": filter_role,
+                "filter_current_relation": filter_relation,
+                "filter_installed_version": filter_installed_version,
+                "filter_installed_revision": filter_installed_revision,
+            },
+        )
+        if not created:
+            msg.subject = subject
+            msg.body = body
+            update_fields = ["subject", "body"]
+            if reach_role and msg.reach_id != reach_role.id:
+                msg.reach = reach_role
+                update_fields.append("reach")
+            if msg.node_origin_id is None and origin_node:
+                msg.node_origin = origin_node
+                update_fields.append("node_origin")
+            if attachments and msg.attachments != attachments:
+                msg.attachments = attachments
+                update_fields.append("attachments")
+            field_updates = {
+                "filter_node": filter_node,
+                "filter_node_feature": filter_feature,
+                "filter_node_role": filter_role,
+                "filter_current_relation": filter_relation,
+                "filter_installed_version": filter_installed_version,
+                "filter_installed_revision": filter_installed_revision,
+            }
+            for field, value in field_updates.items():
+                if getattr(msg, field) != value:
+                    setattr(msg, field, value)
+                    update_fields.append(field)
+            if update_fields:
+                msg.save(update_fields=update_fields)
+        if attachments:
+            msg.apply_attachments(attachments)
+        msg.propagate(seen=normalized_seen)
+        return msg
+
     def propagate(self, seen: list[str] | None = None):
         from core.notifications import notify
         import random
@@ -1532,17 +2081,7 @@ class NetMessage(Entity):
             local_id = str(local.uuid)
             if local_id not in seen:
                 seen.append(local_id)
-            priv_path = (
-                Path(local.base_path or settings.BASE_DIR)
-                / "security"
-                / f"{local.public_endpoint}"
-            )
-            try:
-                private_key = serialization.load_pem_private_key(
-                    priv_path.read_bytes(), password=None
-                )
-            except Exception:
-                private_key = None
+            private_key = local.get_private_key()
         for node_id in seen:
             node = Node.objects.filter(uuid=node_id).first()
             if node and (not local or node.pk != local.pk):
@@ -1592,11 +2131,18 @@ class NetMessage(Entity):
         reach_source = self.filter_node_role or self.reach
         reach_name = reach_source.name if reach_source else None
         role_map = {
+            "Interface": ["Interface", "Terminal"],
             "Terminal": ["Terminal"],
             "Control": ["Control", "Terminal"],
             "Satellite": ["Satellite", "Control", "Terminal"],
+            "Watchtower": [
+                "Watchtower",
+                "Satellite",
+                "Control",
+                "Terminal",
+            ],
             "Constellation": [
-                "Constellation",
+                "Watchtower",
                 "Satellite",
                 "Control",
                 "Terminal",
@@ -1640,72 +2186,45 @@ class NetMessage(Entity):
         seen_list = seen.copy()
         selected_ids = [str(n.uuid) for n in selected]
         payload_seen = seen_list + selected_ids
-        confirmed_peers = dict(self.confirmed_peers or {})
-
         for node in selected:
-            now = timezone.now().isoformat()
-            payload = {
-                "uuid": str(self.uuid),
-                "subject": self.subject,
-                "body": self.body,
-                "seen": payload_seen,
-                "reach": reach_name,
-                "sender": local_id,
-                "origin": origin_uuid,
-            }
-            if self.attachments:
-                payload["attachments"] = self.attachments
-            if self.filter_node:
-                payload["filter_node"] = str(self.filter_node.uuid)
-            if self.filter_node_feature:
-                payload["filter_node_feature"] = self.filter_node_feature.slug
-            if self.filter_node_role:
-                payload["filter_node_role"] = self.filter_node_role.name
-            if self.filter_current_relation:
-                payload["filter_current_relation"] = self.filter_current_relation
-            if self.filter_installed_version:
-                payload["filter_installed_version"] = self.filter_installed_version
-            if self.filter_installed_revision:
-                payload["filter_installed_revision"] = self.filter_installed_revision
-            payload_json = json.dumps(payload, separators=(",", ":"), sort_keys=True)
+            payload = self._build_payload(
+                sender_id=local_id,
+                origin_uuid=origin_uuid,
+                reach_name=reach_name,
+                seen=payload_seen,
+            )
+            payload_json = self._serialize_payload(payload)
             headers = {"Content-Type": "application/json"}
-            if private_key:
+            signature = self._sign_payload(payload_json, private_key)
+            if signature:
+                headers["X-Signature"] = signature
+            success = False
+            for url in node.iter_remote_urls("/nodes/net-message/"):
                 try:
-                    signature = private_key.sign(
-                        payload_json.encode(),
-                        padding.PKCS1v15(),
-                        hashes.SHA256(),
+                    response = requests.post(
+                        url,
+                        data=payload_json,
+                        headers=headers,
+                        timeout=1,
                     )
-                    headers["X-Signature"] = base64.b64encode(signature).decode()
+                    success = bool(response.ok)
                 except Exception:
-                    pass
-            status_entry = {
-                "status": "pending",
-                "status_code": None,
-                "updated": now,
-            }
-            try:
-                response = requests.post(
-                    f"http://{node.address}:{node.port}/nodes/net-message/",
-                    data=payload_json,
-                    headers=headers,
-                    timeout=1,
-                )
-                status_entry["status_code"] = getattr(response, "status_code", None)
-                if getattr(response, "ok", False):
-                    status_entry["status"] = "acknowledged"
-                else:
-                    status_entry["status"] = "failed"
-            except Exception:
-                status_entry["status"] = "error"
+                    logger.exception(
+                        "Failed to propagate NetMessage %s to node %s via %s",
+                        self.pk,
+                        node.pk,
+                        url,
+                    )
+                    continue
+                if success:
+                    break
+            if success:
+                self.clear_queue_for_node(node)
+            else:
+                self.queue_for_node(node, payload_seen)
             self.propagated_to.add(node)
-            confirmed_peers[str(node.uuid)] = status_entry
 
         save_fields: list[str] = []
-        if confirmed_peers != (self.confirmed_peers or {}):
-            self.confirmed_peers = confirmed_peers
-            save_fields.append("confirmed_peers")
-
         if total_known and self.propagated_to.count() >= total_known:
             self.complete = True
             save_fields.append("complete")
@@ -1714,6 +2233,32 @@ class NetMessage(Entity):
             self.save(update_fields=save_fields)
 
 
+class PendingNetMessage(models.Model):
+    """Queued :class:`NetMessage` awaiting delivery to a downstream node."""
+
+    node = models.ForeignKey(
+        Node, on_delete=models.CASCADE, related_name="pending_net_messages"
+    )
+    message = models.ForeignKey(
+        NetMessage,
+        on_delete=models.CASCADE,
+        related_name="pending_deliveries",
+    )
+    seen = models.JSONField(default=list)
+    queued_at = models.DateTimeField(auto_now_add=True)
+    stale_at = models.DateTimeField()
+
+    class Meta:
+        unique_together = ("node", "message")
+        ordering = ("queued_at",)
+
+    def __str__(self) -> str:  # pragma: no cover - simple representation
+        return f"{self.message_id} → {self.node_id}"
+
+    @property
+    def is_stale(self) -> bool:
+        return self.stale_at <= timezone.now()
+
 class ContentSample(Entity):
     """Collected content such as text snippets or screenshots."""