arthexis 0.1.18__py3-none-any.whl → 0.1.20__py3-none-any.whl

nodes/tests.py

@@ -2,6 +2,7 @@ import os
 
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings")
 import django
+import pytest
 
 try:  # Use the pytest-specific setup when available for database readiness
     from tests.conftest import safe_setup as _safe_setup  # type: ignore
@@ -18,6 +19,7 @@ from types import SimpleNamespace
 import unittest.mock as mock
 from unittest.mock import patch, call, MagicMock
 from django.core import mail
+from django.core.cache import cache
 from django.core.mail import EmailMessage
 from django.core.management import call_command
 import socket
@@ -38,6 +40,7 @@ from django.contrib.auth.models import Permission
 from django_celery_beat.models import IntervalSchedule, PeriodicTask
 from django.conf import settings
 from django.utils import timezone
+from urllib.parse import urlparse
 from dns import resolver as dns_resolver
 from . import dns as dns_utils
 from selenium.common.exceptions import WebDriverException
@@ -56,11 +59,12 @@ from .models import (
     NodeFeature,
     NodeFeatureAssignment,
     NetMessage,
+    PendingNetMessage,
     NodeManager,
     DNSRecord,
 )
 from .backends import OutboxEmailBackend
-from .tasks import capture_node_screenshot, sample_clipboard
+from .tasks import capture_node_screenshot, poll_unreachable_upstream, sample_clipboard
 from cryptography.hazmat.primitives.asymmetric import rsa, padding
 from cryptography.hazmat.primitives import serialization, hashes
 from core.models import Package, PackageRelease, SecurityGroup, RFID, EnergyAccount
@@ -68,16 +72,16 @@ from core.models import Package, PackageRelease, SecurityGroup, RFID, EnergyAcco
 
 class NodeBadgeColorTests(TestCase):
     def setUp(self):
-        self.constellation, _ = NodeRole.objects.get_or_create(name="Constellation")
+        self.watchtower, _ = NodeRole.objects.get_or_create(name="Watchtower")
         self.control, _ = NodeRole.objects.get_or_create(name="Control")
 
-    def test_constellation_role_defaults_to_goldenrod(self):
+    def test_watchtower_role_defaults_to_goldenrod(self):
         node = Node.objects.create(
-            hostname="constellation",
+            hostname="watchtower",
             address="10.1.0.1",
             port=8000,
             mac_address="00:aa:bb:cc:dd:01",
-            role=self.constellation,
+            role=self.watchtower,
         )
         self.assertEqual(node.badge_color, "#daa520")
 
@@ -97,7 +101,7 @@ class NodeBadgeColorTests(TestCase):
             address="10.1.0.3",
             port=8002,
             mac_address="00:aa:bb:cc:dd:03",
-            role=self.constellation,
+            role=self.watchtower,
             badge_color="#123456",
         )
         self.assertEqual(node.badge_color, "#123456")
@@ -110,6 +114,7 @@ class NodeTests(TestCase):
         self.user = User.objects.create_user(username="nodeuser", password="pwd")
         self.client.force_login(self.user)
         NodeRole.objects.get_or_create(name="Terminal")
+        NodeRole.objects.get_or_create(name="Interface")
 
 
 class NodeGetLocalDatabaseUnavailableTests(SimpleTestCase):
@@ -177,7 +182,7 @@ class NodeGetLocalTests(TestCase):
 
     def test_register_current_updates_role_from_lock_file(self):
         NodeRole.objects.get_or_create(name="Terminal")
-        NodeRole.objects.get_or_create(name="Constellation")
+        NodeRole.objects.get_or_create(name="Watchtower")
         with TemporaryDirectory() as tmp:
             base = Path(tmp)
             lock_dir = base / "locks"
@@ -202,7 +207,7 @@ class NodeGetLocalTests(TestCase):
             self.assertTrue(created)
             self.assertEqual(node.role.name, "Terminal")
 
-            role_file.write_text("Constellation")
+            role_file.write_text("Watchtower")
             with override_settings(BASE_DIR=base):
                 with (
                     patch(
@@ -221,7 +226,67 @@ class NodeGetLocalTests(TestCase):
 
             self.assertFalse(created_again)
             node.refresh_from_db()
-            self.assertEqual(node.role.name, "Constellation")
+            self.assertEqual(node.role.name, "Watchtower")
+
+            role_file.write_text("Constellation")
+            with override_settings(BASE_DIR=base):
+                with (
+                    patch(
+                        "nodes.models.Node.get_current_mac",
+                        return_value="00:aa:bb:cc:dd:ee",
+                    ),
+                    patch("nodes.models.socket.gethostname", return_value="role-host"),
+                    patch(
+                        "nodes.models.socket.gethostbyname", return_value="127.0.0.1"
+                    ),
+                    patch("nodes.models.revision.get_revision", return_value="rev"),
+                    patch.object(Node, "ensure_keys"),
+                    patch.object(Node, "notify_peers_of_update"),
+                ):
+                    Node.register_current()
+
+            node.refresh_from_db()
+            self.assertEqual(node.role.name, "Watchtower")
+
+    def test_register_current_respects_node_hostname_env(self):
+        with TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            with override_settings(BASE_DIR=base):
+                with (
+                    patch.dict(os.environ, {"NODE_HOSTNAME": "gway-002"}, clear=False),
+                    patch("nodes.models.Node.get_current_mac", return_value="00:11:22:33:44:55"),
+                    patch("nodes.models.socket.gethostname", return_value="localhost"),
+                    patch("nodes.models.socket.gethostbyname", return_value="127.0.0.1"),
+                    patch("nodes.models.revision.get_revision", return_value="rev"),
+                    patch.object(Node, "ensure_keys"),
+                    patch.object(Node, "notify_peers_of_update"),
+                ):
+                    node, created = Node.register_current()
+        self.assertTrue(created)
+        self.assertEqual(node.hostname, "gway-002")
+        self.assertEqual(node.public_endpoint, "gway-002")
+
+    def test_register_current_respects_public_endpoint_env(self):
+        with TemporaryDirectory() as tmp:
+            base = Path(tmp)
+            with override_settings(BASE_DIR=base):
+                with (
+                    patch.dict(
+                        os.environ,
+                        {"NODE_HOSTNAME": "gway-alpha", "NODE_PUBLIC_ENDPOINT": "gway-002"},
+                        clear=False,
+                    ),
+                    patch("nodes.models.Node.get_current_mac", return_value="00:11:22:33:44:56"),
+                    patch("nodes.models.socket.gethostname", return_value="localhost"),
+                    patch("nodes.models.socket.gethostbyname", return_value="127.0.0.1"),
+                    patch("nodes.models.revision.get_revision", return_value="rev"),
+                    patch.object(Node, "ensure_keys"),
+                    patch.object(Node, "notify_peers_of_update"),
+                ):
+                    node, created = Node.register_current()
+        self.assertTrue(created)
+        self.assertEqual(node.hostname, "gway-alpha")
+        self.assertEqual(node.public_endpoint, "gway-002")
 
     def test_register_and_list_node(self):
         response = self.client.post(
@@ -309,6 +374,43 @@ class NodeGetLocalTests(TestCase):
         self.assertNotEqual(node_one.public_endpoint, node_two.public_endpoint)
         self.assertTrue(node_two.public_endpoint.startswith("duplicate-host-"))
 
+    def test_register_node_assigns_interface_role_and_returns_uuid(self):
+        NodeRole.objects.get_or_create(name="Interface")
+        private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        public_bytes = private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        ).decode()
+        token = "interface-token"
+        signature = base64.b64encode(
+            private_key.sign(
+                token.encode(),
+                padding.PKCS1v15(),
+                hashes.SHA256(),
+            )
+        ).decode()
+        mac = "aa:bb:cc:dd:ee:99"
+        payload = {
+            "hostname": "interface",
+            "address": "127.0.0.1",
+            "port": 8443,
+            "mac_address": mac,
+            "public_key": public_bytes,
+            "token": token,
+            "signature": signature,
+            "role": "Interface",
+        }
+        response = self.client.post(
+            reverse("register-node"),
+            data=json.dumps(payload),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIn("uuid", data)
+        node = Node.objects.get(mac_address=mac)
+        self.assertEqual(node.role.name, "Interface")
+
     def test_register_node_feature_toggle(self):
         NodeFeature.objects.get_or_create(
             slug="clipboard-poll", defaults={"display": "Clipboard Poll"}
@@ -1236,6 +1338,7 @@ class NodeRegisterCurrentTests(TestCase):
         existing_role.refresh_from_db()
         self.assertEqual(existing_role.description, "updated via attachment")
 
+    @pytest.mark.feature("clipboard-poll")
     def test_clipboard_polling_creates_task(self):
         feature, _ = NodeFeature.objects.get_or_create(
             slug="clipboard-poll", defaults={"display": "Clipboard Poll"}
@@ -1253,6 +1356,7 @@ class NodeRegisterCurrentTests(TestCase):
         NodeFeatureAssignment.objects.filter(node=node, feature=feature).delete()
         self.assertFalse(PeriodicTask.objects.filter(name=task_name).exists())
 
+    @pytest.mark.feature("screenshot-poll")
     def test_screenshot_polling_creates_task(self):
         feature, _ = NodeFeature.objects.get_or_create(
             slug="screenshot-poll", defaults={"display": "Screenshot Poll"}
@@ -1381,6 +1485,7 @@ class NodeAdminTests(TestCase):
         action_url = reverse("admin:core_rfid_scan")
         self.assertContains(response, f'href="{action_url}"')
 
+    @pytest.mark.feature("rpi-camera")
     def test_node_feature_list_shows_all_actions_for_rpi_camera(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1393,6 +1498,7 @@ class NodeAdminTests(TestCase):
         self.assertContains(response, f'href="{snapshot_url}"')
         self.assertContains(response, f'href="{stream_url}"')
 
+    @pytest.mark.feature("audio-capture")
     def test_node_feature_list_shows_waveform_action_when_enabled(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1403,6 +1509,7 @@ class NodeAdminTests(TestCase):
         action_url = reverse("admin:nodes_nodefeature_view_waveform")
         self.assertContains(response, f'href="{action_url}"')
 
+    @pytest.mark.feature("screenshot-poll")
     def test_node_feature_list_hides_default_action_when_disabled(self):
         self._create_local_node()
         NodeFeature.objects.get_or_create(
@@ -1495,6 +1602,7 @@ class NodeAdminTests(TestCase):
             response, reverse("admin:nodes_node_register_current")
         )
 
+    @pytest.mark.feature("screenshot-poll")
     @patch("nodes.admin.capture_screenshot")
     def test_capture_site_screenshot_from_admin(self, mock_capture_screenshot):
         screenshot_dir = settings.LOG_DIR / "screenshots"
@@ -1540,6 +1648,48 @@ class NodeAdminTests(TestCase):
         self.assertEqual(response.status_code, 200)
         self.assertContains(response, "data:image/png;base64")
 
+    @patch("nodes.admin.requests.post")
+    def test_proxy_view_uses_remote_login_url(self, mock_post):
+        self.client.get(reverse("admin:nodes_node_register_current"))
+        local_node = Node.objects.get()
+        remote = Node.objects.create(
+            hostname="remote",
+            address="192.0.2.10",
+            port=8443,
+            mac_address="aa:bb:cc:dd:ee:ff",
+        )
+        mock_post.return_value = SimpleNamespace(
+            ok=True,
+            json=lambda: {
+                "login_url": "https://remote.example/nodes/proxy/login/token",
+                "expires": "2025-01-01T00:00:00",
+            },
+            status_code=200,
+            text="ok",
+        )
+        response = self.client.get(
+            reverse("admin:nodes_node_proxy", args=[remote.pk])
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertTemplateUsed(response, "admin/nodes/node/proxy.html")
+        self.assertContains(response, "<iframe", html=False)
+        mock_post.assert_called()
+        payload = json.loads(mock_post.call_args[1]["data"])
+        self.assertEqual(payload.get("requester"), str(local_node.uuid))
+
+    def test_proxy_link_displayed_for_remote_nodes(self):
+        Node.objects.create(
+            hostname="remote",
+            address="203.0.113.1",
+            port=8000,
+            mac_address="aa:aa:aa:aa:aa:01",
+        )
+        response = self.client.get(reverse("admin:nodes_node_changelist"))
+        proxy_url = reverse("admin:nodes_node_proxy", args=[1])
+        self.assertContains(response, proxy_url)
+
+
+    @pytest.mark.feature("screenshot-poll")
     @override_settings(SCREENSHOT_SOURCES=["/one", "/two"])
     @patch("nodes.admin.capture_screenshot")
     def test_take_screenshots_action(self, mock_capture):
@@ -1572,6 +1722,7 @@ class NodeAdminTests(TestCase):
         samples = list(ContentSample.objects.filter(kind=ContentSample.IMAGE))
         self.assertEqual(samples[0].transaction_uuid, samples[1].transaction_uuid)
 
+    @pytest.mark.feature("screenshot-poll")
     @patch("nodes.admin.capture_screenshot")
     def test_take_screenshot_default_action_creates_sample(
         self, mock_capture_screenshot
@@ -1646,6 +1797,7 @@ class NodeAdminTests(TestCase):
             response, "Completed 0 of 1 feature check(s) successfully.", html=False
         )
 
+    @pytest.mark.feature("screenshot-poll")
     def test_enable_selected_features_enables_manual_feature(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1690,6 +1842,7 @@ class NodeAdminTests(TestCase):
             html=False,
         )
 
+    @pytest.mark.feature("screenshot-poll")
     def test_take_screenshot_default_action_requires_enabled_feature(self):
         self._create_local_node()
         NodeFeature.objects.get_or_create(
@@ -1704,6 +1857,7 @@ class NodeAdminTests(TestCase):
         self.assertEqual(ContentSample.objects.count(), 0)
         self.assertContains(response, "Screenshot Poll feature is not enabled")
 
+    @pytest.mark.feature("rpi-camera")
     @patch("nodes.admin.capture_rpi_snapshot")
     def test_take_snapshot_default_action_creates_sample(self, mock_snapshot):
         node = self._create_local_node()
@@ -1726,6 +1880,7 @@ class NodeAdminTests(TestCase):
         change_url = reverse("admin:nodes_contentsample_change", args=[sample.pk])
         self.assertEqual(response.redirect_chain[-1][0], change_url)
 
+    @pytest.mark.feature("rpi-camera")
     def test_view_stream_requires_enabled_feature(self):
         self._create_local_node()
         NodeFeature.objects.get_or_create(
@@ -1741,6 +1896,7 @@ class NodeAdminTests(TestCase):
             response, "Raspberry Pi Camera feature is not enabled on this node."
         )
 
+    @pytest.mark.feature("rpi-camera")
     def test_view_stream_renders_when_feature_enabled(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1756,6 +1912,7 @@ class NodeAdminTests(TestCase):
         self.assertContains(response, expected_stream)
         self.assertContains(response, "camera-stream__frame")
 
+    @pytest.mark.feature("rpi-camera")
     def test_view_stream_uses_configured_stream_url(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1773,6 +1930,7 @@ class NodeAdminTests(TestCase):
         self.assertEqual(response.context_data["stream_embed"], "iframe")
         self.assertContains(response, configured_stream)
 
+    @pytest.mark.feature("rpi-camera")
     def test_view_stream_detects_mjpeg_stream(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1789,6 +1947,7 @@ class NodeAdminTests(TestCase):
         self.assertEqual(response.context_data["stream_embed"], "mjpeg")
         self.assertContains(response, "<img", html=False)
 
+    @pytest.mark.feature("rpi-camera")
     def test_view_stream_marks_rtsp_stream_as_unsupported(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -1805,6 +1964,7 @@ class NodeAdminTests(TestCase):
         self.assertEqual(response.context_data["stream_embed"], "unsupported")
         self.assertContains(response, "camera-stream__unsupported")
 
+    @pytest.mark.feature("audio-capture")
     def test_view_waveform_requires_enabled_feature(self):
         self._create_local_node()
         NodeFeature.objects.get_or_create(
@@ -1820,6 +1980,7 @@ class NodeAdminTests(TestCase):
             response, "Audio Capture feature is not enabled on this node."
         )
 
+    @pytest.mark.feature("audio-capture")
     def test_view_waveform_renders_when_feature_enabled(self):
         node = self._create_local_node()
         feature, _ = NodeFeature.objects.get_or_create(
@@ -2142,6 +2303,160 @@ class NodeAdminTests(TestCase):
         self.assertEqual(post_data["mac_address"], local.mac_address)
 
 
+class NodeProxyGatewayTests(TestCase):
+    def setUp(self):
+        cache.clear()
+        self.client = Client()
+        self.private_key = rsa.generate_private_key(
+            public_exponent=65537, key_size=2048
+        )
+        public_key = self.private_key.public_key().public_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PublicFormat.SubjectPublicKeyInfo,
+        ).decode()
+        self.node = Node.objects.create(
+            hostname="requester",
+            address="127.0.0.1",
+            port=8000,
+            mac_address="aa:bb:cc:dd:ee:aa",
+            public_key=public_key,
+        )
+        patcher = patch("requests.post")
+        self.addCleanup(patcher.stop)
+        self.mock_requests_post = patcher.start()
+        self.mock_requests_post.return_value = SimpleNamespace(
+            ok=True,
+            status_code=200,
+            json=lambda: {},
+            text="",
+        )
+
+    def tearDown(self):
+        cache.clear()
+
+    def _sign(self, payload):
+        body = json.dumps(payload, separators=(",", ":"), sort_keys=True)
+        signature = base64.b64encode(
+            self.private_key.sign(
+                body.encode(), padding.PKCS1v15(), hashes.SHA256()
+            )
+        ).decode()
+        return body, signature
+
+    def test_proxy_session_creates_login_url(self):
+        payload = {
+            "requester": str(self.node.uuid),
+            "user": {
+                "username": "proxy-user",
+                "email": "proxy@example.com",
+                "first_name": "Proxy",
+                "last_name": "User",
+                "is_staff": True,
+                "is_superuser": True,
+                "groups": [],
+                "permissions": [],
+            },
+            "target": "/admin/",
+        }
+        body, signature = self._sign(payload)
+        response = self.client.post(
+            reverse("node-proxy-session"),
+            data=body,
+            content_type="application/json",
+            HTTP_X_SIGNATURE=signature,
+        )
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertIn("login_url", data)
+        user = get_user_model().objects.get(username="proxy-user")
+        self.assertTrue(user.is_staff)
+        parsed = urlparse(data["login_url"])
+        login_response = self.client.get(parsed.path)
+        self.assertEqual(login_response.status_code, 302)
+        self.assertEqual(login_response["Location"], "/admin/")
+        self.assertEqual(self.client.session.get("_auth_user_id"), str(user.pk))
+        second = self.client.get(parsed.path)
+        self.assertEqual(second.status_code, 410)
+
+    def test_proxy_execute_lists_nodes(self):
+        Node.objects.create(
+            hostname="target",
+            address="127.0.0.5",
+            port=8010,
+            mac_address="aa:bb:cc:dd:ee:bb",
+        )
+        payload = {
+            "requester": str(self.node.uuid),
+            "action": "list",
+            "model": "nodes.Node",
+            "filters": {"hostname": "target"},
+            "credentials": {
+                "username": "suite-user",
+                "password": "secret",
+                "first_name": "Suite",
+                "last_name": "User",
+            },
+        }
+        body, signature = self._sign(payload)
+        response = self.client.post(
+            reverse("node-proxy-execute"),
+            data=body,
+            content_type="application/json",
+            HTTP_X_SIGNATURE=signature,
+        )
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(len(data.get("objects", [])), 1)
+        record = data["objects"][0]
+        self.assertEqual(record["fields"]["hostname"], "target")
+        user = get_user_model().objects.get(username="suite-user")
+        self.assertTrue(user.is_superuser)
+
+    def test_proxy_execute_requires_valid_password_for_existing_user(self):
+        User = get_user_model()
+        User.objects.create_user(username="suite-user", password="correct")
+        payload = {
+            "requester": str(self.node.uuid),
+            "action": "list",
+            "model": "nodes.Node",
+            "credentials": {
+                "username": "suite-user",
+                "password": "wrong",
+            },
+        }
+        body, signature = self._sign(payload)
+        response = self.client.post(
+            reverse("node-proxy-execute"),
+            data=body,
+            content_type="application/json",
+            HTTP_X_SIGNATURE=signature,
+        )
+        self.assertEqual(response.status_code, 403)
+
+    def test_proxy_execute_schema_returns_models(self):
+        payload = {
+            "requester": str(self.node.uuid),
+            "action": "schema",
+            "credentials": {
+                "username": "suite-user",
+                "password": "secret",
+            },
+        }
+        body, signature = self._sign(payload)
+        response = self.client.post(
+            reverse("node-proxy-execute"),
+            data=body,
+            content_type="application/json",
+            HTTP_X_SIGNATURE=signature,
+        )
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        models = data.get("models", [])
+        self.assertTrue(models)
+        suite_names = {entry.get("suite_name") for entry in models}
+        self.assertIn("Nodes", suite_names)
+
+
 class NodeRFIDAPITests(TestCase):
     def test_import_endpoint_applies_payload_without_creating_accounts(self):
         remote = Node.objects.create(
@@ -2315,37 +2630,14 @@ class NetMessageAdminTests(TransactionTestCase):
         self.assertEqual(form["subject"].value(), "Re: Ping")
         self.assertEqual(str(form["filter_node"].value()), str(node.pk))
 
-
-class LastNetMessageViewTests(TestCase):
-    def setUp(self):
-        self.client = Client()
-        NodeRole.objects.get_or_create(name="Terminal")
-
-    def test_returns_latest_message(self):
-        NetMessage.objects.create(subject="old", body="msg1")
-        latest = NetMessage.objects.create(subject="new", body="msg2")
-        resp = self.client.get(reverse("last-net-message"))
-        self.assertEqual(resp.status_code, 200)
-        self.assertEqual(
-            resp.json(),
-            {
-                "subject": "new",
-                "body": "msg2",
-                "admin_url": reverse(
-                    "admin:nodes_netmessage_change", args=[latest.pk]
-                ),
-            },
-        )
-
-
 class NetMessageReachTests(TestCase):
     def setUp(self):
         self.roles = {}
-        for name in ["Terminal", "Control", "Satellite", "Constellation"]:
+        for name in ["Terminal", "Control", "Satellite", "Watchtower"]:
             self.roles[name], _ = NodeRole.objects.get_or_create(name=name)
         self.nodes = {}
         for idx, name in enumerate(
-            ["Terminal", "Control", "Satellite", "Constellation"], start=1
+            ["Terminal", "Control", "Satellite", "Watchtower"], start=1
         ):
             self.nodes[name] = Node.objects.create(
                 hostname=name.lower(),
@@ -2389,15 +2681,15 @@ class NetMessageReachTests(TestCase):
         self.assertEqual(mock_post.call_count, 3)
 
     @patch("requests.post")
-    def test_constellation_reach_prioritizes_constellation(self, mock_post):
+    def test_watchtower_reach_prioritizes_watchtower(self, mock_post):
         msg = NetMessage.objects.create(
-            subject="s", body="b", reach=self.roles["Constellation"]
+            subject="s", body="b", reach=self.roles["Watchtower"]
         )
         with patch.object(Node, "get_local", return_value=None):
             msg.propagate()
         roles = set(msg.propagated_to.values_list("role__name", flat=True))
         self.assertEqual(
-            roles, {"Constellation", "Satellite", "Control", "Terminal"}
+            roles, {"Watchtower", "Satellite", "Control", "Terminal"}
         )
         self.assertEqual(mock_post.call_count, 4)
 
@@ -2597,13 +2889,6 @@ class NetMessagePropagationTests(TestCase):
         self.assertNotIn(sender_addr, targets)
         self.assertEqual(msg.propagated_to.count(), 4)
         self.assertTrue(msg.complete)
-        self.assertEqual(len(msg.confirmed_peers), mock_post.call_count)
-        self.assertTrue(
-            all(entry["status"] == "acknowledged" for entry in msg.confirmed_peers.values())
-        )
-        self.assertTrue(
-            all(entry["status_code"] == 200 for entry in msg.confirmed_peers.values())
-        )
 
     @patch("requests.post")
     @patch("core.notifications.notify", return_value=False)
@@ -2689,10 +2974,240 @@ class NetMessagePropagationTests(TestCase):
         ):
             msg.propagate()
 
-        self.assertTrue(msg.confirmed_peers)
-        self.assertTrue(
-            all(entry["status"] == "error" for entry in msg.confirmed_peers.values())
+        self.assertEqual(msg.propagated_to.count(), len(self.remotes))
+        self.assertTrue(msg.complete)
+
+
+class NetMessageQueueTests(TestCase):
+    def setUp(self):
+        self.role, _ = NodeRole.objects.get_or_create(name="Terminal")
+        self.feature, _ = NodeFeature.objects.get_or_create(
+            slug="celery-queue", defaults={"display": "Celery Queue"}
+        )
+
+    def test_propagate_queues_unreachable_downstream(self):
+        local = Node.objects.create(
+            hostname="local",
+            address="10.0.0.1",
+            port=8000,
+            mac_address="00:11:22:33:44:10",
+            role=self.role,
+            public_endpoint="local",
+        )
+        downstream = Node.objects.create(
+            hostname="downstream",
+            address="10.0.0.2",
+            port=8001,
+            mac_address="00:11:22:33:44:11",
+            role=self.role,
+            current_relation=Node.Relation.DOWNSTREAM,
+        )
+        message = NetMessage.objects.create(subject="Queued", body="Body", reach=self.role)
+        with patch.object(Node, "get_local", return_value=local), patch.object(
+            Node, "get_private_key", return_value=None
+        ), patch("core.notifications.notify", return_value=False), patch(
+            "requests.post", side_effect=Exception("fail")
+        ):
+            message.propagate()
+
+        entry = PendingNetMessage.objects.get(node=downstream, message=message)
+        self.assertIn(str(downstream.uuid), entry.seen)
+        self.assertGreater(entry.stale_at, timezone.now())
+
+    def test_queue_limit_enforced(self):
+        downstream = Node.objects.create(
+            hostname="limit",
+            address="10.0.0.3",
+            port=8002,
+            mac_address="00:11:22:33:44:12",
+            role=self.role,
+            current_relation=Node.Relation.DOWNSTREAM,
+            message_queue_length=1,
+        )
+        msg1 = NetMessage.objects.create(subject="Old", body="One", reach=self.role)
+        msg2 = NetMessage.objects.create(subject="New", body="Two", reach=self.role)
+
+        msg1.queue_for_node(downstream, [str(downstream.uuid)])
+        msg2.queue_for_node(downstream, [str(downstream.uuid)])
+
+        entries = list(PendingNetMessage.objects.filter(node=downstream))
+        self.assertEqual(len(entries), 1)
+        self.assertEqual(entries[0].message, msg2)
+
+    def test_queue_duplicate_updates_stale(self):
+        downstream = Node.objects.create(
+            hostname="dup",
+            address="10.0.0.4",
+            port=8003,
+            mac_address="00:11:22:33:44:13",
+            role=self.role,
+            current_relation=Node.Relation.DOWNSTREAM,
         )
+        message = NetMessage.objects.create(subject="Dup", body="Dup", reach=self.role)
+        first = timezone.now()
+        second = first + timedelta(minutes=5)
+        with patch(
+            "nodes.models.timezone.now", side_effect=[first, second, second]
+        ):
+            message.queue_for_node(downstream, ["first"])
+            message.queue_for_node(downstream, ["second"])
+
+        entry = PendingNetMessage.objects.get(node=downstream, message=message)
+        self.assertEqual(entry.seen, ["second"])
+        self.assertEqual(entry.queued_at, second)
+        self.assertEqual(entry.stale_at, second + timedelta(hours=1))
+
+    def test_pull_endpoint_returns_and_clears_messages(self):
+        local_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        downstream_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        local = Node.objects.create(
+            hostname="hub",
+            address="10.0.0.5",
+            port=8004,
+            mac_address="00:11:22:33:44:14",
+            role=self.role,
+            public_endpoint="hub",
+        )
+        downstream = Node.objects.create(
+            hostname="remote",
+            address="10.0.0.6",
+            port=8005,
+            mac_address="00:11:22:33:44:15",
+            role=self.role,
+            current_relation=Node.Relation.DOWNSTREAM,
+            public_key=downstream_key.public_key()
+            .public_bytes(
+                serialization.Encoding.PEM,
+                serialization.PublicFormat.SubjectPublicKeyInfo,
+            )
+            .decode(),
+        )
+        message = NetMessage.objects.create(subject="Fresh", body="Body", reach=self.role)
+        stale_message = NetMessage.objects.create(subject="Stale", body="Body", reach=self.role)
+        now = timezone.now()
+        PendingNetMessage.objects.create(
+            node=downstream,
+            message=message,
+            seen=[str(downstream.uuid)],
+            stale_at=now + timedelta(minutes=30),
+        )
+        stale_entry = PendingNetMessage.objects.create(
+            node=downstream,
+            message=stale_message,
+            seen=["stale"],
+            stale_at=now - timedelta(minutes=5),
+        )
+        PendingNetMessage.objects.filter(pk=stale_entry.pk).update(
+            queued_at=now - timedelta(minutes=5)
+        )
+
+        def fake_get_private(node_obj):
+            if node_obj.pk == local.pk:
+                return local_key
+            return None
+
+        payload = {"requester": str(downstream.uuid)}
+        body = json.dumps(payload, separators=(",", ":"), sort_keys=True)
+        signature = base64.b64encode(
+            downstream_key.sign(
+                body.encode(),
+                padding.PKCS1v15(),
+                hashes.SHA256(),
+            )
+        ).decode()
+
+        with patch.object(Node, "get_local", return_value=local), patch.object(
+            Node, "get_private_key", return_value=local_key
+        ):
+            response = self.client.post(
+                reverse("net-message-pull"),
+                data=body,
+                content_type="application/json",
+                HTTP_X_SIGNATURE=signature,
+            )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.json()
+        self.assertEqual(len(data.get("messages", [])), 1)
+        payload_data = data["messages"][0]["payload"]
+        self.assertEqual(payload_data["uuid"], str(message.uuid))
+        self.assertFalse(
+            PendingNetMessage.objects.filter(node=downstream, message=message).exists()
+        )
+        self.assertFalse(
+            PendingNetMessage.objects.filter(
+                node=downstream, message=stale_message
+            ).exists()
+        )
+        response_signature = data["messages"][0]["signature"]
+        local_public = local_key.public_key()
+        local_public.verify(
+            base64.b64decode(response_signature),
+            json.dumps(payload_data, separators=(",", ":"), sort_keys=True).encode(),
+            padding.PKCS1v15(),
+            hashes.SHA256(),
+        )
+
+    def test_poll_task_fetches_messages(self):
+        local_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        upstream_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        local = Node.objects.create(
+            hostname="downstream",
+            address="10.0.0.7",
+            port=8006,
+            mac_address="00:11:22:33:44:16",
+            role=self.role,
+            public_endpoint="downstream",
+        )
+        upstream = Node.objects.create(
+            hostname="upstream",
+            address="127.0.0.2",
+            port=8010,
+            mac_address="00:11:22:33:44:17",
+            role=self.role,
+            current_relation=Node.Relation.UPSTREAM,
+            public_key=upstream_key.public_key()
+            .public_bytes(
+                serialization.Encoding.PEM,
+                serialization.PublicFormat.SubjectPublicKeyInfo,
+            )
+            .decode(),
+        )
+        NodeFeatureAssignment.objects.create(node=local, feature=self.feature)
+        payload = {
+            "uuid": str(uuid.uuid4()),
+            "subject": "Update",
+            "body": "Body",
+            "seen": [str(local.uuid)],
+            "origin": str(upstream.uuid),
+            "sender": str(upstream.uuid),
+        }
+        payload_text = json.dumps(payload, separators=(",", ":"), sort_keys=True)
+        payload_signature = base64.b64encode(
+            upstream_key.sign(
+                payload_text.encode(),
+                padding.PKCS1v15(),
+                hashes.SHA256(),
+            )
+        ).decode()
+        response = MagicMock()
+        response.ok = True
+        response.json.return_value = {
+            "messages": [{"payload": payload, "signature": payload_signature}]
+        }
+
+        with patch.object(Node, "get_local", return_value=local), patch.object(
+            Node, "get_private_key", return_value=local_key
+        ), patch("nodes.tasks.requests.post", return_value=response) as mock_post, patch.object(
+            NetMessage, "propagate"
+        ) as mock_propagate:
+            poll_unreachable_upstream()
+
+        created = NetMessage.objects.get(uuid=payload["uuid"])
+        self.assertEqual(created.subject, "Update")
+        self.assertEqual(created.node_origin, upstream)
+        mock_post.assert_called_once()
+        mock_propagate.assert_called_once()
 
 
 class NetMessageSignatureTests(TestCase):
@@ -2930,6 +3445,7 @@ class ContentSampleTransactionTests(TestCase):
             sample1.save()
 
 
+@pytest.mark.feature("clipboard-poll")
 class ContentSampleAdminTests(TestCase):
     def setUp(self):
         User = get_user_model()
@@ -3106,6 +3622,7 @@ class EmailOutboxTests(TestCase):
 
 
 class ClipboardTaskTests(TestCase):
+    @pytest.mark.feature("clipboard-poll")
     @patch("nodes.tasks.pyperclip.paste")
     def test_sample_clipboard_task_creates_sample(self, mock_paste):
         mock_paste.return_value = "task text"
@@ -3130,6 +3647,7 @@ class ClipboardTaskTests(TestCase):
             ContentSample.objects.filter(kind=ContentSample.TEXT).count(), 1
         )
 
+    @pytest.mark.feature("screenshot-poll")
     @patch("nodes.tasks.capture_screenshot")
     def test_capture_node_screenshot_task(self, mock_capture):
         node = Node.objects.create(
@@ -3152,6 +3670,7 @@ class ClipboardTaskTests(TestCase):
         self.assertEqual(screenshot.path, "screenshots/test.png")
         self.assertEqual(screenshot.method, "TASK")
 
+    @pytest.mark.feature("screenshot-poll")
     @patch("nodes.tasks.capture_screenshot")
     def test_capture_node_screenshot_handles_error(self, mock_capture):
         Node.objects.create(
@@ -3231,7 +3750,7 @@ class NodeRoleAdminTests(TestCase):
 
 class NodeFeatureFixtureTests(TestCase):
     def test_rfid_scanner_fixture_includes_control_role(self):
-        for name in ("Terminal", "Satellite", "Constellation", "Control"):
+        for name in ("Terminal", "Satellite", "Watchtower", "Control"):
             NodeRole.objects.get_or_create(name=name)
         fixture_path = (
             Path(__file__).resolve().parent
@@ -3243,6 +3762,7 @@ class NodeFeatureFixtureTests(TestCase):
         role_names = set(feature.roles.values_list("name", flat=True))
         self.assertIn("Control", role_names)
 
+    @pytest.mark.feature("ap-router")
     def test_ap_router_fixture_limits_roles(self):
         for name in ("Control", "Satellite"):
             NodeRole.objects.get_or_create(name=name)
@@ -3293,6 +3813,7 @@ class NodeFeatureTests(TestCase):
         self.assertEqual(action.url_name, "admin:nodes_nodefeature_celery_report")
         self.assertEqual(feature.get_default_action(), action)
 
+    @pytest.mark.feature("rpi-camera")
     def test_rpi_camera_feature_has_multiple_actions(self):
         feature = NodeFeature.objects.create(
             slug="rpi-camera", display="Raspberry Pi Camera"
@@ -3303,6 +3824,7 @@ class NodeFeatureTests(TestCase):
         self.assertIn("Take a Snapshot", labels)
         self.assertIn("View stream", labels)
 
+    @pytest.mark.feature("audio-capture")
     def test_audio_capture_feature_has_view_waveform_action(self):
         feature = NodeFeature.objects.create(
             slug="audio-capture", display="Audio Capture"
@@ -3480,6 +4002,7 @@ class NodeFeatureTests(TestCase):
         )
         self.assertEqual(mock_find_command.call_count, 2)
 
+    @pytest.mark.feature("ap-router")
     @patch("nodes.models.Node._hosts_gelectriic_ap", return_value=True)
     def test_ap_router_detection(self, mock_hosts):
         control_role, _ = NodeRole.objects.get_or_create(name="Control")
@@ -3499,6 +4022,7 @@ class NodeFeatureTests(TestCase):
             NodeFeatureAssignment.objects.filter(node=node, feature=feature).exists()
         )
 
+    @pytest.mark.feature("ap-router")
     @patch("nodes.models.Node._hosts_gelectriic_ap", return_value=True)
     def test_ap_router_detection_with_public_mode_lock(self, mock_hosts):
         control_role, _ = NodeRole.objects.get_or_create(name="Control")
@@ -3523,6 +4047,7 @@ class NodeFeatureTests(TestCase):
             NodeFeatureAssignment.objects.filter(node=node, feature=router).exists()
         )
 
+    @pytest.mark.feature("ap-router")
     @patch("nodes.models.Node._hosts_gelectriic_ap", side_effect=[True, False])
     def test_ap_router_removed_when_not_hosting(self, mock_hosts):
         control_role, _ = NodeRole.objects.get_or_create(name="Control")