dronecot 2.3.1__tar.gz → 2.3.3__tar.gz

{dronecot-2.3.1 → dronecot-2.3.3}/CHANGELOG.md

@@ -1,3 +1,11 @@
+## DroneCOT 2.3.3
+
+- Fixed the AntSDR scanner-push listener retaining active client tasks while
+  DroneCOT shut down after a TAK transport failure. The listener now owns and
+  closes its server, client tasks and writers with bounded waits, allowing the
+  PyTAK exception to reach systemd and trigger a reconnecting restart.
+- Require PyTAK 7.4.1 for bounded first-exception worker shutdown.
+
 ## DroneCOT 2.3.1
 
 - **Fix: CoT `ce`/`le` reported ODID accuracy ENUM CODES as if they were metres.**

{dronecot-2.3.1/dronecot.egg-info → dronecot-2.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dronecot
-Version: 2.3.1
+Version: 2.3.3
 Summary: Display Drones in Team Awareness Kit (TAK).
 Home-page: https://github.com/snstac/dronecot
 Maintainer: Greg Albrecht <oss@undef.net>
@@ -22,7 +22,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytak>=7.3.12
+Requires-Dist: pytak>=7.4.1
 Requires-Dist: paho-mqtt<2.0.0
 Requires-Dist: bitstruct
 Requires-Dist: pymavlink

{dronecot-2.3.1 → dronecot-2.3.3/dronecot.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dronecot
-Version: 2.3.1
+Version: 2.3.3
 Summary: Display Drones in Team Awareness Kit (TAK).
 Home-page: https://github.com/snstac/dronecot
 Maintainer: Greg Albrecht <oss@undef.net>
@@ -22,7 +22,7 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: pytak>=7.3.12
+Requires-Dist: pytak>=7.4.1
 Requires-Dist: paho-mqtt<2.0.0
 Requires-Dist: bitstruct
 Requires-Dist: pymavlink

{dronecot-2.3.1 → dronecot-2.3.3}/dronecot.egg-info/SOURCES.txt

@@ -31,8 +31,11 @@ src/dronecot/wifi_capture.py
 src/dronecot/wifi_parse.py
 tests/test_adsb_vehicle.py
 tests/test_bluez_capture.py
+tests/test_dji_listener.py
 tests/test_functions.py
+tests/test_mavlink_identity.py
 tests/test_rid_normalize.py
 tests/test_rid_track.py
 tests/test_sensor_detail.py
+tests/test_status.py
 tests/test_wifi_parse.py

{dronecot-2.3.1 → dronecot-2.3.3}/dronecot.egg-info/requires.txt

@@ -1,4 +1,4 @@
-pytak>=7.3.12
+pytak>=7.4.1
 paho-mqtt<2.0.0
 bitstruct
 pymavlink

{dronecot-2.3.1 → dronecot-2.3.3}/setup.cfg

@@ -48,7 +48,7 @@ package_dir =
 	dronecot = src/dronecot
 python_requires = >=3.9, <4
 install_requires = 
-	pytak >= 7.3.12
+	pytak >= 7.4.1
 	paho-mqtt < 2.0.0
 	bitstruct
 	pymavlink

dronecot-2.3.3/src/dronecot/VERSION

@@ -0,0 +1 @@
+2.3.3

{dronecot-2.3.1 → dronecot-2.3.3}/src/dronecot/__init__.py

@@ -18,6 +18,17 @@
 
 """Drone Open Remote ID to TAK Gateway (with DJI Drone ID and UDP Remote ID support)."""
 
+from pathlib import Path
+
+# Read from the same VERSION file setup.cfg packages, so the version an operator
+# sees in the management UI cannot drift from the version of the installed .deb.
+__version__ = (
+    Path(__file__)
+    .resolve()
+    .parent.joinpath("VERSION")
+    .read_text(encoding="utf-8")
+    .strip()
+)
 
 from .constants import (  # NOQA
     DEFAULT_MQTT_TOPIC,

{dronecot-2.3.1 → dronecot-2.3.3}/src/dronecot/classes.py

@@ -27,7 +27,7 @@ import ssl
 from pathlib import Path
 
 from socket import socket
-from typing import Optional, Union
+from typing import Optional, Set, Union
 from urllib.parse import urlparse
 
 import lzma
@@ -42,6 +42,46 @@ import pytak
 import dronecot
 
 
+class _NoStatus:
+    """Stand-in for pytak.StatusWriter on a pytak too old to have one.
+
+    AryaOS boxes are updated as packages, so dronecot can land on a host whose
+    pytak predates StatusWriter (added in 7.4.0) -- 7.3.13 is still in the field.
+    A hard reference would raise AttributeError at import and take the gateway
+    down over its telemetry helper, which is exactly backwards: moving CoT is
+    the job, reporting on it is not.
+
+    Degrading here is safe because it is VISIBLE. With nothing writing
+    /run/dronecot/status.json, the Cockpit plugin reports "No status from this
+    gateway ... may be running a pytak too old to report status" rather than
+    rendering an empty decode feed as though the sky were empty.
+    """
+
+    def count(self, *args, **kwargs) -> None:
+        return None
+
+    def record(self, *args, **kwargs) -> None:
+        return None
+
+    def set(self, *args, **kwargs) -> None:
+        return None
+
+    def write(self, *args, **kwargs) -> bool:
+        return False
+
+
+# Resolved at import so a missing StatusWriter is a startup-time decision
+# rather than an AttributeError on the first Remote ID advertisement.
+_StatusWriter = getattr(pytak, "StatusWriter", None)
+
+
+def make_status(app_name: str, version: str):
+    """Return a status writer, or a no-op if this pytak has none."""
+    if _StatusWriter is None:
+        return _NoStatus()
+    return _StatusWriter(app_name, version=version)
+
+
 class SerialWorker(pytak.QueueWorker):
     """Queue Worker for MAVLink serial Open Drone ID."""
 
@@ -82,13 +122,64 @@ class SerialWorker(pytak.QueueWorker):
 
         return serial_port, baud_rate
 
-    def _mavlink_pack_to_parse_payload_schema(self, messages, pack_size) -> dict:
+    @staticmethod
+    def _id_or_mac_to_mac(id_or_mac) -> Optional[str]:
+        """Render a MAVLink OpenDroneID ``id_or_mac`` field as an advertiser MAC.
+
+        ODID carries a 20-byte ``id_or_mac`` naming the TRANSMITTER, and it is the
+        only per-aircraft grouping key a MAVLink serial feed offers -- unlike a
+        BLE or Wi-Fi capture there is no radio MAC in the metadata.
+
+        Measured on a DroneScout DS110: populated on 58 of 58 packs, as
+        db:13:81:94:13:55 followed by 14 zero bytes.
+
+        Only a MAC-shaped value (6 significant bytes then padding) is returned.
+        The field may instead carry an opaque ID string, and calling that a MAC
+        would put a wrong-looking value in the CoT UID; such feeds keep the
+        previous behaviour until there is real hardware to test against.
+        """
+        if id_or_mac is None:
+            return None
+        raw = bytes(bytearray(id_or_mac))
+        if not any(raw) or len(raw) < 6 or any(raw[6:]):
+            return None
+        return ":".join("%02X" % b for b in raw[:6])
+
+    def _mavlink_meta(self, id_or_mac=None, payload_type: Optional[str] = None) -> dict:
+        """Sensor metadata for a MAVLink-sourced record.
+
+        The wireless path builds this via rid_normalize.bytes_to_rid_dict; the
+        MAVLink path used to skip it entirely and hand up a record with no
+        ``data`` sub-dict at all. Two consequences, both seen on a live box:
+
+          * no transmitter identity, so a pack carrying only System/OperatorID
+            and no BasicID had nothing to aggregate on and rendered as
+            op-Unknown-BasicID_0 -- every operator in range on ONE track;
+          * no sensor_id, so CoT was attributed to the default
+            dronecot_<hostname> rather than the configured SENSOR_ID, making it
+            impossible to tell which receiver saw a contact.
+        """
+        data = dronecot.rid_normalize.uas_meta_defaults(self.config)
+        if payload_type:
+            data["type"] = payload_type
+        mac = self._id_or_mac_to_mac(id_or_mac)
+        if mac:
+            data["MAC address"] = mac
+        return data
+
+    def _mavlink_pack_to_parse_payload_schema(
+        self, messages, pack_size, id_or_mac=None
+    ) -> dict:
         """Convert MAVLink OPEN_DRONE_ID_MESSAGE_PACK to parse_payload-like schema."""
         parsed = dronecot.odid.message_pack_to_dict(messages, pack_size)
-        return dronecot.rid_normalize.odid_parsed_to_rid_dict(parsed)
+        rid = dronecot.rid_normalize.odid_parsed_to_rid_dict(parsed)
+        rid["data"] = self._mavlink_meta(
+            id_or_mac, payload_type="MAVLink OPEN_DRONE_ID_MESSAGE_PACK"
+        )
+        return rid
 
     @staticmethod
-    def _adsb_vehicle_to_rid_dict(msg) -> dict:
+    def _adsb_vehicle_to_rid_dict(msg, meta: Optional[dict] = None) -> dict:
         """Convert a MAVLink ADSB_VEHICLE to a RIDWorker dict.
 
         The DroneScout Bridge (and similar Remote ID receivers) can emit detected
@@ -113,7 +204,7 @@ class SerialWorker(pytak.QueueWorker):
             "SpeedHorizontal": getattr(msg, "hor_velocity", 0) / 100.0,  # cm/s -> m/s
             "SpeedVertical": getattr(msg, "ver_velocity", 0) / 100.0,
             "Direction": getattr(msg, "heading", 0) / 100.0,  # cdeg -> deg
-            "data": {"type": "MAVLink ADSB_VEHICLE"},
+            "data": dict(meta) if meta else {"type": "MAVLink ADSB_VEHICLE"},
         }
 
     async def run(self, _=-1) -> None:
@@ -180,11 +271,19 @@ class SerialWorker(pytak.QueueWorker):
                 msg_type = msg.get_type()
                 if msg_type == "OPEN_DRONE_ID_MESSAGE_PACK":
                     parsed_payload = self._mavlink_pack_to_parse_payload_schema(
-                        msg.messages, msg.msg_pack_size
+                        msg.messages,
+                        msg.msg_pack_size,
+                        getattr(msg, "id_or_mac", None),
                     )
                     await self.put_queue(parsed_payload)
                 elif msg_type == "ADSB_VEHICLE":
-                    rid = self._adsb_vehicle_to_rid_dict(msg)
+                    rid = self._adsb_vehicle_to_rid_dict(
+                        msg,
+                        self._mavlink_meta(
+                            getattr(msg, "id_or_mac", None),
+                            payload_type="MAVLink ADSB_VEHICLE",
+                        ),
+                    )
                     if rid:
                         await self.put_queue(rid)
                 elif msg_type == "HEARTBEAT":
@@ -670,6 +769,19 @@ class RIDWorker(pytak.QueueWorker):
             else None
         )
 
+        # Runtime status for Cockpit. Systemd gives us /run/dronecot via
+        # RuntimeDirectory=, so this lands where the plugin looks for it.
+        #
+        # RIDWorker owns the status file rather than the capture workers because
+        # every RF source (Wi-Fi, BLE, BlueZ, serial MAVLink, MQTT, UDP) funnels
+        # through here. Instrumenting the capture workers instead would give N
+        # writers racing on one path, and each would see only its own slice.
+        self.status = make_status("dronecot", dronecot.__version__)
+
+    def _tracked(self) -> int:
+        """Number of transmitters the aggregator is currently holding."""
+        return len(self.aggregator) if self.aggregator is not None else 0
+
     async def handle_data(self, data: dict) -> None:
         """Handle Data from receiver: Render to CoT, put on TX queue.
 
@@ -680,39 +792,118 @@ class RIDWorker(pytak.QueueWorker):
         """
         self._logger.debug("Handling data: %s", data)
 
+        if not data:
+            # A decoder upstream produced nothing. Deliberately NOT counted as
+            # received: inflating rx with non-messages would make a receiver
+            # that hears only noise look like one hearing drones.
+            return
+
+        self.status.count("rx")
+
         if "status" in data and "position" not in data.get("topic", ""):
             self._logger.info("Processing sensor status data")
             event = dronecot.cot_to_xml(data, self.config, "sensor_status_to_cot")
+            if event:
+                self.status.count("emitted")
+                self.status.count("sensor_status")
+            else:
+                self.status.count("no_cot")
+            self.status.write()
             if event:
                 await self.put_queue(event)
             return
 
         if self.aggregator is not None:
             merged = self.aggregator.update(data)
+            self.status.set(tracked=self._tracked())
             if merged is None:
                 # Absorbed into a track that has no renderable position yet; it
                 # will contribute to the CoT rendered by a later advertisement.
+                # This is the COMMON case on BLE legacy, where a transmitter
+                # rotates BasicID -> Location -> System across advertisements,
+                # so it is counted rather than logged per-message.
                 self._logger.debug("Merged RID message, awaiting position")
+                self.status.count("merged_awaiting_position")
+                self.status.write()
                 return
             data = merged
 
         self._logger.info("Processing RID data")
         cot_funcs = ["rid_uas_to_cot_xml", "rid_op_to_cot_xml"]
+        events = []
         for func in cot_funcs:
             event = dronecot.cot_to_xml(data, self.config, func)
             if event:
-                await self.put_queue(event)
+                events.append(event)
+
+        # Record EVERY message that survives aggregation, not only the ones that
+        # produce a marker. A drone heard without a position still proves the
+        # receiver is working, and a feed that showed only plotted aircraft
+        # would sit empty on a perfectly healthy box -- which reads as a fault.
+        # The `placed` flag distinguishes the two without hiding either.
+        meta = data.get("data") or {}
+        self.status.record(
+            uas=data.get("BasicID") or data.get("BasicID_0"),
+            operator=data.get("OperatorID"),
+            mac=meta.get("MAC address"),
+            rssi=meta.get("RSSI"),
+            # "WiFi beacon", "WiFi NaN", "BLE", "BLE legacy (BlueZ)",
+            # "MAVLink ADSB_VEHICLE", or the configured SENSOR_PAYLOAD_TYPE.
+            source=meta.get("type"),
+            placed=bool(events),
+        )
+
+        if events:
+            self.status.count("emitted", len(events))
+        else:
+            self.status.count("no_cot")
+        self.status.write()
+
+        for event in events:
+            await self.put_queue(event)
 
     async def run(self, _=-1) -> None:
         """Run the main process loop."""
         self._logger.info("Running RIDWorker")
 
-        while True:
-            data = await self.net_queue.get()
-            if not data:
-                continue
+        # Write immediately, before any traffic arrives. A quiet sky is normal
+        # and can last hours; without this the management UI would report "no
+        # status from this gateway" -- indistinguishable from a gateway that
+        # failed to start -- for that entire time.
+        self.status.set(
+            tracked=self._tracked(),
+            # Scheme only. FEED_URL can carry MQTT credentials and status.json
+            # is world-readable on tmpfs.
+            feed=urlparse(str(self.config.get("FEED_URL", ""))).scheme or None,
+        )
+        self.status.write(force=True)
 
-            await self.handle_data(data)
+        # Heartbeat runs as its own task because the loop below blocks on
+        # net_queue.get(). Folding it into that await (via wait_for) would make
+        # every idle timeout look like an exception path; a separate task keeps
+        # the receive path unchanged.
+        heartbeat = asyncio.ensure_future(self._heartbeat())
+        try:
+            while True:
+                data = await self.net_queue.get()
+                if not data:
+                    continue
+
+                await self.handle_data(data)
+        finally:
+            heartbeat.cancel()
+
+    async def _heartbeat(self, interval: float = 5.0) -> None:
+        """Keep the status file fresh while no drones are in range.
+
+        The UI decides liveness from whether this file keeps changing, so an
+        idle-but-healthy gateway MUST keep writing; otherwise an empty sky would
+        be reported as a wedged service.
+        """
+        while True:
+            await asyncio.sleep(interval)
+            self.status.set(tracked=self._tracked())
+            self.status.write(force=True)
 
 
 class RXMockWorker(pytak.RXWorker):
@@ -834,8 +1025,41 @@ class DJIFileWorker(_DJIFeedWorker):
 class DJIListenerWorker(pytak.QueueWorker):
     """Accept incoming TCP connections from DJI RF scanners (server mode)."""
 
-    async def _handle_client(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+    _CLOSE_TIMEOUT = 1.0
+
+    def __init__(self, queue: asyncio.Queue, config) -> None:
+        super().__init__(queue, config)
+        self._server: Optional[asyncio.AbstractServer] = None
+        self._client_tasks: Set[asyncio.Task] = set()
+        self._client_writers: Set[asyncio.StreamWriter] = set()
+        self._close_lock = asyncio.Lock()
+        self._closed = False
+
+    def _client_connected(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
+        """Start and retain ownership of a scanner client task."""
+        task = asyncio.ensure_future(self._handle_client(reader, writer))
+        self._client_tasks.add(task)
+        task.add_done_callback(self._client_tasks.discard)
+
+    async def _close_writer(self, writer: asyncio.StreamWriter) -> None:
+        """Close one scanner connection without allowing shutdown to hang."""
+        writer.close()
+        try:
+            await asyncio.wait_for(
+                writer.wait_closed(), timeout=self._CLOSE_TIMEOUT
+            )
+        except (asyncio.TimeoutError, ConnectionError, OSError):
+            transport = getattr(writer, "transport", None)
+            if transport is not None:
+                transport.abort()
+
+    async def _handle_client(
+        self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+    ) -> None:
         peer = writer.get_extra_info("peername")
+        self._client_writers.add(writer)
         self._logger.info("DJI client connected from %s", peer)
         try:
             while True:
@@ -855,19 +1079,75 @@ class DJIListenerWorker(pytak.QueueWorker):
                     break
                 self.queue.put_nowait(data)
                 await asyncio.sleep(0.001)
+        except asyncio.CancelledError:
+            raise
         except Exception as exc:
             self._logger.error("DJI client error %s: %s", peer, exc)
         finally:
-            writer.close()
-            await writer.wait_closed()
+            self._client_writers.discard(writer)
+            await self._close_writer(writer)
+
+    async def close(self) -> None:
+        """Stop accepting clients and release every owned scanner connection."""
+        async with self._close_lock:
+            if self._closed:
+                return
+            self._closed = True
+
+            if self._server is not None:
+                self._server.close()
+                try:
+                    await asyncio.wait_for(
+                        self._server.wait_closed(), timeout=self._CLOSE_TIMEOUT
+                    )
+                except asyncio.TimeoutError:
+                    self._logger.warning("Timed out closing DJI listener socket")
+                self._server = None
+
+            client_tasks = {
+                task
+                for task in self._client_tasks
+                if task is not asyncio.current_task()
+            }
+            for task in client_tasks:
+                task.cancel()
+
+            writers = list(self._client_writers)
+            if writers:
+                await asyncio.gather(
+                    *(self._close_writer(writer) for writer in writers),
+                    return_exceptions=True,
+                )
+
+            if client_tasks:
+                _, stalled = await asyncio.wait(
+                    client_tasks, timeout=self._CLOSE_TIMEOUT
+                )
+                for task in stalled:
+                    self._logger.warning(
+                        "DJI client task did not stop within %.1fs",
+                        self._CLOSE_TIMEOUT,
+                    )
+                    task.cancel()
 
     async def run(self, _=-1) -> None:
         bind = self.config.get("DJI_BIND_ADDRESS", dronecot.DEFAULT_DJI_BIND_ADDRESS)
         port = self.config.get("DJI_TCP_PORT", dronecot.DEFAULT_DJI_TCP_PORT)
         self._logger.info("Running %s on %s:%s", self.__class__, bind, port)
-        server = await asyncio.start_server(self._handle_client, bind, port)
-        async with server:
-            await server.serve_forever()
+        server = await asyncio.start_server(
+            self._client_connected, bind, port
+        )
+        async with self._close_lock:
+            if self._closed:
+                server.close()
+                await server.wait_closed()
+                raise asyncio.CancelledError
+            self._server = server
+        try:
+            async with self._server:
+                await self._server.serve_forever()
+        finally:
+            await self.close()
 
 
 # Backward-compatible aliases matching djicot names

{dronecot-2.3.1 → dronecot-2.3.3}/src/dronecot/functions.py

@@ -269,7 +269,15 @@ def _rid_identity(data: dict) -> Tuple[str, Optional[str]]:
             # all. Fall back to the feed that delivered it so two receivers do
             # not collide, rather than the shared Unknown-BasicID_0 placeholder
             # that put every such aircraft on one track.
-            feed = src_data.get("sensor_id") or src_data.get("sensor ID")
+            # Checked at BOTH levels, for the same reason the MAC is: MQTT and
+            # serial feeds put this at the top level. Looking only in the
+            # sub-dict meant this rung silently never fired for them.
+            feed = (
+                src_data.get("sensor_id")
+                or src_data.get("sensor ID")
+                or data.get("sensor_id")
+                or data.get("sensor ID")
+            )
             feed = str(feed).strip() if feed else ""
             uasid = f"FEED-{feed}" if feed else "Unknown-BasicID_0"
 

dronecot-2.3.3/tests/test_dji_listener.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# Copyright Sensors & Signals LLC https://www.snstac.com/
+# SPDX-License-Identifier: Apache-2.0
+"""Lifecycle regression tests for the AntSDR scanner-push listener."""
+
+import asyncio
+
+import pytest
+
+import pytak
+from dronecot.classes import DJIListenerWorker
+
+
+async def _wait_for_listener(worker):
+    for _ in range(100):
+        if worker._server is not None:  # pylint: disable=protected-access
+            return worker._server  # pylint: disable=protected-access
+        await asyncio.sleep(0.01)
+    raise AssertionError("DJI listener did not start")
+
+
+@pytest.mark.asyncio
+async def test_listener_close_releases_active_client():
+    """An active AntSDR connection must not keep DroneCOT alive on shutdown."""
+    queue = asyncio.Queue()
+    worker = DJIListenerWorker(
+        queue,
+        {"DJI_BIND_ADDRESS": "127.0.0.1", "DJI_TCP_PORT": 0},
+    )
+    run_task = asyncio.create_task(worker.run())
+    writer = None
+
+    try:
+        server = await _wait_for_listener(worker)
+        port = server.sockets[0].getsockname()[1]
+        reader, writer = await asyncio.open_connection("127.0.0.1", port)
+        writer.write(b"dji_O,test-frame\n")
+        await writer.drain()
+        assert await asyncio.wait_for(queue.get(), timeout=1.0) == b"dji_O,test-frame"
+
+        await asyncio.wait_for(worker.close(), timeout=2.0)
+        result = await asyncio.wait_for(
+            asyncio.gather(run_task, return_exceptions=True), timeout=2.0
+        )
+
+        assert isinstance(result[0], asyncio.CancelledError)
+        assert await asyncio.wait_for(reader.read(), timeout=1.0) == b""
+        assert not worker._client_tasks  # pylint: disable=protected-access
+        assert not worker._client_writers  # pylint: disable=protected-access
+
+        # Cleanup is intentionally idempotent because PyTAK and run() can call
+        # it concurrently during first-exception shutdown.
+        await worker.close()
+    finally:
+        if writer is not None:
+            writer.close()
+        if not run_task.done():
+            run_task.cancel()
+            await asyncio.gather(run_task, return_exceptions=True)
+        await worker.close()
+
+
+@pytest.mark.asyncio
+async def test_transport_failure_exits_with_active_listener_client():
+    """A TAK failure must escape CLITool while an AntSDR remains connected."""
+    fail = asyncio.Event()
+
+    class _FailingTransportWorker:
+        async def run(self):
+            await fail.wait()
+            raise ConnectionAbortedError("WebSocket closed by server")
+
+        async def close(self):
+            return
+
+    config = {
+        "COT_URL": "wss://takserver.example.com:8443/takproto/1",
+        "PYTAK_NO_HELLO": True,
+    }
+    listener = DJIListenerWorker(
+        asyncio.Queue(),
+        {"DJI_BIND_ADDRESS": "127.0.0.1", "DJI_TCP_PORT": 0},
+    )
+    clitool = pytak.CLITool(config)
+    clitool.add_task(listener)
+    clitool.add_task(_FailingTransportWorker())
+    run_task = asyncio.create_task(clitool.run())
+    writer = None
+
+    try:
+        server = await _wait_for_listener(listener)
+        port = server.sockets[0].getsockname()[1]
+        reader, writer = await asyncio.open_connection("127.0.0.1", port)
+
+        fail.set()
+        with pytest.raises(ConnectionAbortedError, match="closed by server"):
+            await asyncio.wait_for(run_task, timeout=2.0)
+
+        assert await asyncio.wait_for(reader.read(), timeout=1.0) == b""
+        assert not listener._client_tasks  # pylint: disable=protected-access
+        assert not listener._client_writers  # pylint: disable=protected-access
+    finally:
+        if writer is not None:
+            writer.close()
+        if not run_task.done():
+            run_task.cancel()
+            await asyncio.gather(run_task, return_exceptions=True)
+        await listener.close()

dronecot-2.3.3/tests/test_mavlink_identity.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+# Copyright Sensors & Signals LLC https://www.snstac.com/
+# SPDX-License-Identifier: Apache-2.0
+"""Transmitter identity on the MAVLink serial feed (DroneScout Bridge).
+
+Regression tests for operator tracks collapsing onto op-Unknown-BasicID_0.
+
+Measured on a live box (aryaos-4f11, DroneScout DS110, dronecot 2.3.1), 30s of
+CoT on the Mesh SA group, attributed by source IP:
+
+    op-Unknown-BasicID_0          x62   <- every operator on ONE track
+    RID.1787F04BM24010011195.uas  x54   correct
+    op-1787F04BM24010011195       x54   correct
+
+The aircraft UID resolved but the operator UID did not. Cause: the MAVLink path
+called rid_normalize.odid_parsed_to_rid_dict directly and attached no ``data``
+metadata at all, unlike the wireless path (bytes_to_rid_dict). With no MAC and no
+sensor_id, a pack carrying only System/OperatorID had no transmitter identity to
+key on, so functions._rid_identity fell all the way to Unknown-BasicID_0.
+
+MAVLink OpenDroneID carries a 20-byte ``id_or_mac`` naming the transmitter. The
+DS110 populates it on every pack -- measured 58 of 58, as
+db:13:81:94:13:55 + 14 zero bytes -- and it is the only per-aircraft grouping key
+a serial feed offers.
+"""
+
+import types
+import unittest
+from unittest import mock
+
+from dronecot.classes import SerialWorker
+from dronecot.functions import _rid_identity
+
+# Exactly what the DS110 sent, including the 14 bytes of padding.
+DS110_ID_OR_MAC = bytes.fromhex("db13819413550000000000000000000000000000")
+DS110_MAC = "DB:13:81:94:13:55"
+
+
+def _worker(sensor_id="dronescout"):
+    """A SerialWorker with just enough state for the metadata helpers."""
+    w = SerialWorker.__new__(SerialWorker)
+    w.config = {"SENSOR_ID": sensor_id}
+    return w
+
+
+class IdOrMacTestCase(unittest.TestCase):
+    def test_real_ds110_value_renders_as_mac(self):
+        self.assertEqual(SerialWorker._id_or_mac_to_mac(DS110_ID_OR_MAC), DS110_MAC)
+
+    def test_all_zeros_is_no_identity(self):
+        self.assertIsNone(SerialWorker._id_or_mac_to_mac(bytes(20)))
+
+    def test_absent_is_no_identity(self):
+        self.assertIsNone(SerialWorker._id_or_mac_to_mac(None))
+
+    def test_opaque_id_is_not_claimed_to_be_a_mac(self):
+        """Payload past byte 6 means this is an ID string, not a MAC.
+
+        Rendering it as one would put a wrong-looking value in the CoT UID, so it
+        is left alone rather than guessed at.
+        """
+        self.assertIsNone(SerialWorker._id_or_mac_to_mac(b"SERIAL12345678901234"))
+
+    def test_accepts_a_bare_six_byte_field(self):
+        self.assertEqual(
+            SerialWorker._id_or_mac_to_mac(bytes.fromhex("db13819413 55".replace(" ", ""))),
+            DS110_MAC,
+        )
+
+
+class MavlinkMetaTestCase(unittest.TestCase):
+    def test_meta_carries_identity_and_configured_sensor_id(self):
+        meta = _worker()._mavlink_meta(DS110_ID_OR_MAC, payload_type="pack")
+        self.assertEqual(meta["MAC address"], DS110_MAC)
+        # Previously absent, so CoT was attributed to dronecot_<hostname> and you
+        # could not tell which receiver saw a contact.
+        self.assertEqual(meta["sensor_id"], "dronescout")
+        self.assertEqual(meta["type"], "pack")
+
+    def test_no_identity_still_yields_sensor_id(self):
+        meta = _worker()._mavlink_meta(bytes(20))
+        self.assertNotIn("MAC address", meta)
+        self.assertEqual(meta["sensor_id"], "dronescout")
+
+
+class PackIdentityTestCase(unittest.TestCase):
+    """The actual regression: a pack with no BasicID must still identify itself."""
+
+    def _pack(self, parsed):
+        with mock.patch("dronecot.odid.message_pack_to_dict", return_value=parsed):
+            return _worker()._mavlink_pack_to_parse_payload_schema(
+                b"\x00" * 25, 1, DS110_ID_OR_MAC
+            )
+
+    def test_pack_without_basicid_gets_a_distinct_uid(self):
+        rid = self._pack({"OperatorLatitude": 37.7, "OperatorLongitude": -122.4})
+        uasid, mac = _rid_identity(rid)
+        self.assertEqual(mac, DS110_MAC)
+        # The bug: this was "Unknown-BasicID_0" for every transmitter in range.
+        self.assertNotEqual(uasid, "Unknown-BasicID_0")
+        self.assertEqual(uasid, "MAC-DB1381941355")
+
+    def test_pack_with_basicid_prefers_the_real_serial(self):
+        rid = self._pack({"UASID": "1787F04BM24010011195"})
+        uasid, mac = _rid_identity(rid)
+        self.assertEqual(uasid, "1787F04BM24010011195")
+        self.assertEqual(mac, DS110_MAC)
+
+    def test_two_transmitters_do_not_collapse(self):
+        """The DS110 reports many aircraft over one port; they must stay distinct."""
+        other = bytes.fromhex("aabbccddeeff" + "00" * 14)
+        with mock.patch("dronecot.odid.message_pack_to_dict", return_value={}):
+            a = _worker()._mavlink_pack_to_parse_payload_schema(
+                b"\x00" * 25, 1, DS110_ID_OR_MAC
+            )
+            b = _worker()._mavlink_pack_to_parse_payload_schema(
+                b"\x00" * 25, 1, other
+            )
+        self.assertNotEqual(_rid_identity(a)[0], _rid_identity(b)[0])
+
+
+class SensorIdFallbackTestCase(unittest.TestCase):
+    """_rid_identity looked for sensor_id only in data['data'].
+
+    The MAC was already checked at both levels because "MQTT and serial feeds put
+    it at the top level" -- the same is true of sensor_id, so that rung of the
+    fallback silently never fired for those feeds.
+    """
+
+    def test_top_level_sensor_id_is_used(self):
+        uasid, mac = _rid_identity({"sensor_id": "dronescout"})
+        self.assertIsNone(mac)
+        self.assertEqual(uasid, "FEED-dronescout")
+
+    def test_nested_sensor_id_still_works(self):
+        uasid, _ = _rid_identity({"data": {"sensor_id": "dronescout"}})
+        self.assertEqual(uasid, "FEED-dronescout")
+
+    def test_nothing_at_all_is_still_the_last_resort(self):
+        uasid, _ = _rid_identity({})
+        self.assertEqual(uasid, "Unknown-BasicID_0")
+
+
+if __name__ == "__main__":
+    unittest.main()

dronecot-2.3.3/tests/test_status.py

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Tests for the runtime status surface RIDWorker writes for Cockpit.
+
+These drive the coroutine with ``asyncio.run()`` rather than declaring bare
+``async def`` tests. That is not a style choice: pytest-asyncio is not installed
+in the build/CI environment for this package, and pytest SKIPS bare async tests
+while still reporting the run as passed -- i.e. tests that cannot fail. Calling
+asyncio.run() explicitly keeps them real regardless of plugins.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import struct
+
+import pytest
+
+import pytak
+
+from dronecot import classes, rid_normalize, rid_track
+
+
+ODID_MESSAGE_SIZE = 25
+PROTO_VERSION = 2
+
+TEST_MAC = "AA:BB:CC:DD:EE:01"
+TEST_SERIAL = "1596F3AD8E2B4C5D6E7F"
+
+needs_statuswriter = pytest.mark.skipif(
+    not hasattr(pytak, "StatusWriter"),
+    reason="installed pytak has no StatusWriter (pre-7.4.0)",
+)
+
+
+def _basic_id_message(serial: str = TEST_SERIAL) -> bytes:
+    """A single ODID BasicID message (type 0): serial, no position."""
+    msg = bytearray(ODID_MESSAGE_SIZE)
+    msg[0] = (0 << 4) | PROTO_VERSION
+    msg[1] = (1 << 4) | 2
+    msg[2:22] = serial.encode("ascii").ljust(20, b"\x00")
+    return bytes(msg)
+
+
+def _location_message(lat: float = 37.76, lon: float = -122.4) -> bytes:
+    """A single ODID Location message (type 1): position, no serial."""
+    msg = bytearray(ODID_MESSAGE_SIZE)
+    msg[0] = (1 << 4) | PROTO_VERSION
+    msg[1] = 0x20  # Status=2 (Airborne)
+    msg[2] = 90
+    msg[3] = 10
+    msg[5:9] = struct.pack("i", int(lat * 1e7))
+    msg[9:13] = struct.pack("i", int(lon * 1e7))
+    msg[13:15] = struct.pack("H", 2000 + 200)
+    msg[15:17] = struct.pack("H", 2000 + 200)
+    msg[17:19] = struct.pack("H", 2000 + 100)
+    return bytes(msg)
+
+
+def _rid(msg: bytes, mac: str = TEST_MAC, rssi: int = -71) -> dict:
+    """Normalize one ODID message the way a BLE capture worker would."""
+    return rid_normalize.bytes_to_rid_dict(
+        msg, {"MAC address": mac, "RSSI": rssi, "type": "BLE legacy (BlueZ)"}
+    )
+
+
+async def _noop_put(event):
+    return None
+
+
+@needs_statuswriter
+class TestStatusSurface:
+    """What the Cockpit plugin reads out of /run/dronecot/status.json.
+
+    RIDWorker already made these decisions -- merge, render, drop -- they just
+    went nowhere an operator could see. The risk with a status surface is that
+    it reports the shape of the code rather than what actually happened, so
+    these assert against real ODID advertisements pushed through the worker.
+    """
+
+    def _worker(self, tmp_path, ttl=120.0, id_grace=5.0):
+        worker = classes.RIDWorker.__new__(classes.RIDWorker)
+        worker.config = {}
+        worker.net_queue = None
+        worker.aggregator = rid_track.ODIDAggregator(ttl=ttl, id_grace=id_grace)
+        worker._logger = logging.getLogger("test")
+        worker.status = pytak.StatusWriter(
+            "dronecot-test", path=str(tmp_path / "status.json")
+        )
+        worker.put_queue = _noop_put
+        return worker
+
+    def _handle(self, worker, data):
+        asyncio.run(worker.handle_data(data))
+
+    def _doc(self, worker):
+        with open(worker.status.path) as handle:
+            return json.load(handle)
+
+    def test_id_only_fragment_is_counted_not_silently_dropped(self, tmp_path):
+        """A BasicID advertisement carries no position, so it renders no CoT.
+
+        On BLE legacy this is the majority of traffic. Before this surface it
+        vanished at debug level, so a receiver hearing a drone loud and clear
+        looked identical to one hearing nothing at all.
+        """
+        worker = self._worker(tmp_path)
+        self._handle(worker, _rid(_basic_id_message()))
+
+        doc = self._doc(worker)
+        assert doc["counters"]["rx"] == 1
+        assert doc["counters"]["merged_awaiting_position"] == 1
+        assert "emitted" not in doc["counters"]
+        # Nothing renderable yet, so nothing claims to be a contact.
+        assert doc["recent"] == []
+        assert doc["tracked"] == 1
+
+    def test_merged_track_emits_and_appears_in_the_feed(self, tmp_path):
+        """Serial from one advertisement, position from the next -> one contact."""
+        worker = self._worker(tmp_path)
+        self._handle(worker, _rid(_basic_id_message()))
+        self._handle(worker, _rid(_location_message()))
+
+        # Writes are rate-limited to 1/sec, so two messages in the same second
+        # leave the file holding the first one's figures. That is by design --
+        # a gateway taking hundreds of advertisements a second must not spend
+        # its time serialising JSON -- and run()'s 5s heartbeat is what
+        # reconciles it. Forcing here stands in for that heartbeat.
+        worker.status.write(force=True)
+
+        doc = self._doc(worker)
+        assert doc["counters"]["rx"] == 2
+        assert doc["counters"]["emitted"] >= 1
+        entry = doc["recent"][-1]
+        assert entry["uas"] == TEST_SERIAL
+        assert entry["placed"] is True
+        assert entry["rssi"] == -71
+        assert entry["source"] == "BLE legacy (BlueZ)"
+        assert entry["mac"] == TEST_MAC
+
+    def test_two_transmitters_are_tracked_separately(self, tmp_path):
+        """tracked is the aggregator's real size, not a message count."""
+        worker = self._worker(tmp_path)
+        self._handle(worker, _rid(_basic_id_message(), mac="AA:BB:CC:DD:EE:01"))
+        self._handle(worker, _rid(_basic_id_message("OTHER"), mac="AA:BB:CC:DD:EE:02"))
+        worker.status.write(force=True)
+
+        assert self._doc(worker)["tracked"] == 2
+
+    def test_empty_payload_is_not_counted_as_received(self, tmp_path):
+        """A decoder that produced nothing is not a drone.
+
+        Counting it would make a receiver hearing only noise indistinguishable
+        from one hearing traffic, which is the exact confusion this file exists
+        to remove.
+        """
+        worker = self._worker(tmp_path)
+        self._handle(worker, {})
+        assert not os.path.exists(worker.status.path)
+
+    def test_unrenderable_record_is_counted_as_no_cot(self, tmp_path):
+        """Aggregation off: a position-less record reaches the renderer and fails.
+
+        With RID_TRACK_TTL=0 there is no aggregator, so the fragment goes
+        straight to cot_to_xml, which cannot place it. That is a distinct
+        outcome from "held for merging" and gets a distinct counter.
+        """
+        worker = self._worker(tmp_path)
+        worker.aggregator = None
+        self._handle(worker, _rid(_basic_id_message()))
+
+        doc = self._doc(worker)
+        assert doc["counters"]["rx"] == 1
+        assert doc["counters"]["no_cot"] == 1
+        assert "merged_awaiting_position" not in doc["counters"]
+        # Still shown in the feed: it WAS decoded, it just could not be placed.
+        assert doc["recent"][-1]["placed"] is False
+        assert doc["recent"][-1]["uas"] == TEST_SERIAL
+
+
+@needs_statuswriter
+class TestStatusHeartbeat:
+    """An idle-but-healthy gateway must keep writing."""
+
+    def test_heartbeat_refreshes_the_file(self, tmp_path):
+        worker = classes.RIDWorker.__new__(classes.RIDWorker)
+        worker.config = {}
+        worker.aggregator = rid_track.ODIDAggregator()
+        worker._logger = logging.getLogger("test")
+        worker.status = pytak.StatusWriter(
+            "dronecot-test", path=str(tmp_path / "status.json")
+        )
+
+        async def _drive():
+            task = asyncio.ensure_future(worker._heartbeat(interval=0.01))
+            await asyncio.sleep(0.05)
+            task.cancel()
+
+        asyncio.run(_drive())
+
+        with open(worker.status.path) as handle:
+            doc = json.load(handle)
+        # No traffic at all, yet the file exists and reports a wall clock the
+        # UI can use to tell "quiet" from "stopped".
+        assert doc["counters"] == {}
+        assert doc["wall_t"] > 0
+        assert doc["tracked"] == 0
+
+
+class TestStatusDegradesVisibly:
+    """A pytak without StatusWriter must not take the gateway down.
+
+    Fleet boxes run pytak 7.3.13, which has no StatusWriter at all.
+    """
+
+    def test_no_op_status_when_pytak_is_too_old(self, monkeypatch):
+        monkeypatch.setattr(classes, "_StatusWriter", None)
+        status = classes.make_status("dronecot", "0.1.0")
+
+        # Every call RIDWorker makes must be safe on the stand-in.
+        status.count("rx")
+        status.count("emitted", 2)
+        status.record(uas="X", placed=True)
+        status.set(tracked=1)
+        assert status.write() is False
+        assert status.write(force=True) is False
+
+    def test_worker_still_handles_data_with_no_statuswriter(self, monkeypatch):
+        """The whole point: no StatusWriter must not break the data path.
+
+        The renderer is stubbed so this test asserts what it claims to --
+        that RIDWorker still aggregates and still enqueues -- rather than
+        incidentally re-testing dronecot's CoT rendering against whatever
+        pytak happens to be installed.
+        """
+        monkeypatch.setattr(classes, "_StatusWriter", None)
+        monkeypatch.setattr(
+            classes.dronecot, "cot_to_xml", lambda data, config, func: b"<event/>"
+        )
+
+        worker = classes.RIDWorker.__new__(classes.RIDWorker)
+        worker.config = {}
+        worker.aggregator = rid_track.ODIDAggregator()
+        worker._logger = logging.getLogger("test")
+        worker.status = classes.make_status("dronecot", "0.1.0")
+
+        emitted = []
+
+        async def _put(event):
+            emitted.append(event)
+
+        worker.put_queue = _put
+
+        asyncio.run(worker.handle_data(_rid(_basic_id_message())))
+        # Held for merging: no position yet, so nothing to send.
+        assert emitted == []
+
+        asyncio.run(worker.handle_data(_rid(_location_message())))
+
+        assert isinstance(worker.status, classes._NoStatus)
+        assert emitted, "CoT must still be produced when status is unavailable"
+
+    def test_real_writer_used_when_available(self):
+        if classes._StatusWriter is None:
+            pytest.skip("installed pytak has no StatusWriter")
+        assert not isinstance(classes.make_status("x", "0"), classes._NoStatus)

dronecot-2.3.1/src/dronecot/VERSION

@@ -1 +0,0 @@
-2.3.1