dronecot 2.3.7__tar.gz → 2.3.9__tar.gz

{dronecot-2.3.7 → dronecot-2.3.9}/CHANGELOG.md

@@ -1,3 +1,10 @@
+## DroneCOT 2.3.8
+
+- Add opt-in `SERIAL_CRLF_NORMALIZE` support for receivers that send binary
+  MAVLink through an ESP console stream configured to expand LF bytes to CRLF.
+  The streaming normalizer reverses expansion across serial read boundaries so
+  pymavlink can validate and decode frames that were previously discarded.
+
 ## DroneCOT 2.3.7
 
 - Reload systemd's unit metadata during Debian package configuration so an

{dronecot-2.3.7/dronecot.egg-info → dronecot-2.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dronecot
-Version: 2.3.7
+Version: 2.3.9
 Summary: Display Drones in Team Awareness Kit (TAK).
 Home-page: https://github.com/snstac/dronecot
 Maintainer: Greg Albrecht <oss@undef.net>

{dronecot-2.3.7 → dronecot-2.3.9/dronecot.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dronecot
-Version: 2.3.7
+Version: 2.3.9
 Summary: Display Drones in Team Awareness Kit (TAK).
 Home-page: https://github.com/snstac/dronecot
 Maintainer: Greg Albrecht <oss@undef.net>

{dronecot-2.3.7 → dronecot-2.3.9}/dronecot.egg-info/SOURCES.txt

@@ -37,5 +37,6 @@ tests/test_mavlink_identity.py
 tests/test_rid_normalize.py
 tests/test_rid_track.py
 tests/test_sensor_detail.py
+tests/test_serial_crlf.py
 tests/test_status.py
 tests/test_wifi_parse.py

dronecot-2.3.9/src/dronecot/VERSION

@@ -0,0 +1 @@
+2.3.9

{dronecot-2.3.7 → dronecot-2.3.9}/src/dronecot/classes.py

@@ -75,11 +75,61 @@ class _NoStatus:
 _StatusWriter = getattr(pytak, "StatusWriter", None)
 
 
-def make_status(app_name: str, version: str):
+class _CRLFNormalizer:
+    """Reverse an LF-to-CRLF transformation on a binary byte stream.
+
+    Some ESP32 USB serial firmware sends MAVLink through a console-oriented
+    VFS stream configured to turn every LF byte into CRLF.  That conversion
+    also touches LF bytes inside MAVLink headers, payloads and checksums, so
+    pymavlink rejects the affected frames.  Keep a trailing CR between reads
+    so a CRLF pair split across pyserial chunks is still reversed exactly.
+
+    This is deliberately opt-in.  Applying it to a compliant binary stream
+    would collapse a legitimate CRLF byte sequence and corrupt that frame.
+    """
+
+    def __init__(self):
+        self._pending_cr = False
+
+    def feed(self, data: bytes) -> bytes:
+        """Return *data* with transmit-side CRLF expansion reversed."""
+        if not data:
+            return b""
+
+        if self._pending_cr:
+            data = b"\r" + data
+            self._pending_cr = False
+
+        if data.endswith(b"\r"):
+            data = data[:-1]
+            self._pending_cr = True
+
+        return data.replace(b"\r\n", b"\n")
+
+
+class _CRLFNormalizingReceiver:
+    """Callable wrapper around a pymavlink connection's ``recv`` method."""
+
+    def __init__(self, receiver):
+        self._receiver = receiver
+        self._normalizer = _CRLFNormalizer()
+
+    def __call__(self, size=None) -> bytes:
+        return self._normalizer.feed(self._receiver(size))
+
+
+def make_status(app_name: str, version: str, path: Optional[str] = None):
     """Return a status writer, or a no-op if this pytak has none."""
     if _StatusWriter is None:
         return _NoStatus()
-    return _StatusWriter(app_name, version=version)
+    return _StatusWriter(app_name, path=path, version=version)
+
+
+def make_worker_status(config):
+    """Build the status writer for one independently managed instance."""
+    app_name = str(config.get("STATUS_APP", "dronecot") or "dronecot")
+    path = str(config.get("STATUS_PATH", "") or "") or None
+    return make_status(app_name, dronecot.__version__, path=path)
 
 
 class SerialWorker(pytak.QueueWorker):
@@ -122,6 +172,18 @@ class SerialWorker(pytak.QueueWorker):
 
         return serial_port, baud_rate
 
+    def _serial_crlf_normalization_enabled(self) -> bool:
+        """Return whether broken console-style CRLF expansion is expected."""
+        value = self.config.get("SERIAL_CRLF_NORMALIZE", "0")
+        return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+    def _configure_serial_connection(self, master) -> None:
+        """Install optional byte-stream compatibility filters on *master*."""
+        if not self._serial_crlf_normalization_enabled():
+            return
+        master.recv = _CRLFNormalizingReceiver(master.recv)
+        self._logger.info("Normalizing CRLF-expanded MAVLink serial input")
+
     @staticmethod
     def _id_or_mac_to_mac(id_or_mac) -> Optional[str]:
         """Render a MAVLink OpenDroneID ``id_or_mac`` field as an advertiser MAC.
@@ -228,6 +290,7 @@ class SerialWorker(pytak.QueueWorker):
                 master = await asyncio.to_thread(
                     mavutil.mavlink_connection, interface, baudrate
                 )
+                self._configure_serial_connection(master)
                 await asyncio.to_thread(master.wait_heartbeat, timeout=20)
                 self._logger.info("MAVLink heartbeat received")
             except SerialException as exc:
@@ -776,7 +839,7 @@ class RIDWorker(pytak.QueueWorker):
         # every RF source (Wi-Fi, BLE, BlueZ, serial MAVLink, MQTT, UDP) funnels
         # through here. Instrumenting the capture workers instead would give N
         # writers racing on one path, and each would see only its own slice.
-        self.status = make_status("dronecot", dronecot.__version__)
+        self.status = make_worker_status(config)
 
     def _tracked(self) -> int:
         """Number of transmitters the aggregator is currently holding."""
@@ -934,10 +997,12 @@ class RXMockWorker(pytak.RXWorker):
 class DJIWorker(pytak.QueueWorker):
     """Process DJI Drone ID data from a net queue and emit CoT events."""
 
-    def __init__(self, tx_queue: asyncio.Queue, config, net_queue: asyncio.Queue) -> None:
+    def __init__(
+        self, tx_queue: asyncio.Queue, config, net_queue: asyncio.Queue
+    ) -> None:
         super().__init__(tx_queue, config)
         self.net_queue = net_queue
-        self.status = make_status("dronecot", dronecot.__version__)
+        self.status = make_worker_status(config)
 
     async def handle_data(self, data) -> None:
         if not data:

dronecot-2.3.9/tests/test_serial_crlf.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Regression tests for CRLF-expanded binary MAVLink serial receivers."""
+
+import unittest
+
+from pymavlink.dialects.v20 import common as mavlink
+
+from dronecot.classes import (
+    SerialWorker,
+    _CRLFNormalizer,
+    _CRLFNormalizingReceiver,
+)
+
+
+class CRLFNormalizerTest(unittest.TestCase):
+    """Exercise the streaming inverse of ESP console newline expansion."""
+
+    def test_reverses_expansion_across_every_chunk_boundary(self):
+        original = b"header\nbody\r\ntail\x00\nend"
+        expanded = original.replace(b"\n", b"\r\n")
+
+        for split in range(len(expanded) + 1):
+            normalizer = _CRLFNormalizer()
+            actual = normalizer.feed(expanded[:split])
+            actual += normalizer.feed(expanded[split:])
+            self.assertEqual(actual, original, f"split={split}")
+
+    def test_preserves_a_cr_not_followed_by_lf(self):
+        normalizer = _CRLFNormalizer()
+        self.assertEqual(normalizer.feed(b"one\r"), b"one")
+        self.assertEqual(normalizer.feed(b"two"), b"\rtwo")
+
+    def test_receiver_wrapper_preserves_state_between_reads(self):
+        chunks = iter((b"abc\r", b"\ndef", b""))
+        receiver = _CRLFNormalizingReceiver(lambda _size: next(chunks))
+        self.assertEqual(receiver(4), b"abc")
+        self.assertEqual(receiver(4), b"\ndef")
+        self.assertEqual(receiver(4), b"")
+
+    def test_recovers_checksum_valid_mavlink_frame(self):
+        encoder = mavlink.MAVLink(None, srcSystem=254, srcComponent=236)
+        # custom_mode=10 guarantees an LF byte inside the binary payload.
+        packet = encoder.heartbeat_encode(34, 8, 0, 10, 4, 3).pack(encoder)
+        self.assertIn(b"\n", packet)
+        expanded = (packet + b"\n").replace(b"\n", b"\r\n")
+
+        normalizer = _CRLFNormalizer()
+        recovered = b"".join(normalizer.feed(bytes((byte,))) for byte in expanded)
+        self.assertEqual(recovered, packet + b"\n")
+
+        parser = mavlink.MAVLink(None)
+        parser.robust_parsing = True
+        messages = []
+        for byte in recovered:
+            message = parser.parse_char(bytes((byte,)))
+            if message is not None and message.get_type() != "BAD_DATA":
+                messages.append(message)
+
+        self.assertEqual([message.get_type() for message in messages], ["HEARTBEAT"])
+        self.assertEqual(messages[0].custom_mode, 10)
+
+
+class SerialWorkerCRLFConfigTest(unittest.TestCase):
+    """Keep the compatibility filter opt-in for standards-compliant feeds."""
+
+    @staticmethod
+    def _worker(value=None):
+        worker = SerialWorker.__new__(SerialWorker)
+        worker.config = {}
+        if value is not None:
+            worker.config["SERIAL_CRLF_NORMALIZE"] = value
+        return worker
+
+    def test_disabled_by_default(self):
+        self.assertFalse(self._worker()._serial_crlf_normalization_enabled())
+
+    def test_accepts_common_true_values(self):
+        for value in ("1", "true", "TRUE", "yes", "on", 1, True):
+            with self.subTest(value=value):
+                self.assertTrue(
+                    self._worker(value)._serial_crlf_normalization_enabled()
+                )
+
+    def test_rejects_false_values(self):
+        for value in ("0", "false", "no", "off", "", 0, False):
+            with self.subTest(value=value):
+                self.assertFalse(
+                    self._worker(value)._serial_crlf_normalization_enabled()
+                )
+
+
+if __name__ == "__main__":
+    unittest.main()

{dronecot-2.3.7 → dronecot-2.3.9}/tests/test_status.py

@@ -330,3 +330,22 @@ class TestStatusDegradesVisibly:
         if classes._StatusWriter is None:
             pytest.skip("installed pytak has no StatusWriter")
         assert not isinstance(classes.make_status("x", "0"), classes._NoStatus)
+
+    def test_instance_status_uses_configured_app_and_path(self, monkeypatch, tmp_path):
+        captured = {}
+
+        def fake_writer(app_name, *, path=None, version=None):
+            captured.update(app_name=app_name, path=path, version=version)
+            return object()
+
+        monkeypatch.setattr(classes, "_StatusWriter", fake_writer)
+        path = str(tmp_path / "status.json")
+        classes.make_worker_status(
+            {"STATUS_APP": "dronecot-dronescout", "STATUS_PATH": path}
+        )
+
+        assert captured == {
+            "app_name": "dronecot-dronescout",
+            "path": path,
+            "version": classes.dronecot.__version__,
+        }

dronecot-2.3.7/src/dronecot/VERSION

@@ -1 +0,0 @@
-2.3.7