python-aidot-cameras 0.15.2__tar.gz → 0.15.4__tar.gz

{python_aidot_cameras-0.15.2/python_aidot_cameras.egg-info → python_aidot_cameras-0.15.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-aidot-cameras
-Version: 0.15.2
+Version: 0.15.4
 Summary: Control AiDot/Leedarson WiFi lights and cameras (WebRTC streaming, two-way audio, PTZ, controls)
 Author-email: cbrightly <chris.brightly@gmail.com>
 License-Expression: MIT

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/aidot_cameras/camera/sdes.py

@@ -38,7 +38,12 @@ class SdesSession:
         teardown_requested=None,
         first_video_pt=None,
         first_audio_pt=None,
+        device_id=None,
     ) -> None:
+        # Which camera this session belongs to, for logging only.  Optional so
+        # an existing caller that does not pass it still works; the logs then
+        # read "?" exactly as they did before.
+        self._device_id  = device_id or "?"
         self._proc       = proc
         self._sdp_path   = sdp_path
         self._outgoing_q = outgoing_q
@@ -177,6 +182,24 @@ class SdesSession:
             "audio_pt": self._first_audio_pt[0],
         }
 
+    def _drained_stderr_tail(self) -> bytes:
+        """The stderr the drain thread already consumed, for the teardown log.
+
+        The serve's stderr is drained continuously on a daemon thread (see
+        ``_start_serve_stderr_drain`` in sdes_open) so an un-drained pipe cannot
+        fill and stall ffmpeg.  A consequence is that by teardown the pipe is at
+        EOF and stop()'s ``stderr.read()`` returns nothing, which would leave
+        ``_log_ffmpeg_stderr`` with an empty buffer and silently drop the
+        teardown diagnostic - the mid-stream ffmpeg error it exists to surface
+        included.  Fall back to the bounded tail the drainer kept.  Returns
+        ``b""`` when there is no drainer or it captured nothing, which
+        ``_log_ffmpeg_stderr`` already treats as "nothing to say".
+        """
+        tail = getattr(self._proc, "_aidot_stderr_tail", None)
+        if not tail:
+            return b""
+        return "\n".join(tail).encode("utf-8", "replace")
+
     def _log_ffmpeg_stderr(self, stderr_bytes: bytes) -> None:
         """Log ffmpeg's captured stderr at a level that matches what happened.
 
@@ -197,7 +220,8 @@ class SdesSession:
             or "Could not find codec parameters" in text
         )
         (_LOGGER.debug if expected_no_media else _LOGGER.warning)(
-            "ffmpeg SDES stderr:\n%s", text
+            "camera %s: ffmpeg SDES stderr:\n%s",
+            getattr(self, "_device_id", "?"), text
         )
 
     @staticmethod
@@ -285,7 +309,7 @@ class SdesSession:
                 self._proc.stderr.close()
             except Exception:
                 _LOGGER.debug("swallowed exception in %s", 'stop', exc_info=True)
-        self._log_ffmpeg_stderr(stderr_bytes)
+        self._log_ffmpeg_stderr(stderr_bytes or self._drained_stderr_tail())
         import os
         try:
             os.unlink(self._sdp_path)

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/aidot_cameras/camera/sdes_open.py

@@ -38,6 +38,38 @@ from .protocol import (
 
 _LOGGER = logging.getLogger(__name__)
 
+
+def _start_serve_stderr_drain(proc, *, maxlines: int = 40) -> None:
+    """Drain a serve ffmpeg's stderr continuously into a bounded tail on the proc.
+
+    The serve is spawned with ``stderr=PIPE`` but the bridge loop only polls the
+    process and reads its RTP sockets - it never reads stderr.  An un-drained
+    ``PIPE`` fills its ~64KB kernel buffer and then blocks ffmpeg on its next
+    stderr write, stalling the serve; and when ffmpeg exits non-zero the reason is
+    otherwise lost (the RTSP-push ANNOUNCE error in particular).  Read it on a
+    daemon thread that ends at EOF when the process exits, keeping the last
+    ``maxlines`` lines on ``proc._aidot_stderr_tail`` for the exit logger.
+    """
+    import collections
+    import threading
+
+    tail = collections.deque(maxlen=maxlines)
+    proc._aidot_stderr_tail = tail
+    if proc.stderr is None:
+        return
+
+    def _drain() -> None:
+        try:
+            for _line in iter(proc.stderr.readline, b""):
+                tail.append(_line.decode("utf-8", "replace").rstrip())
+        except Exception:
+            pass
+
+    threading.Thread(
+        target=_drain, daemon=True, name="aidot-sdes-serve-stderr"
+    ).start()
+
+
 # Strong references to detached per-session helper tasks, so the event loop
 # cannot garbage-collect one mid-flight.  Entries remove themselves on done.
 _SDES_BACKGROUND_TASKS: set = set()
@@ -2793,16 +2825,33 @@ class _SdesOpenMixin:
                                 _br_level = _classify_ffmpeg_exit(
                                     _br_rc, _br_teardown_requested
                                 )
+                                # Name the camera.  These lines used to carry no
+                                # device id at all, and this account runs several
+                                # SDES cameras at once, so a burst of exits could
+                                # not be attributed to one of them without extra
+                                # tooling - which sent a live investigation down
+                                # the wrong path once already.
+                                _br_dev = getattr(self, "device_id", "?")
                                 _br_msg = (
-                                    "SDES bridge: ffmpeg exited with code %d"
-                                    " - stopped by teardown"
+                                    "camera %s: SDES bridge: ffmpeg exited with"
+                                    " code %d - stopped by teardown"
                                     if _br_level < _log_br.WARNING else
-                                    "SDES bridge: ffmpeg exited with code %d"
-                                    " - stream ended"
+                                    "camera %s: SDES bridge: ffmpeg exited with"
+                                    " code %d - stream ended"
                                 )
                                 _log_br.getLogger(__name__).log(
-                                    _br_level, _br_msg, _br_rc
+                                    _br_level, _br_msg, _br_dev, _br_rc
                                 )
+                                if _br_level >= _log_br.WARNING:
+                                    _serr_tail = getattr(
+                                        _br_proc, "_aidot_stderr_tail", None)
+                                    if _serr_tail:
+                                        _log_br.getLogger(__name__).warning(
+                                            "camera %s: SDES serve ffmpeg stderr"
+                                            " (last %d lines):\n%s", _br_dev,
+                                            len(_serr_tail),
+                                            "\n".join(_serr_tail),
+                                        )
                             if _bridge_should_break(
                                 _br_rc, _br_teardown_requested
                             ):
@@ -4462,6 +4511,7 @@ class _SdesOpenMixin:
                 stdout=subprocess.DEVNULL,
                 stderr=subprocess.PIPE,
             )
+            _start_serve_stderr_drain(proc)
             _proc_holder[0] = proc
             _cl(_reap, proc)   # kill ffmpeg if the open is cancelled before hand-off
         except FileNotFoundError:
@@ -4745,6 +4795,7 @@ class _SdesOpenMixin:
                             stdout=subprocess.DEVNULL,
                             stderr=subprocess.PIPE,
                         )
+                        _start_serve_stderr_drain(proc)
                         # Point the shared holder at the live proc immediately.
                         # The bridge thread polls _proc_holder[0]; if it still
                         # sees the terminated old proc it logs "stream ended",
@@ -4866,4 +4917,5 @@ class _SdesOpenMixin:
             teardown_requested=_teardown_holder,
             first_video_pt=_first_video_pt,
             first_audio_pt=_first_audio_pt,
+            device_id=getattr(self, "device_id", None),
         )

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python-aidot-cameras"
-version = "0.15.2"
+version = "0.15.4"
 description = "Control AiDot/Leedarson WiFi lights and cameras (WebRTC streaming, two-way audio, PTZ, controls)"
 readme = "README.md"
 license = "MIT"

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4/python_aidot_cameras.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-aidot-cameras
-Version: 0.15.2
+Version: 0.15.4
 Summary: Control AiDot/Leedarson WiFi lights and cameras (WebRTC streaming, two-way audio, PTZ, controls)
 Author-email: cbrightly <chris.brightly@gmail.com>
 License-Expression: MIT

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/python_aidot_cameras.egg-info/SOURCES.txt

@@ -138,6 +138,7 @@ tests/test_sdes_idle_release.py
 tests/test_sdes_open_cleanup.py
 tests/test_sdes_serve_audio.py
 tests/test_sdes_serve_cmd.py
+tests/test_sdes_serve_stderr_drain.py
 tests/test_sdes_sprop.py
 tests/test_sdes_talk.py
 tests/test_sdes_teardown_exit_log.py

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/tests/test_sdes_ffmpeg_stderr_log.py

@@ -12,8 +12,10 @@ import types
 from aidot_cameras.camera.sdes import SdesSession
 
 
-def _log_fn(last_media):
-    obj = types.SimpleNamespace(last_media_monotonic=last_media)
+def _log_fn(last_media, device_id="dev-abc"):
+    obj = types.SimpleNamespace(
+        last_media_monotonic=last_media, _device_id=device_id
+    )
     return SdesSession._log_ffmpeg_stderr.__get__(obj)
 
 
@@ -31,6 +33,18 @@ def test_no_media_expected_shape_logs_debug(caplog):
         assert recs and recs[0].levelno == logging.DEBUG, msg
 
 
+def test_the_log_names_the_camera(caplog):
+    """Several SDES cameras stream at once; an unattributed line is near-useless.
+
+    A burst of these lines with no device id could not be pinned to a camera
+    during a live investigation, which cost real time and one wrong conclusion.
+    """
+    with caplog.at_level(logging.DEBUG, logger="aidot_cameras.camera.sdes"):
+        _log_fn(1.0, device_id="12b144cb12da")(b"Error writing trailer")
+    recs = _stderr_records(caplog)
+    assert recs and "12b144cb12da" in recs[0].getMessage()
+
+
 def test_no_media_unexpected_error_still_warns(caplog):
     # No media, but NOT the expected empty-output shape -> a real ffmpeg failure.
     with caplog.at_level(logging.DEBUG, logger="aidot_cameras.camera.sdes"):

python_aidot_cameras-0.15.4/tests/test_sdes_serve_stderr_drain.py

@@ -0,0 +1,190 @@
+"""Tests for the SDES serve ffmpeg stderr drain.
+
+The serve is spawned with ``stderr=PIPE`` but the bridge loop never reads it, so an
+un-drained pipe can fill and stall ffmpeg and a non-zero exit loses its reason.
+``_start_serve_stderr_drain`` reads the pipe on a daemon thread and keeps a bounded
+tail on ``proc._aidot_stderr_tail`` for the exit logger.
+"""
+
+import asyncio
+import io
+import logging
+import time
+
+from aidot_cameras.camera.sdes import SdesSession
+from aidot_cameras.camera.sdes_open import _start_serve_stderr_drain
+
+
+class _FakeProc:
+    """Minimal stand-in exposing a readable ``stderr``, like ``subprocess.Popen``."""
+
+    def __init__(self, data: bytes) -> None:
+        self.stderr = io.BytesIO(data)
+
+
+def _tail(proc) -> list:
+    """The captured stderr tail (attribute is set dynamically by the drainer)."""
+    return list(getattr(proc, "_aidot_stderr_tail"))
+
+
+def _wait_until(pred, timeout: float = 2.0) -> None:
+    deadline = time.monotonic() + timeout
+    while not pred():
+        if time.monotonic() > deadline:
+            break
+        time.sleep(0.01)
+
+
+def test_stderr_drain_captures_lines() -> None:
+    proc = _FakeProc(b"line one\nline two\nline three\n")
+    _start_serve_stderr_drain(proc)
+    _wait_until(lambda: _tail(proc)[-1:] == ["line three"])
+    assert _tail(proc) == ["line one", "line two", "line three"]
+
+
+def test_stderr_drain_is_bounded_to_the_tail() -> None:
+    data = b"".join(f"l{i}\n".encode() for i in range(100))
+    proc = _FakeProc(data)
+    _start_serve_stderr_drain(proc, maxlines=10)
+    _wait_until(lambda: _tail(proc)[-1:] == ["l99"])
+    assert _tail(proc) == [f"l{i}" for i in range(90, 100)]
+
+
+def test_stderr_none_is_safe() -> None:
+    class _NoStderr:
+        stderr = None
+
+    proc = _NoStderr()
+    _start_serve_stderr_drain(proc)  # must not raise
+    assert _tail(proc) == []
+
+
+def test_every_piped_spawn_installs_the_drain() -> None:
+    """A serve spawned with an un-drained ``stderr=PIPE`` is the outage itself.
+
+    There are two serve-spawn sites (cold open and key-restart), and the bug
+    shipped because a pipe was opened at one of them with nothing reading it.
+    A third site, or a drain call deleted from an existing one, reintroduces it
+    exactly - and no behavioural test can see the difference until ffmpeg has
+    written ~64KB, which is far past where a unit test looks. Pin the source.
+    """
+    import inspect
+
+    from aidot_cameras.camera import sdes_open
+
+    src = inspect.getsource(sdes_open).splitlines()
+    spawns = [i for i, line in enumerate(src) if "stderr=subprocess.PIPE" in line]
+    assert spawns, (
+        "no piped spawn found in sdes_open - if the serve stopped using a pipe "
+        "this guard is obsolete, but do not delete it without checking why"
+    )
+    for line_no in spawns:
+        window = "\n".join(src[line_no:line_no + 12])
+        assert "_start_serve_stderr_drain(" in window, (
+            f"{sdes_open.__name__}:{line_no + 1} spawns with stderr=PIPE and "
+            "nothing drains it within the next few lines. An un-drained pipe "
+            "fills at ~64KB and blocks ffmpeg on its next write, which kills "
+            "the RTSP publisher and 404s every viewer"
+        )
+
+
+class _TornDownProc:
+    """A serve proc whose stderr pipe the drainer already emptied.
+
+    ``stop()`` reads ``stderr`` after the process is gone; with the drain thread
+    running that read returns nothing, so the teardown diagnostic has to come
+    from the tail the drainer kept.
+    """
+
+    def __init__(self, tail: list) -> None:
+        self._alive = True
+        self.returncode = 0
+        self.stderr = self
+        self._aidot_stderr_tail = tail
+
+    def poll(self):
+        return None if self._alive else self.returncode
+
+    def terminate(self):
+        self._alive = False
+
+    def wait(self, timeout=None):
+        self._alive = False
+        return self.returncode
+
+    def kill(self):
+        self._alive = False
+
+    def read(self):  # stderr.read() - the drainer got there first
+        return b""
+
+
+class _FakeSock:
+    def close(self) -> None:
+        pass
+
+
+class _FakeQ:
+    def put_nowait(self, _x) -> None:
+        pass
+
+
+def _stopped_session(tail: list) -> None:
+    """Run the real ``SdesSession.stop()`` against a drained serve proc."""
+    session = SdesSession(
+        proc=_TornDownProc(tail),
+        sdp_path="/tmp/aidot_test_does_not_exist.sdp",
+        outgoing_q=_FakeQ(),
+        mqtt_fut=None,
+        audio_sock=_FakeSock(),
+        video_sock=_FakeSock(),
+        cmd_chan=[None],
+        talk_state=None,
+    )
+    asyncio.run(session.stop())
+
+
+def test_teardown_logs_the_drained_tail_when_the_pipe_is_empty(caplog) -> None:
+    # Without the fallback this diagnostic is lost entirely: the drain thread
+    # consumed the pipe, so stop()'s stderr.read() has nothing left to log.
+    caplog.set_level(logging.DEBUG, logger="aidot_cameras.camera.sdes")
+    _stopped_session(["frame= 120 fps=20", "Error writing trailer: Broken pipe"])
+    warnings = [
+        r for r in caplog.records
+        if r.levelno == logging.WARNING and "ffmpeg SDES stderr" in r.getMessage()
+    ]
+    assert warnings, "a mid-stream ffmpeg error must still reach the log"
+    assert "Error writing trailer: Broken pipe" in warnings[0].getMessage()
+
+
+def test_teardown_tail_still_demotes_the_expected_no_media_shape(caplog) -> None:
+    # The tail feeds the SAME classifier, so a camera that never sent media does
+    # not start warning once per retry just because the stderr route changed.
+    caplog.set_level(logging.DEBUG, logger="aidot_cameras.camera.sdes")
+    _stopped_session(["Output file is empty, nothing was encoded"])
+    assert not [
+        r for r in caplog.records
+        if r.levelno == logging.WARNING and "ffmpeg SDES stderr" in r.getMessage()
+    ]
+    assert [
+        r for r in caplog.records
+        if r.levelno == logging.DEBUG and "ffmpeg SDES stderr" in r.getMessage()
+    ]
+
+
+def test_teardown_without_a_tail_logs_nothing(caplog) -> None:
+    caplog.set_level(logging.DEBUG, logger="aidot_cameras.camera.sdes")
+    _stopped_session([])
+    assert not [
+        r for r in caplog.records if "ffmpeg SDES stderr" in r.getMessage()
+    ]
+
+
+def test_undecodable_bytes_do_not_crash_the_drain() -> None:
+    proc = _FakeProc(b"good\n\xff\xfe bad bytes\ndone\n")
+    _start_serve_stderr_drain(proc)
+    _wait_until(lambda: _tail(proc)[-1:] == ["done"])
+    tail = _tail(proc)
+    assert tail[0] == "good"
+    assert tail[-1] == "done"
+    assert len(tail) == 3

{python_aidot_cameras-0.15.2 → python_aidot_cameras-0.15.4}/tests/test_sdes_teardown_exit_log.py

@@ -61,6 +61,27 @@ def test_reap_sets_teardown_flag_before_kill():
     )
 
 
+def test_the_bridge_exit_log_names_the_camera():
+    """The exit line is emitted from a closure the tests cannot call directly.
+
+    It is still worth pinning: this account streams several SDES cameras at
+    once, and while these lines carried no device id a burst of exits could not
+    be attributed to a camera at all. Assert the format string takes one, and
+    that the session is handed the id it needs for its own stderr log.
+    """
+    src = inspect.getsource(sdes_open)
+    assert "camera %s: SDES bridge: ffmpeg exited" in src, (
+        "the bridge exit log lost its device id - a burst of these is "
+        "unattributable without it"
+    )
+    assert "camera %s: SDES serve ffmpeg stderr" in src
+    start = src.index("return SdesSession(")
+    assert "device_id=" in src[start:start + 800], (
+        "SdesSession is constructed without a device_id, so its own stderr "
+        "log cannot name the camera"
+    )
+
+
 class _FakeProc:
     def __init__(self):
         self._alive = True