cctv-stack 0.3.0__py3-none-any.whl

cctv/dispatch.py

@@ -0,0 +1,270 @@
+"""Getting review messages off the network thread and into a worker.
+
+The MQTT callback must not do the work. Materialising a clip waits out the flush grace,
+fetches, probes and writes — tens of seconds — and doing that on the network loop's thread
+stops it servicing the socket, so messages are *missed*. That failure is invisible: the
+broker does not know we were busy, the queue does not exist, and the only symptom is clips
+that never appear.
+
+So the callback parses, hands over, and returns. A bounded queue and a worker thread do the
+rest.
+
+## Dropping under load is correct, and blocking is not
+
+MQTT delivery here is at-most-once (see below), so **the sweep is the guarantee and this is
+the latency optimisation**. Given that, a full queue must drop and count rather than block:
+blocking the callback costs messages *and* stalls every other topic, whereas dropping costs
+only latency — the sweep will find the review on its next pass.
+
+## The pre-filter is queue hygiene, not a second decision
+
+Most traffic on `frigate/reviews` is `update` messages about reviews that are still **open**,
+and an open review can never become a clip. Queueing them would let a chatty camera fill a
+bounded queue and cause *closed* reviews to be dropped — losing exactly the messages that
+mattered, to make room for ones that could not be acted on.
+
+So open reviews are dropped at the door. This is not a duplicate of
+[`reviews.decide`][cctv.reviews.decide] — it is the one condition that has to be applied
+before the queue exists, and `decide` still applies it (and every other filter) in the
+worker, because the sweep path has no door to filter at.
+
+## Why not QoS 1
+
+Subscribing at QoS 1 does not upgrade a publisher sending at QoS 0, and a broker restart
+still loses in-flight messages. The completeness guarantee has to come from somewhere that
+does not depend on delivery at all, which is the sweep. Claiming reliability here would
+only move the lie somewhere harder to find.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import queue
+import threading
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Any
+
+from .reviews import (
+    ReviewCandidate,
+    ReviewParseError,
+    candidate_from_message,
+    parse_review,
+)
+
+__all__ = [
+    "DROPPED",
+    "OPEN",
+    "QUEUED",
+    "UNPARSABLE",
+    "DispatchStats",
+    "ReviewDispatcher",
+]
+
+QUEUED = "queued"
+DROPPED = "dropped"
+UNPARSABLE = "unparsable"
+OPEN = "open"
+
+#: Deep enough to ride out a burst of closed reviews, shallow enough that a stall is
+#: noticed as a count rather than as unbounded memory.
+DEFAULT_MAX_PENDING = 1000
+
+
+@dataclass(slots=True)
+class DispatchStats:
+    """Counters for the health check.
+
+    `dropped` and `unparsable` existing as *numbers* is the point: a system that silently
+    discards a tenth of its input and one that is simply idle look identical from the
+    outside.
+    """
+
+    received: int = 0
+    queued: int = 0
+    handle: int = 0
+    open_reviews: int = 0
+    unparsable: int = 0
+    dropped: int = 0
+    handler_errors: int = 0
+    last_error: str | None = None
+    _reasons: dict[str, int] = field(default_factory=dict, repr=False)
+
+    def note(self, reason: str) -> None:
+        self._reasons[reason] = self._reasons.get(reason, 0) + 1
+
+    @property
+    def reasons(self) -> Mapping[str, int]:
+        return dict(self._reasons)
+
+    @property
+    def lost(self) -> int:
+        """Messages that will only be recovered by the sweep."""
+        return self.dropped + self.unparsable
+
+
+class ReviewDispatcher:
+    """Bounded hand-off between the MQTT callback and the materialiser.
+
+    `handler` receives a closed [`ReviewCandidate`][cctv.reviews.ReviewCandidate] and may
+    take as long as it likes: it runs on the worker, never on the callback.
+    """
+
+    def __init__(
+        self,
+        *,
+        handler: Callable[[ReviewCandidate], Any],
+        max_pending: int = DEFAULT_MAX_PENDING,
+        on_open: Callable[[ReviewCandidate], Any] | None = None,
+        clock: Callable[[], float] | None = None,
+    ) -> None:
+        if max_pending < 1:
+            raise ValueError("max_pending must be at least 1.")
+        self.handler = handler
+        self.max_pending = max_pending
+        self.on_open = on_open
+        self.stats = DispatchStats()
+
+        self._queue: queue.Queue[ReviewCandidate] = queue.Queue(maxsize=max_pending)
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    # -----------------------------------------------------------------------------------
+    # Called from the network thread. Must not block, and must not raise.
+    # -----------------------------------------------------------------------------------
+
+    def dispatch(self, message: Mapping[str, Any]) -> str:
+        """Parse and enqueue one `frigate/reviews` message.
+
+        Returns what happened, so a caller can count it. **Never raises**: an exception
+        escaping an MQTT callback is swallowed by the network loop, so a single bad message
+        would otherwise stop every future message from being seen.
+        """
+        self.stats.received += 1
+
+        candidate = self._parse(message)
+        if candidate is None:
+            # Counted, not just noted. `lost` is derived from this, and a discard that is
+            # only in the reason map would make the health check report nothing discarded.
+            self.stats.unparsable += 1
+            return UNPARSABLE
+
+        if candidate.is_open:
+            # The majority of traffic, and none of it can ever be a clip. See the module
+            # docstring: queueing it would evict the messages that matter.
+            self.stats.open_reviews += 1
+            self.stats.note("review still open")
+            if self.on_open is not None:
+                _swallow(self.on_open, candidate)
+            return OPEN
+
+        try:
+            self._queue.put_nowait(candidate)
+        except queue.Full:
+            self.stats.dropped += 1
+            self.stats.note("queue full")
+            return DROPPED
+
+        self.stats.queued += 1
+        return QUEUED
+
+    def _parse(self, message: Mapping[str, Any]) -> ReviewCandidate | None:
+        """Parse without trusting the shape.
+
+        `after` is read on **every** message type. The restart experiment showed Frigate
+        closes a review without publishing `end` at all, so a trigger keyed on the message
+        type would lose that clip silently.
+        """
+        if not isinstance(message, Mapping):
+            self.stats.note("message is not an object")
+            return None
+        try:
+            if "after" in message:
+                return candidate_from_message(message)
+            # Accept a bare review object too: the sweep reads that shape from the API, and
+            # a test or a replay tool will hand it over in the same form.
+            return parse_review(message)
+        except ReviewParseError as exc:
+            self.stats.note(str(exc)[:120])
+            return None
+
+    # -----------------------------------------------------------------------------------
+    # The worker
+    # -----------------------------------------------------------------------------------
+
+    def work_once(self, *, timeout: float | None = None) -> ReviewCandidate | None:
+        """Take one candidate and handle it. Returns what was handled, if anything.
+
+        Synchronous so tests need neither a broker nor a thread.
+        """
+        try:
+            candidate = self._queue.get(timeout=timeout)
+        except queue.Empty:
+            return None
+
+        try:
+            self.handler(candidate)
+        except Exception as exc:
+            # A handler failure is counted, not fatal. The review is still in Frigate, so
+            # the sweep will offer it again — which is the whole reason this can be
+            # forgiving.
+            self.stats.handler_errors += 1
+            self.stats.last_error = f"{type(exc).__name__}: {exc}"[:200]
+            self.stats.note("handler error")
+        else:
+            self.stats.handle += 1
+        finally:
+            self._queue.task_done()
+
+        return candidate
+
+    def drain(self, *, limit: int | None = None) -> int:
+        """Handle everything currently queued. Returns how many were handled."""
+        handled = 0
+        while limit is None or handled < limit:
+            if self.work_once(timeout=0) is None:
+                break
+            handled += 1
+        return handled
+
+    @property
+    def pending(self) -> int:
+        return self._queue.qsize()
+
+    # -----------------------------------------------------------------------------------
+    # Thread lifecycle
+    # -----------------------------------------------------------------------------------
+
+    def start(self) -> None:
+        if self._thread is not None:
+            raise RuntimeError("Dispatcher already started.")
+        self._stop.clear()
+        self._thread = threading.Thread(target=self._run, name="review-dispatcher", daemon=True)
+        self._thread.start()
+
+    def _run(self) -> None:
+        while not self._stop.is_set():
+            self.work_once(timeout=0.5)
+
+    def stop(self, *, timeout: float = 5.0) -> None:
+        self._stop.set()
+        if self._thread is not None:
+            self._thread.join(timeout=timeout)
+            self._thread = None
+
+    def __enter__(self) -> ReviewDispatcher:
+        self.start()
+        return self
+
+    def __exit__(self, *exc_info: object) -> None:
+        self.stop()
+
+
+def _swallow(callback: Callable[[ReviewCandidate], Any], candidate: ReviewCandidate) -> None:
+    """Run an optional observer, ignoring whatever it does.
+
+    Used for the `on_open` hook, which exists so a caller can log or count open reviews
+    without that being able to break the callback.
+    """
+    with contextlib.suppress(Exception):
+        callback(candidate)

cctv/ffprobe.py

@@ -0,0 +1,246 @@
+"""Reading what `ffprobe` says about a clip.
+
+The output of this module is the evidence for [INV-14](../../../docs/architecture/invariants.md).
+Without it, "are the clips right?" is an opinion; with it, it is a query against the
+`clips` table's quality columns.
+
+## `max_packet_gap_s` is advisory, and nothing may gate on it
+
+The natural way to detect a hole in a clip is the largest gap between packet timestamps.
+On this stack that detector **does not work**. Measured on a healthy 30-second clip cut
+with `-c copy`: the largest gap was **3.55 s**, against a 2.55 s keyframe interval. The
+likely cause is PTS discontinuities at the joins between recording segments, which
+`-c copy` reproduces faithfully because it is not re-encoding anything.
+
+So a threshold that looks reasonable — say 2.0 s — flags nearly every healthy clip. The
+field is still recorded, because the data is real and a future detector may want it, but
+**truncation is detected by coverage, not by gaps.**
+
+See [ADR-0015](../../../docs/adr/0015-clip-boundaries-and-padding.md) for the three
+candidate replacements, the preferred one being to read Frigate's own `recordings` table
+rather than inferring anything from the media.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from itertools import pairwise
+from pathlib import Path
+from typing import Any
+
+from .clip import DEFAULT_COVERAGE_OK
+
+__all__ = [
+    "ClipFacts",
+    "ProbeError",
+    "build_probe_command",
+    "coverage_ratio",
+    "looks_truncated",
+    "max_packet_gap",
+    "parse_probe",
+    "run_probe",
+]
+
+#: The binary, overridable because a container's ffmpeg is rarely on `PATH`. Frigate
+#: itself vendors several builds under `/usr/lib/ffmpeg/<version>/bin/`, and our own image
+#: will have its own arrangement — so the path is configuration, not a constant.
+FFPROBE_ENV = "FFPROBE_BIN"
+
+
+class ProbeError(RuntimeError):
+    """`ffprobe` could not read the file, or is not installed."""
+
+
+@dataclass(frozen=True, slots=True)
+class ClipFacts:
+    """What we learned about a clip by inspecting it rather than trusting the fetch."""
+
+    duration_s: float | None
+    video_codec: str | None
+    audio_codec: str | None
+    has_audio: bool
+    format_name: str | None
+    packet_count: int
+    #: Advisory only — see the module docstring. Never gate on this.
+    max_packet_gap_s: float | None
+    keyframe_interval_s: float | None
+
+    def coverage_against(self, requested_s: float) -> float | None:
+        return coverage_ratio(self.duration_s, requested_s)
+
+
+def build_probe_command(path: str) -> list[str]:
+    """The `ffprobe` invocation, as a list so nothing is ever shell-interpreted.
+
+    `-v error` keeps stderr useful: warnings about non-monotonic DTS would otherwise
+    drown the actual error on a genuinely broken file.
+    """
+    return [
+        "ffprobe",
+        "-v",
+        "error",
+        "-show_format",
+        "-show_streams",
+        "-show_packets",
+        "-of",
+        "json",
+        path,
+    ]
+
+
+def run_probe(
+    path: str | Path,
+    *,
+    ffprobe_bin: str | None = None,
+    timeout_s: float = 60.0,
+) -> ClipFacts:
+    """Run `ffprobe` on a finished file and return the facts.
+
+    Separate from [`parse_probe`][cctv.ffprobe.parse_probe] so the parsing — where the
+    interesting logic is — stays testable without a subprocess, and so every way the
+    subprocess can fail is handled in one place:
+
+    * the binary is missing (`FileNotFoundError`) — which is the normal case in a container
+      unless the image installs it, hence `FFPROBE_BIN`
+    * it hangs
+    * it exits non-zero on a file that is not really a video
+
+    Raises:
+        ProbeError: for all of the above. A clip we cannot measure is a clip we cannot
+            trust, so this is never swallowed.
+    """
+    binary = ffprobe_bin or os.environ.get(FFPROBE_ENV) or "ffprobe"
+    command = build_probe_command(str(path))
+    command[0] = binary
+
+    try:
+        result = subprocess.run(command, capture_output=True, timeout=timeout_s, check=False)
+    except FileNotFoundError as exc:
+        raise ProbeError(
+            f"{binary!r} is not installed or not on PATH. Set {FFPROBE_ENV} to its full "
+            f"path: a container image rarely puts ffmpeg on PATH, and Frigate itself "
+            f"vendors its builds under /usr/lib/ffmpeg/<version>/bin/."
+        ) from exc
+    except subprocess.TimeoutExpired as exc:
+        raise ProbeError(f"{binary} did not finish within {timeout_s}s for {path}.") from exc
+
+    if result.returncode != 0:
+        stderr = result.stderr.decode("utf-8", errors="replace").strip()[:400]
+        raise ProbeError(f"{binary} exited {result.returncode} for {path}: {stderr}")
+
+    try:
+        payload = json.loads(result.stdout.decode("utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ProbeError(f"{binary} produced output that is not JSON for {path}: {exc}") from exc
+
+    return parse_probe(payload)
+
+
+def parse_probe(payload: Mapping[str, Any]) -> ClipFacts:
+    """Turn `ffprobe -of json` output into facts.
+
+    Deliberately tolerant of missing fields. `ffprobe` omits keys rather than emitting
+    nulls, and a clip with no audio stream has no audio fields at all — which must be
+    distinguishable from "we failed to read the file", so a missing duration stays `None`
+    rather than becoming `0.0`.
+    """
+    fmt = payload.get("format") or {}
+    streams = payload.get("streams") or []
+    packets = payload.get("packets") or []
+
+    video = _first_stream(streams, "video")
+    audio = _first_stream(streams, "audio")
+
+    return ClipFacts(
+        duration_s=_as_float(fmt.get("duration")),
+        video_codec=video.get("codec_name") if video else None,
+        audio_codec=audio.get("codec_name") if audio else None,
+        has_audio=audio is not None,
+        format_name=fmt.get("format_name"),
+        packet_count=len(packets),
+        max_packet_gap_s=max_packet_gap(_packet_times(packets)),
+        keyframe_interval_s=_keyframe_interval(streams),
+    )
+
+
+def coverage_ratio(actual_s: float | None, requested_s: float) -> float | None:
+    """`actual / requested`. Above 1.0 is normal, not an error — see `cctv.clip`."""
+    if actual_s is None or requested_s <= 0:
+        return None
+    return actual_s / requested_s
+
+
+def looks_truncated(
+    facts: ClipFacts, requested_s: float, *, ok: float = DEFAULT_COVERAGE_OK
+) -> bool:
+    """Whether the clip is short enough to be worth another attempt.
+
+    Coverage-based by construction. **Do not add a gap check here** — the module docstring
+    explains why the gap detector flags healthy clips on this stack.
+    """
+    ratio = facts.coverage_against(requested_s)
+    if ratio is None:
+        return True  # unmeasurable is not the same as fine
+    return ratio < ok
+
+
+def max_packet_gap(times: Sequence[float]) -> float | None:
+    """Largest gap between consecutive packet timestamps, in seconds.
+
+    Advisory. Returns `None` for fewer than two packets, because a single packet has no
+    gaps and reporting `0.0` would look like a passing check.
+    """
+    if len(times) < 2:
+        return None
+    # `pairwise`, not `zip(times, times[1:])` — the latter needs `strict=False`, and
+    # `strict=True` on it raises because the second sequence is one element shorter. That
+    # off-by-one has been written twice here; `pairwise` cannot express it.
+    return max(b - a for a, b in pairwise(sorted(times)))
+
+
+def _first_stream(streams: Sequence[Mapping[str, Any]], kind: str) -> Mapping[str, Any] | None:
+    for stream in streams:
+        if stream.get("codec_type") == kind:
+            return stream
+    return None
+
+
+def _packet_times(packets: Sequence[Mapping[str, Any]]) -> list[float]:
+    times: list[float] = []
+    for packet in packets:
+        value = _as_float(packet.get("pts_time"))
+        if value is None:
+            value = _as_float(packet.get("dts_time"))
+        if value is not None:
+            times.append(value)
+    return times
+
+
+def _as_float(value: object) -> float | None:
+    if value is None:
+        return None
+    try:
+        return float(value)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        # ffprobe emits "N/A" for an unreadable value.
+        return None
+
+
+def _keyframe_interval(streams: Sequence[Mapping[str, Any]]) -> float | None:
+    """Mean spacing between video frames, used only to sanity-check a gap reading."""
+    video = _first_stream(streams, "video")
+    if not video:
+        return None
+    rate = video.get("avg_frame_rate") or video.get("r_frame_rate")
+    if not isinstance(rate, str) or "/" not in rate:
+        return None
+    numerator, _, denominator = rate.partition("/")
+    try:
+        frames = float(numerator) / float(denominator)
+    except (ValueError, ZeroDivisionError):
+        return None
+    return None if frames <= 0 else 1.0 / frames