cctv-stack 0.3.0__py3-none-any.whl

cctv/sweep.py

@@ -0,0 +1,254 @@
+"""The reconciliation sweep: find reviews we have not turned into clips, and turn them.
+
+MQTT is the fast path and it is lossy — at-most-once from our side, and worse than that:
+Frigate closes a review at start-up **without publishing `end` at all**. Measured, with
+the subscriber on a broker that never restarted. A deployment trusting `end` loses that
+clip silently. So the sweep is not a backstop for a rare failure; it is the mechanism
+that makes materialisation complete, and MQTT is only the latency optimisation.
+
+## `after` filters on `end_time`, not `start_time` — and the watermark depends on it
+
+Frigate's `/api/review` applies the window as:
+
+```python
+clauses = [(start_time < before) & ((end_time.is_null()) | (end_time > after))]
+```
+
+So a review that *started* before the watermark but *ended* after it is still returned.
+That is why the watermark is the highest `end_time` fully processed rather than a
+`start_time`: advancing past a start time would skip any review that was still open when
+we last looked, which is exactly when a restart happens.
+
+## Progressing without losing work
+
+If a review defers — the flush race, a transient Frigate failure — the watermark must not
+move past it, or the next sweep never sees it again and the clip is lost by *retrying*.
+But it also cannot stay put, or one persistently failing review blocks the sweep forever.
+
+So the watermark advances to the highest `end_time` terminally decided, and then retreats
+to just before the earliest *deferred* review. Deferrals are therefore revisited, and
+everything else progresses.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from typing import Protocol
+
+from . import db
+from .config import CameraConfig, CamerasConfig
+from .frigate import FrigateError
+from .materialise import CREATED, DEFERRED, FAILED, SKIPPED, MaterialiseOutcome
+from .reviews import ReviewCandidate, candidates_from_reviews, decide
+from .timeutil import Instant
+from .timeutil import now as _now
+
+__all__ = [
+    "DEFAULT_LOOKBACK_S",
+    "WATERMARK_EPSILON",
+    "WATERMARK_NAME",
+    "ReviewLister",
+    "ReviewSweep",
+    "SweepResult",
+]
+
+#: Where the watermark lives in `sweep_state`. Named for reviews rather than events,
+#: since ADR-0016.
+WATERMARK_NAME = "reviews_sweep"
+
+#: On a first run there is no watermark, and Frigate may hold months of history whose
+#: footage has long expired. Sweeping all of it would produce a burst of doomed fetches, so
+#: a fresh install starts from a bounded lookback instead.
+DEFAULT_LOOKBACK_S = 3600.0
+
+#: The `after` filter is strictly `end_time > after`, so a deferred review is only
+#: re-fetched if the watermark sits *just* before it.
+WATERMARK_EPSILON = 0.001
+
+
+class ReviewLister(Protocol):
+    def list_reviews(
+        self, *, after: Instant | None = None, before: Instant | None = None, limit: int = ...
+    ) -> list[dict]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class SweepResult:
+    """What one sweep did. Counts rather than a boolean, because "how far behind am I" is
+    the question the health check and the dashboard both need answered.
+    """
+
+    considered: int = 0
+    materialised: int = 0
+    skipped: int = 0
+    deferred: int = 0
+    failed: int = 0
+    unknown_camera: int = 0
+    watermark: float | None = None
+    #: Set only when the sweep itself could not run — Frigate unreachable, for instance.
+    #: Not for per-review failures, which are counted instead.
+    failure: str | None = None
+    outcomes: tuple[MaterialiseOutcome, ...] = field(default=(), repr=False)
+
+    @property
+    def ok(self) -> bool:
+        return self.failure is None
+
+    @property
+    def needs_another_run(self) -> bool:
+        """True when something is waiting to be retried."""
+        return self.deferred > 0
+
+
+@dataclass(frozen=True, slots=True)
+class _Decision:
+    candidate: ReviewCandidate
+    camera: CameraConfig | None
+    outcome: MaterialiseOutcome | None = None
+    skip_reason: str | None = None
+
+
+class ReviewSweep:
+    """Walks the reviews Frigate holds and materialises the ones we are missing."""
+
+    def __init__(
+        self,
+        *,
+        conn: sqlite3.Connection,
+        client: ReviewLister,
+        materialiser,
+        clock: Callable[[], Instant] = _now,
+        lookback_s: float = DEFAULT_LOOKBACK_S,
+        limit: int = 200,
+        watermark_name: str = WATERMARK_NAME,
+    ) -> None:
+        self.conn = conn
+        self.client = client
+        self.materialiser = materialiser
+        self._clock = clock
+        self.lookback_s = lookback_s
+        self.limit = limit
+        self.watermark_name = watermark_name
+
+    # -----------------------------------------------------------------------------------
+
+    def run(self, *, config: CamerasConfig) -> SweepResult:
+        """One pass. Intended to be called on a timer, not in a tight loop."""
+        now = self._clock()
+        after = self._resume_point(now)
+
+        try:
+            raw = self.client.list_reviews(after=after, before=now + 1, limit=self.limit)
+        except FrigateError as exc:
+            # Frigate unreachable. The watermark does not move and the failure count rises,
+            # so health can tell "idle" apart from "failing for an hour".
+            db.set_watermark(
+                self.conn,
+                name=self.watermark_name,
+                watermark=after,
+                succeeded=False,
+                note=str(exc)[:200],
+                now=now,
+            )
+            return SweepResult(watermark=after, failure=f"could not list reviews: {exc}")
+
+        candidates = candidates_from_reviews(raw)
+        if not candidates:
+            db.set_watermark(
+                self.conn, name=self.watermark_name, watermark=after, succeeded=True, now=now
+            )
+            return SweepResult(watermark=after)
+
+        known = db.known_review_ids(self.conn, [c.review_id for c in candidates])
+        decisions = [
+            self._handle(candidate, config=config, known=known) for candidate in candidates
+        ]
+
+        watermark = self._advance(decisions, resume_point=after)
+        db.set_watermark(
+            self.conn, name=self.watermark_name, watermark=watermark, succeeded=True, now=now
+        )
+
+        outcomes = tuple(d.outcome for d in decisions if d.outcome is not None)
+        return SweepResult(
+            considered=len(decisions),
+            materialised=sum(1 for o in outcomes if o.status == CREATED),
+            skipped=sum(1 for d in decisions if d.skip_reason is not None)
+            + sum(1 for o in outcomes if o.status == SKIPPED),
+            deferred=sum(1 for o in outcomes if o.status == DEFERRED),
+            failed=sum(1 for o in outcomes if o.status == FAILED),
+            unknown_camera=sum(1 for d in decisions if d.skip_reason == "unknown camera"),
+            watermark=watermark,
+            outcomes=outcomes,
+        )
+
+    # -----------------------------------------------------------------------------------
+
+    def _handle(
+        self,
+        candidate: ReviewCandidate,
+        *,
+        config: CamerasConfig,
+        known: set[str],
+    ) -> _Decision:
+        camera = config.cameras.get(candidate.camera)
+        if camera is None or not camera.enabled:
+            # A review for a camera that is not in `cameras.yml`, or was retired. Not an
+            # error: Frigate may still hold history for a camera we no longer manage.
+            return _Decision(candidate, None, skip_reason="unknown camera")
+
+        verdict = decide(
+            candidate,
+            severities=camera.clip.severities,
+            objects=camera.clip.objects,
+            known_review_ids=known,
+        )
+        if not verdict.materialise:
+            return _Decision(candidate, camera, skip_reason=verdict.reason)
+
+        outcome = self.materialiser.materialise(
+            candidate,
+            pre_roll_s=camera.clip.pre_roll_s,
+            post_roll_s=camera.clip.post_roll_s,
+        )
+        return _Decision(candidate, camera, outcome=outcome)
+
+    def _resume_point(self, now: Instant) -> Instant:
+        state = db.get_watermark(self.conn, self.watermark_name)
+        if state is None or state["watermark"] is None:
+            return now - self.lookback_s
+        return float(state["watermark"])
+
+    def _advance(self, decisions: list[_Decision], *, resume_point: Instant) -> Instant:
+        """The new watermark: past everything settled, but not past anything deferred."""
+        settled: list[float] = []
+        deferred: list[float] = []
+
+        for decision in decisions:
+            end = decision.candidate.end_time
+            if end is None:
+                # An open review has no end to advance to, and it will come back anyway:
+                # the listing treats a null `end_time` as "inside the window" always.
+                continue
+            if decision.outcome is not None and decision.outcome.status == DEFERRED:
+                deferred.append(end)
+            else:
+                # Created, skipped or permanently failed — all terminal. A failure does not
+                # hold the watermark back, or one un-fetchable review would freeze the
+                # sweep forever; it is alerted on instead.
+                settled.append(end)
+
+        watermark = max(settled) if settled else resume_point
+        if deferred:
+            # Retreat to just before the earliest deferral, because the listing's filter is
+            # strictly `end_time > after` — sitting exactly on it would hide it.
+            watermark = min(watermark, min(deferred) - WATERMARK_EPSILON)
+        # Never backwards: redoing work is safe, skipping it is not.
+        return max(watermark, resume_point)
+
+
+def cameras_for(config: CamerasConfig) -> Mapping[str, CameraConfig]:
+    """Convenience for callers that want the enabled cameras only."""
+    return config.enabled_cameras()

cctv/timeutil.py

@@ -0,0 +1,154 @@
+"""Instants and their rendering.
+
+Two kinds of time value exist in this system and confusing them is the single most
+common bug of this kind. See [ADR-0017](../../../docs/adr/0017-time-handling.md) and
+[INV-12](../../../docs/architecture/invariants.md).
+
+* An **instant** is epoch seconds — a bare `float`. It is the same instant everywhere
+  on earth, so it carries no timezone and cannot be "in UTC" or "in IST". This is what
+  Frigate sends, what the database stores, and what every comparison uses.
+* A **rendered** time is a wall-clock value, produced from an instant plus a timezone.
+  This is what a human reads: path components, log lines, the status page.
+
+**Storage is never rendered.** Rendering happens only at the edges, only through this
+module and [`cctv.paths`][cctv.paths], and always with an explicitly supplied timezone.
+
+Why not just store local-time strings: an instant cannot be wrong and cannot lose
+information, so any timezone stays derivable from it. A rendered string *can* be wrong,
+and during a DST fallback two distinct instants render to the same string — which would
+mean a silent overwrite.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import re
+import time
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+__all__ = [
+    "Instant",
+    "TimeConfigError",
+    "format_rfc3339",
+    "has_dst",
+    "now",
+    "render",
+    "resolve_zone",
+    "to_instant",
+]
+
+#: Epoch seconds. An alias, not a class — the point is that a bare `float` is the
+#: correct representation, and wrapping it would invite arithmetic on the wrapper.
+type Instant = float
+
+
+class TimeConfigError(ValueError):
+    """A timezone name that cannot be used. Raised at config load, never at request time."""
+
+
+# `+05:30`, `-0800`, `+05` — a fixed offset. These parse as "valid" but carry no DST
+# history, so a site that observes DST would render wrongly for part of the year.
+_OFFSET_LIKE = re.compile(r"^[+-]\d{2}:?\d{2}$|^[+-]\d{2}$")
+
+
+def resolve_zone(name: str) -> ZoneInfo:
+    """Resolve an IANA timezone name, once, at config load.
+
+    Resolving here and passing `ZoneInfo` around — rather than passing a string and
+    resolving per call — is what makes "resolved once at config load" enforceable
+    ([INV-12](../../../docs/architecture/invariants.md)) instead of a convention.
+
+    Rejects bare offsets, because they cannot express DST history. Rejects names
+    absent from tzdata, because `ZoneInfo` would otherwise fail deep inside a worker
+    on the first clip of the day rather than at start-up.
+
+    Raises:
+        TimeConfigError: the name is missing, is an offset, or is unknown.
+    """
+    if not isinstance(name, str) or not name.strip():
+        raise TimeConfigError(
+            "site.timezone is required and must be a non-empty IANA name. "
+            "It is never guessed from the host: a container's TZ is easy to get "
+            "wrong across six services, and wrong here means silently misplaced clips."
+        )
+
+    name = name.strip()
+
+    if _OFFSET_LIKE.match(name):
+        raise TimeConfigError(
+            f"{name!r} is a UTC offset, not a timezone. Offsets carry no DST history, "
+            f"so a site that observes DST would render wrongly for part of the year. "
+            f"Use an IANA name such as 'Asia/Kolkata'."
+        )
+
+    try:
+        return ZoneInfo(name)
+    except (ZoneInfoNotFoundError, ValueError) as exc:
+        raise TimeConfigError(
+            f"Unknown timezone {name!r}. It must be an IANA name present in tzdata — "
+            f"either the system zoneinfo database or the 'tzdata' package."
+        ) from exc
+
+
+def now() -> Instant:
+    """The current instant.
+
+    Use this instead of `datetime.now()`. `datetime.now()` without a timezone returns a
+    naive local value, which is neither an instant nor a rendering — it is a wart, and
+    a lint rule forbids it in service code.
+    """
+    return time.time()
+
+
+def is_aware(value: _dt.datetime) -> bool:
+    """Whether a datetime carries timezone information."""
+    return value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None
+
+
+def to_instant(value: _dt.datetime) -> Instant:
+    """Convert an **aware** datetime to an instant.
+
+    A naive datetime is rejected rather than assumed to be UTC or local. Both
+    assumptions have shipped as production bugs; refusing to guess is cheaper.
+    """
+    if not is_aware(value):
+        raise ValueError(
+            "A naive datetime has no defined instant. Attach a timezone (usually "
+            "`cctv.timeutil.UTC`) before converting. Guessing between UTC and local "
+            "time is the bug this check exists to prevent."
+        )
+    return value.timestamp()
+
+
+def render(instant: Instant, zone: ZoneInfo) -> _dt.datetime:
+    """Render an instant as an aware wall-clock value in `zone`.
+
+    Total: every instant renders. That is the direction that matters, because
+    instants are what we start from. Going the other way — a wall-clock value to an
+    instant — is *partial*, since a DST spring-forward skips an hour entirely and a
+    fallback repeats one. Nothing in this system does it.
+    """
+    return _dt.datetime.fromtimestamp(instant, tz=zone)
+
+
+def format_rfc3339(instant: Instant, zone: ZoneInfo) -> str:
+    """Render an instant for a human: logs, the status page, notifications."""
+    return render(instant, zone).isoformat(timespec="milliseconds")
+
+
+def has_dst(zone: ZoneInfo, year: int = 2026) -> bool:
+    """Whether `zone` observes DST in `year`.
+
+    Used by tests and by an installer warning. A DST-observing site cannot be treated
+    as though it behaves like IST: Indian Standard Time is UTC+05:30 with no DST, so
+    the fallback-collision case in
+    [ADR-0006](../../../docs/adr/0006-canonical-clip-layout.md) cannot arise there —
+    but it certainly can in Sydney, which is the value in the example config.
+    """
+    early = render(_dt.datetime(year, 1, 15, 12, tzinfo=_dt.UTC).timestamp(), zone)
+    mid = render(_dt.datetime(year, 7, 15, 12, tzinfo=_dt.UTC).timestamp(), zone)
+    return early.utcoffset() != mid.utcoffset()
+
+
+#: Convenience alias, since a great deal of test and generator code needs it.
+UTC = _dt.UTC