topicgate 1.1.0__py3-none-any.whl

topicgate/app/services/broker_snapshot_service.py

@@ -0,0 +1,403 @@
+import asyncio
+import base64
+import math
+import time
+from collections.abc import Awaitable, Callable
+from datetime import datetime, timezone
+from uuid import UUID
+
+from topicgate.app.models.broker_snapshot import (
+    BrokerSnapshot,
+    SnapshotBrokerIdentity,
+    SnapshotCompleteness,
+    SnapshotFreshness,
+    SnapshotLimitation,
+    SnapshotPayload,
+    SnapshotPayloadEncoding,
+    SnapshotResultLimit,
+    SnapshotSettling,
+    SnapshotTopicState,
+    SnapshotTopicStatus,
+)
+from topicgate.app.services.broker_resolver import BrokerResolver
+from topicgate.app.topicgate_runtime import TopicGateRuntime
+from topicgate.core.models.broker_summary import BrokerSummary
+from topicgate.core.models.connection_status import ConnectionStatus
+from topicgate.core.models.mqtt_observation import MqttObservation, ObservationSource
+from topicgate.core.models.subscription import Subscription
+from topicgate.core.mqtt_topics import mqtt_filter_matches
+from topicgate.core.observer_limits import MAX_OBSERVED_TOPICS
+from topicgate.core.payload_limits import MAX_RENDERED_PAYLOAD_BYTES
+
+DEFAULT_SNAPSHOT_RESULT_LIMIT = 100
+MAX_SNAPSHOT_RESULT_LIMIT = MAX_OBSERVED_TOPICS
+DEFAULT_SNAPSHOT_WAIT_SECONDS = 1.0
+MAX_SNAPSHOT_WAIT_SECONDS = 5.0
+
+Clock = Callable[[], datetime]
+MonotonicClock = Callable[[], float]
+Sleep = Callable[[float], Awaitable[None]]
+
+
+class BrokerSnapshotService:
+    """Build bounded broker snapshots for any application presentation layer."""
+
+    def __init__(
+        self,
+        runtime: TopicGateRuntime,
+        *,
+        resolver: BrokerResolver | None = None,
+        clock: Clock | None = None,
+        monotonic_clock: MonotonicClock | None = None,
+        sleep: Sleep | None = None,
+    ) -> None:
+        self._runtime = runtime
+        self._resolver = resolver or BrokerResolver(runtime)
+        self._clock = clock or (lambda: datetime.now(timezone.utc))
+        self._monotonic_clock = monotonic_clock or time.monotonic
+        self._sleep = sleep or asyncio.sleep
+
+    async def build(
+        self,
+        broker: UUID | str,
+        *,
+        topic_filter: str = "#",
+        max_age_seconds: float | None = None,
+        result_limit: int = DEFAULT_SNAPSHOT_RESULT_LIMIT,
+        payload_limit_bytes: int = MAX_RENDERED_PAYLOAD_BYTES,
+    ) -> BrokerSnapshot:
+        return self.build_current(
+            broker,
+            topic_filter=topic_filter,
+            max_age_seconds=max_age_seconds,
+            result_limit=result_limit,
+            payload_limit_bytes=payload_limit_bytes,
+        )
+
+    def build_current(
+        self,
+        broker: UUID | str,
+        *,
+        topic_filter: str = "#",
+        max_age_seconds: float | None = None,
+        result_limit: int = DEFAULT_SNAPSHOT_RESULT_LIMIT,
+        payload_limit_bytes: int = MAX_RENDERED_PAYLOAD_BYTES,
+    ) -> BrokerSnapshot:
+        """Build current broker state without activation, waiting, or I/O."""
+        resolved = self._resolver.resolve(broker)
+        validated_filter = self._validate_topic_filter(topic_filter)
+        max_age_seconds = self._validate_max_age(max_age_seconds)
+        result_limit = self._validate_result_limit(result_limit)
+        payload_limit_bytes = self._validate_payload_limit(payload_limit_bytes)
+
+        return self._capture(
+            resolved,
+            topic_filter=validated_filter,
+            max_age_seconds=max_age_seconds,
+            result_limit=result_limit,
+            payload_limit_bytes=payload_limit_bytes,
+            requested_wait_seconds=0.0,
+            actual_wait_seconds=0.0,
+        )
+
+    async def observe(
+        self,
+        broker: UUID | str,
+        *,
+        topic_filter: str = "#",
+        max_age_seconds: float | None = None,
+        result_limit: int = DEFAULT_SNAPSHOT_RESULT_LIMIT,
+        payload_limit_bytes: int = MAX_RENDERED_PAYLOAD_BYTES,
+        wait_seconds: float = DEFAULT_SNAPSHOT_WAIT_SECONDS,
+    ) -> BrokerSnapshot:
+        resolved = self._resolver.resolve(broker)
+        validated_filter = self._validate_topic_filter(topic_filter)
+        max_age_seconds = self._validate_max_age(max_age_seconds)
+        result_limit = self._validate_result_limit(result_limit)
+        payload_limit_bytes = self._validate_payload_limit(payload_limit_bytes)
+        wait_seconds = self._validate_wait_seconds(wait_seconds)
+
+        with self._runtime.control_operation("reconnect and observe"):
+            await self._runtime.activate_broker(resolved.id)
+            actual_wait_seconds = await self._wait(wait_seconds)
+            return self._capture(
+                resolved,
+                topic_filter=validated_filter,
+                max_age_seconds=max_age_seconds,
+                result_limit=result_limit,
+                payload_limit_bytes=payload_limit_bytes,
+                requested_wait_seconds=wait_seconds,
+                actual_wait_seconds=actual_wait_seconds,
+            )
+
+    def _capture(
+        self,
+        resolved: BrokerSummary,
+        *,
+        topic_filter: str,
+        max_age_seconds: float | None,
+        result_limit: int,
+        payload_limit_bytes: int,
+        requested_wait_seconds: float,
+        actual_wait_seconds: float,
+    ) -> BrokerSnapshot:
+        captured_at = self._as_utc(self._clock())
+        model = self._runtime.get_observer_model(resolved.id)
+        matching = sorted(
+            (
+                state
+                for state in model.topic_states.values()
+                if mqtt_filter_matches(topic_filter, state.topic)
+            ),
+            key=lambda state: state.topic,
+        )
+        aged = tuple(
+            (state, self._age_seconds(captured_at, state.received_at))
+            for state in matching
+        )
+        fresh = tuple(
+            (state, age)
+            for state, age in aged
+            if max_age_seconds is None or age <= max_age_seconds
+        )
+        stale_count = len(aged) - len(fresh)
+        selected = fresh[:result_limit]
+        omitted_by_limit = len(fresh) - len(selected)
+        status = self._runtime.get_connection_status(resolved.id)
+        dropped_message_count = self._runtime.get_dropped_message_count(
+            resolved.id
+        )
+        connected_at = self._runtime.get_connected_at(resolved.id)
+        observation_started_at = self._runtime.get_observation_started_at(
+            resolved.id
+        )
+        topics = tuple(
+            self._topic_snapshot(
+                state,
+                age,
+                payload_limit_bytes,
+                observation_started_at=observation_started_at,
+            )
+            for state, age in selected
+        )
+
+        observed_for_seconds = (
+            None
+            if observation_started_at is None
+            else self._age_seconds(captured_at, observation_started_at)
+        )
+        limitations = self._limitations(
+            status=status,
+            matching=matching,
+            topics=topics,
+            observation_started_at=observation_started_at,
+            dropped_message_count=dropped_message_count,
+            stale_count=stale_count,
+            omitted_by_limit=omitted_by_limit,
+        )
+
+        return BrokerSnapshot(
+            broker=SnapshotBrokerIdentity(resolved.id, resolved.name),
+            connection_status=self._status_value(status),
+            captured_at=captured_at,
+            connected_at=(
+                None if connected_at is None else self._as_utc(connected_at)
+            ),
+            observation_started_at=(
+                None
+                if observation_started_at is None
+                else self._as_utc(observation_started_at)
+            ),
+            observed_for_seconds=observed_for_seconds,
+            topic_filter=topic_filter,
+            topics=topics,
+            dropped_message_count=dropped_message_count,
+            freshness=SnapshotFreshness(max_age_seconds, stale_count),
+            results=SnapshotResultLimit(
+                limit=result_limit,
+                total=len(aged),
+                returned=len(topics),
+                omitted=len(aged) - len(topics),
+                omitted_as_stale=stale_count,
+                omitted_by_limit=omitted_by_limit,
+                truncated=omitted_by_limit > 0,
+            ),
+            settling=SnapshotSettling(
+                requested_seconds=requested_wait_seconds,
+                maximum_seconds=MAX_SNAPSHOT_WAIT_SECONDS,
+                actual_seconds=actual_wait_seconds,
+            ),
+            completeness=SnapshotCompleteness(False, limitations),
+        )
+
+    async def _wait(self, wait_seconds: float) -> float:
+        if wait_seconds == 0:
+            return 0.0
+        started_at = self._monotonic_clock()
+        await self._sleep(wait_seconds)
+        return max(0.0, self._monotonic_clock() - started_at)
+
+    @classmethod
+    def _topic_snapshot(
+        cls,
+        state: MqttObservation,
+        age_seconds: float,
+        payload_limit_bytes: int,
+        *,
+        observation_started_at: datetime | None,
+    ) -> SnapshotTopicState:
+        return SnapshotTopicState(
+            topic=state.topic,
+            payload=cls._render_payload(state, payload_limit_bytes),
+            qos=state.qos,
+            retain=state.retain,
+            received_at=cls._as_utc(state.received_at),
+            age_seconds=age_seconds,
+            message_count=state.message_count,
+            source=state.source,
+            status=cls._topic_status(state, observation_started_at),
+        )
+
+    @classmethod
+    def _topic_status(
+        cls,
+        state: MqttObservation,
+        observation_started_at: datetime | None,
+    ) -> SnapshotTopicStatus:
+        if (
+            observation_started_at is not None
+            and cls._as_utc(state.received_at)
+            < cls._as_utc(observation_started_at)
+        ):
+            return SnapshotTopicStatus.STALE
+        if state.source == ObservationSource.STORED:
+            return SnapshotTopicStatus.CACHED
+        return SnapshotTopicStatus.LIVE
+
+    @staticmethod
+    def _render_payload(
+        state: MqttObservation,
+        payload_limit_bytes: int,
+    ) -> SnapshotPayload:
+        available = bytes(state.payload)
+        original_size = max(state.payload_size or 0, len(available))
+        rendered = available[:payload_limit_bytes]
+        try:
+            available.decode("utf-8")
+        except UnicodeDecodeError:
+            encoding = SnapshotPayloadEncoding.BASE64
+            value = base64.b64encode(rendered).decode("ascii")
+            rendered_size = len(rendered)
+        else:
+            encoding = SnapshotPayloadEncoding.UTF8
+            value = rendered.decode("utf-8", errors="ignore")
+            rendered_size = len(value.encode("utf-8"))
+
+        ingestion_truncated = original_size > len(available)
+        rendering_truncated = rendered_size < len(available)
+        return SnapshotPayload(
+            encoding=encoding,
+            value=value,
+            original_size=original_size,
+            available_size=len(available),
+            rendered_size=rendered_size,
+            ingestion_truncated=ingestion_truncated,
+            rendering_truncated=rendering_truncated,
+            truncated=ingestion_truncated or rendering_truncated,
+        )
+
+    @staticmethod
+    def _limitations(
+        *,
+        status: object,
+        matching: list[MqttObservation],
+        topics: tuple[SnapshotTopicState, ...],
+        observation_started_at: datetime | None,
+        dropped_message_count: int,
+        stale_count: int,
+        omitted_by_limit: int,
+    ) -> tuple[SnapshotLimitation, ...]:
+        limitations = [
+            SnapshotLimitation.CURRENT_STATE_ONLY,
+            SnapshotLimitation.RETAINED_DELIVERY_UNCONFIRMED,
+        ]
+        if BrokerSnapshotService._status_value(status) != ConnectionStatus.CONNECTED:
+            limitations.append(SnapshotLimitation.BROKER_DISCONNECTED)
+        if observation_started_at is None:
+            limitations.append(SnapshotLimitation.OBSERVATION_NOT_STARTED)
+        if any(state.source == ObservationSource.STORED for state in matching):
+            limitations.append(
+                SnapshotLimitation.STORED_STATE_PREDATES_OBSERVATION
+            )
+        if dropped_message_count:
+            limitations.append(SnapshotLimitation.DROPPED_MESSAGES)
+        if stale_count:
+            limitations.append(SnapshotLimitation.STALE_STATES_OMITTED)
+        if omitted_by_limit:
+            limitations.append(SnapshotLimitation.RESULT_LIMIT_REACHED)
+        if any(topic.payload.truncated for topic in topics):
+            limitations.append(SnapshotLimitation.PAYLOAD_TRUNCATED)
+        return tuple(limitations)
+
+    @staticmethod
+    def _validate_topic_filter(topic_filter: str) -> str:
+        return Subscription(topic_filter).topic_filter
+
+    @staticmethod
+    def _validate_max_age(max_age_seconds: float | None) -> float | None:
+        if max_age_seconds is None:
+            return None
+        value = float(max_age_seconds)
+        if not math.isfinite(value) or value < 0:
+            raise ValueError("max_age_seconds must be a finite non-negative value.")
+        return value
+
+    @staticmethod
+    def _validate_result_limit(result_limit: int) -> int:
+        if isinstance(result_limit, bool) or not isinstance(result_limit, int):
+            raise ValueError("result_limit must be an integer.")
+        if not 1 <= result_limit <= MAX_SNAPSHOT_RESULT_LIMIT:
+            raise ValueError(
+                f"result_limit must be between 1 and {MAX_SNAPSHOT_RESULT_LIMIT}."
+            )
+        return result_limit
+
+    @staticmethod
+    def _validate_payload_limit(payload_limit_bytes: int) -> int:
+        if isinstance(payload_limit_bytes, bool) or not isinstance(
+            payload_limit_bytes, int
+        ):
+            raise ValueError("payload_limit_bytes must be an integer.")
+        if not 0 <= payload_limit_bytes <= MAX_RENDERED_PAYLOAD_BYTES:
+            raise ValueError(
+                "payload_limit_bytes must be between 0 and "
+                f"{MAX_RENDERED_PAYLOAD_BYTES}."
+            )
+        return payload_limit_bytes
+
+    @staticmethod
+    def _validate_wait_seconds(wait_seconds: float) -> float:
+        value = float(wait_seconds)
+        if not math.isfinite(value) or not 0 <= value <= MAX_SNAPSHOT_WAIT_SECONDS:
+            raise ValueError(
+                "wait_seconds must be between 0 and "
+                f"{MAX_SNAPSHOT_WAIT_SECONDS}."
+            )
+        return value
+
+    @staticmethod
+    def _age_seconds(captured_at: datetime, observed_at: datetime) -> float:
+        return max(
+            0.0,
+            (captured_at - BrokerSnapshotService._as_utc(observed_at)).total_seconds(),
+        )
+
+    @staticmethod
+    def _as_utc(value: datetime) -> datetime:
+        if value.tzinfo is None:
+            return value.replace(tzinfo=timezone.utc)
+        return value.astimezone(timezone.utc)
+
+    @staticmethod
+    def _status_value(status: object) -> str:
+        return str(getattr(status, "value", status)).lower()

topicgate/app/services/control_operation_service.py

@@ -0,0 +1,158 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar
+import threading
+import time
+from uuid import uuid4
+
+from sqlalchemy import text
+
+from topicgate.infrastructure.database.database_context import DatabaseContext
+
+
+class ControlOperationConflict(RuntimeError):
+    """Raised when another TopicGate process owns the control lease."""
+
+
+class ControlOperationService:
+    """Coordinate state-changing desktop and MCP work across processes."""
+
+    def __init__(
+        self,
+        database: DatabaseContext,
+        owner: str,
+        *,
+        lease_seconds: float = 30.0,
+    ) -> None:
+        self._database = database
+        self._owner = owner
+        self._lease_seconds = lease_seconds
+        self._token: ContextVar[str | None] = ContextVar(
+            "topicgate_control_lease_token",
+            default=None,
+        )
+        with self._database.session() as session:
+            self._seen_generation = int(
+                session.execute(
+                    text(
+                        "SELECT generation FROM control_operation_state WHERE id = 1"
+                    )
+                ).scalar_one()
+            )
+
+    @contextmanager
+    def operation(self, name: str) -> Iterator[None]:
+        inherited_token = self._token.get()
+        if inherited_token is not None:
+            yield
+            return
+
+        token = str(uuid4())
+        self._acquire(name, token)
+        context_token = self._token.set(token)
+        stopped = threading.Event()
+        renewer = threading.Thread(
+            target=self._renew_until_stopped,
+            args=(name, token, stopped),
+            daemon=True,
+            name="topicgate-control-lease",
+        )
+        renewer.start()
+        try:
+            yield
+        finally:
+            stopped.set()
+            renewer.join(timeout=1.0)
+            self._release(token)
+            self._token.reset(context_token)
+
+    def _acquire(self, name: str, token: str) -> None:
+        now = time.time()
+        expires_at = now + self._lease_seconds
+        statement = text(
+            "INSERT INTO control_operation_lease "
+            "(id, owner, operation, token, expires_at) "
+            "VALUES (1, :owner, :operation, :token, :expires_at) "
+            "ON CONFLICT(id) DO UPDATE SET "
+            "owner = excluded.owner, operation = excluded.operation, "
+            "token = excluded.token, expires_at = excluded.expires_at "
+            "WHERE control_operation_lease.expires_at <= :now"
+        )
+        with self._database.transaction() as session:
+            generation = int(
+                session.execute(
+                    text(
+                        "SELECT generation FROM control_operation_state WHERE id = 1"
+                    )
+                ).scalar_one()
+            )
+            if generation != self._seen_generation:
+                raise ControlOperationConflict(
+                    "TopicGate configuration changed in another desktop or MCP "
+                    "process. Restart this process to reload the latest broker, "
+                    "subscription, retention, and credential state before retrying."
+                )
+            result = session.execute(
+                statement,
+                {
+                    "owner": self._owner,
+                    "operation": name,
+                    "token": token,
+                    "expires_at": expires_at,
+                    "now": now,
+                },
+            )
+            if result.rowcount == 1:
+                return
+            conflict = session.execute(
+                text(
+                    "SELECT owner, operation, expires_at "
+                    "FROM control_operation_lease WHERE id = 1"
+                )
+            ).one()
+        remaining = max(0.0, float(conflict.expires_at) - now)
+        raise ControlOperationConflict(
+            f"TopicGate {conflict.owner} is already running "
+            f"'{conflict.operation}'. Retry after it finishes "
+            f"(lease expires in at most {remaining:.0f} seconds)."
+        )
+
+    def _renew_until_stopped(
+        self,
+        name: str,
+        token: str,
+        stopped: threading.Event,
+    ) -> None:
+        interval = max(0.01, self._lease_seconds / 3)
+        while not stopped.wait(interval):
+            with self._database.transaction() as session:
+                session.execute(
+                    text(
+                        "UPDATE control_operation_lease "
+                        "SET operation = :operation, expires_at = :expires_at "
+                        "WHERE id = 1 AND token = :token"
+                    ),
+                    {
+                        "operation": name,
+                        "expires_at": time.time() + self._lease_seconds,
+                        "token": token,
+                    },
+                )
+
+    def _release(self, token: str) -> None:
+        with self._database.transaction() as session:
+            deleted = session.execute(
+                text(
+                    "DELETE FROM control_operation_lease "
+                    "WHERE id = 1 AND token = :token"
+                ),
+                {"token": token},
+            )
+            if deleted.rowcount == 1:
+                session.execute(
+                    text(
+                        "UPDATE control_operation_state "
+                        "SET generation = generation + 1 WHERE id = 1"
+                    )
+                )
+                self._seen_generation += 1