digitalkin 1.0.0.dev19__tar.gz → 1.0.0.dev21__tar.gz

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalkin
-Version: 1.0.0.dev19
+Version: 1.0.0.dev21
 Summary: SDK to build kin used in DigitalKin
 Author-email: "DigitalKin.ai" <contact@digitalkin.ai>
 License: Attribution-NonCommercial-ShareAlike 4.0 International

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/pyproject.toml

@@ -37,7 +37,7 @@
         "pydantic-settings>=2.14.1",
         "redis[hiredis]>=7.4.0,<9",
     ]
-    version = "1.0.0.dev19"
+    version = "1.0.0.dev21"
 
     [project.optional-dependencies]
         performance = [ "uvloop>=0.21" ]

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/__version__.py

@@ -5,4 +5,4 @@ from importlib.metadata import PackageNotFoundError, version
 try:
     __version__ = version("digitalkin")
 except PackageNotFoundError:
-    __version__ = "1.0.0.dev19"
+    __version__ = "1.0.0.dev21"

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/community/agno/hitl.py

@@ -611,7 +611,7 @@ class AgnoHitlRunner:
         stream: Any,
         send: Callable[[BaseAgentRunEvent], Coroutine[Any, Any, None]],
         thread_id: str,
-        run_output_cls: type,
+        run_output_cls: type[RunOutput],
     ) -> PauseInfo | None:
         """Drain an Agno stream, forward events, persist on pause.
 

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/grpc_servers/exceptions.py

@@ -19,6 +19,10 @@ class SecurityError(ServerError):
     """Error related to security configuration."""
 
 
+class PermissionDeniedError(ServerError):
+    """Remote service rejected the call with gRPC PERMISSION_DENIED."""
+
+
 class ServerStateError(ServerError):
     """Error related to server state (e.g., already started, not started)."""
 

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/grpc_servers/gateway_servicer.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import asyncio
 import contextlib
 import json
+import random
 import time
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
@@ -34,7 +35,7 @@ from digitalkin.services.communication.exceptions import InvalidConsumerAddressE
 from digitalkin.services.communication.grpc_communication import GrpcCommunication
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator, AsyncIterator
+    from collections.abc import AsyncGenerator, AsyncIterator, Callable
 
     from digitalkin.core.task_manager.module_runner import ModuleRunner
     from digitalkin.core.task_manager.redis.redis_client import RedisClient
@@ -157,7 +158,7 @@ class GatewayServicer:
         await self._m2m.stop()
         await self._registry.shutdown()
 
-    async def StartStream(  # noqa: C901, PLR0911, PLR0915
+    async def StartStream(  # noqa: PLR0911, PLR0915
         self,
         request: Any,
         context: grpc.aio.ServicerContext,
@@ -208,13 +209,18 @@ class GatewayServicer:
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
 
         if self._registry.get(task_id) is not None:
-            logger.debug("Dedup: session exists, reusing", extra=log_extra)
-            return gateway_pb2.StartStreamResponse(accepted=True, task_id=task_id)
+            logger.info(
+                "[VALIDATE RD1] StartStream refused: a dial is already live for task_id=%s",
+                task_id,
+                extra=log_extra,
+            )  # TODO(validate): remove after prod validation
+            return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
         timer.mark("dedup_check")
 
         # Durable at-most-once guard: survives session teardown and spans replicas.
-        # A re-issued StartStream for an already-run task is accepted but does NOT
-        # re-dial/re-execute — the caller resumes the existing output via Stream/from_seq.
+        # Reconnection is server-driven (the live dial-back auto re-dials the same
+        # consumer), so a re-issued StartStream for an already-claimed/running task is
+        # REFUSED — one task_id maps to exactly one dial.
         try:
             claim = await self._idempotency.claim(task_id, SharedRedisListener.PROCESS_ID)
         except RedisError:
@@ -226,59 +232,13 @@ class GatewayServicer:
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
         timer.mark("idempotency_claim")
         if claim is not ClaimResult.CLAIMED:
-            # Re-issued StartStream for an already-claimed task. The prior dial-back
-            # is dead (the dedup branch above returned if its session were still live),
-            # so re-dial in resume mode to re-attach the restarted consumer from its
-            # cursor — as long as the durable stream is still retained.
-            try:
-                stream_len = await self._redis_client.xlen(f"task:{task_id}:stream")
-            except RedisError:
-                logger.warning(
-                    "[VALIDATE R2] StartStream xlen Redis failure → accepted=False",
-                    extra=log_extra,
-                    exc_info=True,
-                )  # TODO(validate): remove after prod validation
-                return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
-            if stream_len == 0:
-                logger.info(
-                    "[VALIDATE IDEM1] no live stream to resume (TTL expired) task_id=%s (%s)",
-                    task_id,
-                    claim.name,
-                    extra=log_extra,
-                )  # TODO(validate): remove after prod validation
-                return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
-            # Atomic guard: recheck + register + spawn with no await that yields
-            # between them, so a concurrent re-poke falls into the dedup branch and acks.
-            if self._registry.get(task_id) is not None:
-                logger.debug("resume-dial already scheduled, reusing", extra=log_extra)
-                return gateway_pb2.StartStreamResponse(accepted=True, task_id=task_id)
-            session = StreamSession(task_id=task_id)
-            registered = await self._registry.register(
-                session,
-                setup_id=request.setup_id,
-                mission_id=request.mission_id,
-            )
-            if not registered:
-                logger.warning("Resume-dial rejected (capacity)", extra=log_extra)
-                return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
             logger.info(
-                "[VALIDATE IDEM1] resume-dial scheduled task_id=%s (%s) stream_len=%d",
+                "[VALIDATE RD1] StartStream refused: task already claimed/running task_id=%s (%s)",
                 task_id,
                 claim.name,
-                stream_len,
                 extra=log_extra,
             )  # TODO(validate): remove after prod validation
-            self._spawn(
-                self._dial_consumer(
-                    task_id=task_id,
-                    mission_id=request.mission_id,
-                    setup_id=request.setup_id,
-                    address=client_address,
-                    resume=True,
-                ),
-                name=f"dial_consumer_{task_id}",
-            )
-            return gateway_pb2.StartStreamResponse(accepted=True, task_id=task_id)
+            return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
 
         session = StreamSession(task_id=task_id)
         accepted = await self._registry.register(
@@ -763,35 +723,133 @@ class GatewayServicer:
                 return
             yield resp
 
-    async def _dial_consumer(  # noqa: C901, PLR0912, PLR0914, PLR0915
+    async def _dial_consumer(  # noqa: C901
         self,
         task_id: str,
         mission_id: str,
         setup_id: str,
         address: str,
-        *,
-        resume: bool = False,
     ) -> None:
-        """Dial the consumer's GatewayService.Stream and run the BiDi.
-
-        Fresh dial: sends ``stream.init``, then on the consumer's first reply
-        (the query) invokes the ``ModuleRunner`` and drains module outputs.
-        Resume dial: evicts any stale channel, sends ``stream.resume``, reads
-        the consumer's cursor from the first reply's ``from_seq``, skips the
-        runner (module already running/done), and drains from that cursor.
-        Channel is pooled by ``GrpcClientWrapper``.
+        """Dial the consumer's GatewayService.Stream, with server-side auto-reconnect.
+
+        First attempt is a fresh dial (``stream.init`` → consumer query →
+        ``ModuleRunner`` → drain from seq 0). If the BiDi dies before the stream
+        is fully delivered and the module is still producing, re-dial the SAME
+        address in resume mode (``stream.resume`` → consumer replies with its
+        ``from_seq`` → drain from that cursor, deduping the Redis queue) with
+        jittered backoff until the client returns or the reconnect window
+        (``DIGITALKIN_GATEWAY_DIAL_BACK_RECONNECT_WINDOW_S``) elapses. The module
+        runs as a separate task, spawned exactly once, so it keeps writing to
+        Redis across reconnects.
 
         Args:
             task_id: Task to push.
             mission_id: Mission ID (logging context).
             setup_id: Setup ID (logging context).
             address: ``host:port`` of the consumer's GatewayService.
-            resume: Re-attach to an existing task's output instead of starting it.
         """
         log_extra = {"task_id": task_id, "setup_id": setup_id, "mission_id": mission_id}
         # Bind IDs so the dial-back BiDi + logs carry them (task-local spawned context).
         RequestContext.bind(task_id=task_id, setup_id=setup_id, mission_id=mission_id)
 
+        module_spawned = False
+
+        def _mark_spawned() -> None:
+            nonlocal module_spawned
+            module_spawned = True
+
+        reconnect = get_gateway_settings().dial_reconnect
+        deadline: float | None = None
+        attempt = 0
+        resume = False
+        try:
+            while True:
+                disconnected = await self._run_dial_attempt(
+                    task_id=task_id,
+                    mission_id=mission_id,
+                    setup_id=setup_id,
+                    address=address,
+                    resume=resume,
+                    on_runner_spawn=_mark_spawned,
+                )
+                if not disconnected:
+                    break
+                # Client BiDi died mid-stream. Re-dial only while there is a
+                # retained stream to resume and the reboot window hasn't elapsed.
+                if not module_spawned:
+                    break
+                session = self._registry.get(task_id)
+                if session is None or session._stop_event.is_set():  # noqa: SLF001
+                    break
+                try:
+                    stream_len = await self._redis_client.xlen(f"task:{task_id}:stream")
+                except RedisError:
+                    break
+                if stream_len == 0:
+                    break
+                now = time.monotonic()
+                if deadline is None:
+                    deadline = now + reconnect.window_s
+                if now >= deadline:
+                    logger.info(
+                        "[VALIDATE RD1] dial-back gave up after %.0fs reconnect window: task_id=%s attempts=%d",
+                        reconnect.window_s,
+                        task_id,
+                        attempt,
+                    )  # TODO(validate): remove after prod validation
+                    break
+                attempt += 1
+                delay = min(random.uniform(reconnect.backoff_base_s, reconnect.backoff_max_s), deadline - now)  # noqa: S311
+                logger.info(
+                    "[VALIDATE RD1] dial-back disconnected — re-dial attempt %d in %.1fs (resume): task_id=%s",
+                    attempt,
+                    delay,
+                    task_id,
+                )  # TODO(validate): remove after prod validation
+                await asyncio.sleep(max(0.0, delay))
+                resume = True
+        finally:
+            # End-of-stream cleanup (once). Output stream is left intact for replay.
+            try:
+                removed = await self._registry.unregister(task_id)
+                if removed is not None:
+                    await removed.teardown()
+            except Exception:
+                logger.exception("end-of-stream unregister failed", extra=log_extra)
+
+    async def _run_dial_attempt(  # noqa: C901, PLR0912, PLR0914, PLR0915
+        self,
+        *,
+        task_id: str,
+        mission_id: str,
+        setup_id: str,
+        address: str,
+        resume: bool,
+        on_runner_spawn: Callable[[], None],
+    ) -> bool:
+        """Run one dial-back BiDi attempt (fresh or resume).
+
+        Fresh (``resume=False``): sends ``stream.init``, spawns the
+        ``ModuleRunner`` on the consumer's first reply (calling
+        ``on_runner_spawn``), drains from seq 0. Resume: sends ``stream.resume``,
+        reads the consumer's cursor from the first reply's ``from_seq``, skips
+        the runner, drains from that cursor. Releases the channel before
+        returning; does NOT unregister the session (the caller owns lifecycle).
+
+        Args:
+            task_id: Task to push.
+            mission_id: Mission ID (logging context).
+            setup_id: Setup ID (logging context).
+            address: ``host:port`` of the consumer's GatewayService.
+            resume: Re-attach to an existing task's output instead of starting it.
+            on_runner_spawn: Called once, when this attempt spawns the module runner.
+
+        Returns:
+            True if the BiDi died before delivering ``stream.end`` (a re-dial
+            candidate); False on clean completion or a terminal failure.
+        """
+        log_extra = {"task_id": task_id, "setup_id": setup_id, "mission_id": mission_id}
+
         async def _fail(code: str, message: str) -> None:
             """Emit a fatal to Redis (fresh dial) or log only (resume).
 
@@ -816,7 +874,7 @@ class GatewayServicer:
                 StreamErrorCode.DIAL_BACK_INTERNAL.value,
                 "gateway has no client_config to dial back with",
             )
-            return
+            return False
         # setup_version_id is unused on the dial-back path.
         comm = GrpcCommunication(
             mission_id=mission_id,
@@ -838,14 +896,14 @@ class GatewayServicer:
                 StreamErrorCode.DIAL_BACK_UNREACHABLE.value,
                 f"dial-back channel build failed: {exc}",
             )
-            return
+            return False
         except OSError as exc:
             logger.exception("dial_consumer: channel build failed addr=%s", address, extra=log_extra)
             await _fail(
                 StreamErrorCode.DIAL_BACK_UNREACHABLE.value,
                 f"dial-back channel build failed: {type(exc).__name__}: {exc}",
             )
-            return
+            return False
         t_stub = time.perf_counter_ns()
         logger.info("→ Dial-back channel ready to %s", address, extra=log_extra)
 
@@ -878,7 +936,7 @@ class GatewayServicer:
                 extra=log_extra,
             )
             await release()
-            return
+            return False
 
         # Outbound is StreamServer, inbound is StreamClient — both share
         # field tags so re-wrapping ``_consume_from_redis`` output is a rename.
@@ -893,8 +951,11 @@ class GatewayServicer:
         outgoing_done = asyncio.Event()
         # Consumer's resume cursor, captured from the first reply's ``from_seq``.
         resume_cursor = 0
+        # Set once the reader reaches EOS and stream.end is delivered to the consumer.
+        delivered_eos = False
 
         async def _outgoing() -> AsyncGenerator:
+            nonlocal delivered_eos
             try:
                 yield init_server
                 logger.info(
@@ -915,6 +976,7 @@ class GatewayServicer:
                     try:
                         cli_msg = await asyncio.wait_for(anext(reader_iter), timeout=idle_timeout)
                     except StopAsyncIteration:
+                        delivered_eos = True
                         break
                     except asyncio.TimeoutError:
                         logger.warning(
@@ -957,6 +1019,7 @@ class GatewayServicer:
         async def _runner_fatal(code: str, message: str) -> None:
             await self._emit_fatal_to_redis(task_id, code=code, message=message, log_extra=log_extra)
 
+        retriable = False
         t_pre_stream = time.perf_counter_ns()
         logger.info(
             "[dial-debug] pre_stream dt_since_ready=%.3fms ch_state=%s",
@@ -1030,7 +1093,7 @@ class GatewayServicer:
                             "gateway has no ModuleRunner configured",
                         )
                         output_started.set()
-                        return
+                        return False
                     self._spawn(
                         self._module_runner.run(
                             upstream.data,
@@ -1041,6 +1104,7 @@ class GatewayServicer:
                         ),
                         name=f"module_runner_{task_id}",
                     )
+                    on_runner_spawn()
                     output_started.set()
                     first = False
                     continue
@@ -1082,6 +1146,8 @@ class GatewayServicer:
                 )
                 # Suppress DIAL_BACK_NO_QUERY in finally — RPC error already emitted.
                 output_started.set()
+            # Client BiDi died before the stream finished → re-dial candidate.
+            retriable = not delivered_eos
         except _GrpcUsageError:
             t_fail = time.perf_counter_ns()
             logger.warning(
@@ -1098,6 +1164,7 @@ class GatewayServicer:
                     "dial-back channel closed before BiDi could start",
                 )
                 output_started.set()
+            retriable = not delivered_eos
         except (RuntimeError, AssertionError, ValueError):
             logger.exception("dial_consumer unexpected error", extra=log_extra)
             if not output_started.is_set():
@@ -1120,10 +1187,4 @@ class GatewayServicer:
             # Unblock _outgoing if consumer never replied.
             output_started.set()
             await release()
-            # End-of-stream cleanup; output stream is left intact for replay.
-            try:
-                removed = await self._registry.unregister(task_id)
-                if removed is not None:
-                    await removed.teardown()
-            except Exception:
-                logger.exception("end-of-stream unregister failed", extra=log_extra)
+        return retriable

digitalkin-1.0.0.dev21/src/digitalkin/grpc_servers/interceptors/permission.py

@@ -0,0 +1,103 @@
+"""Client-side permission middleware for gRPC service-data access.
+
+A single cross-cutting interceptor: any unary call a backend rejects with
+``PERMISSION_DENIED`` surfaces as :class:`PermissionDeniedError`, so callers
+never handle permission per service. Attached on every channel, it covers all
+data services and module-to-module calls uniformly.
+
+The interceptor *returns* a terminal denied call rather than raising: raising a
+non-``AioRpcError`` from an aio interceptor leaks into the intercepted call's
+``__del__`` (grpc only swallows ``AioRpcError``/``CancelledError`` there), so we
+mirror grpc's own ``UnaryUnaryCallResponse`` pattern and raise from ``__await__``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+import grpc
+import grpc.aio
+
+from digitalkin.grpc_servers.exceptions import PermissionDeniedError
+from digitalkin.logger import logger
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable
+
+
+class _DeniedUnaryUnaryCall(grpc.aio.UnaryUnaryCall):
+    """A finished unary call that raises ``PermissionDeniedError`` when awaited."""
+
+    def __init__(self, method: str, details: str) -> None:
+        self._message = f"[{method}] {details or 'permission denied'}"
+        self._details = details
+
+    def cancel(self) -> bool:  # noqa: PLR6301
+        return False
+
+    def cancelled(self) -> bool:  # noqa: PLR6301
+        return False
+
+    def done(self) -> bool:  # noqa: PLR6301
+        return True
+
+    def add_done_callback(self, unused_callback: Any) -> None:
+        """No-op: the call is already finished.
+
+        Args:
+            unused_callback: Ignored; present to satisfy the call interface.
+        """
+
+    def time_remaining(self) -> float | None:  # noqa: PLR6301
+        return None
+
+    async def initial_metadata(self) -> grpc.aio.Metadata:  # noqa: PLR6301
+        return grpc.aio.Metadata()
+
+    async def trailing_metadata(self) -> grpc.aio.Metadata:  # noqa: PLR6301
+        return grpc.aio.Metadata()
+
+    async def code(self) -> grpc.StatusCode:  # noqa: PLR6301
+        return grpc.StatusCode.PERMISSION_DENIED
+
+    async def details(self) -> str:
+        return self._details
+
+    async def debug_error_string(self) -> None:  # noqa: PLR6301
+        return None
+
+    async def wait_for_connection(self) -> None:
+        """No-op: the call already resolved to a permission denial."""
+
+    def __await__(self) -> Any:
+        raise PermissionDeniedError(self._message)
+        yield  # unreachable; marks this method a generator so it is awaitable
+
+
+class PermissionClientInterceptor(grpc.aio.UnaryUnaryClientInterceptor):
+    """Map a ``PERMISSION_DENIED`` unary reply to ``PermissionDeniedError``."""
+
+    async def intercept_unary_unary(  # noqa: PLR6301
+        self,
+        continuation: Callable[[grpc.aio.ClientCallDetails, Any], Awaitable[Any]],
+        client_call_details: grpc.aio.ClientCallDetails,
+        request: Any,
+    ) -> Any:
+        """Return a denied call on PERMISSION_DENIED, else pass the real call through.
+
+        Args:
+            continuation: Downstream call continuation.
+            client_call_details: Original call details.
+            request: Request message.
+
+        Returns:
+            The downstream call, or a terminal call raising PermissionDeniedError on await.
+        """
+        call = await continuation(client_call_details, request)
+        if await call.code() == grpc.StatusCode.PERMISSION_DENIED:
+            method = client_call_details.method
+            method_name = method.decode() if isinstance(method, bytes) else method
+            details = await call.details()
+            logger.warning("permission denied on %s: %s", method_name, details or "")
+            return _DeniedUnaryUnaryCall(method_name, details)
+        return call

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/grpc_servers/utils/grpc_client_wrapper.py

@@ -16,6 +16,7 @@ import grpc.aio
 
 from digitalkin.core.resilience.bulkhead import Bulkhead
 from digitalkin.grpc_servers.exceptions import CircuitOpenError, ServerError
+from digitalkin.grpc_servers.interceptors.permission import PermissionClientInterceptor
 from digitalkin.grpc_servers.interceptors.request_ids import RequestIdClientInterceptor
 from digitalkin.grpc_servers.utils.circuit_breaker import CircuitBreaker
 from digitalkin.logger import logger
@@ -103,7 +104,7 @@ class GrpcClientWrapper:
 
         credentials = self._build_channel_credentials(config)
         grpc_compression = config.compression.to_grpc()
-        interceptors = [RequestIdClientInterceptor()]
+        interceptors = [RequestIdClientInterceptor(), PermissionClientInterceptor()]
         if credentials is not None:
             channel = grpc.aio.secure_channel(
                 config.address,

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/grpc_servers/utils/grpc_error_handler.py

@@ -4,7 +4,7 @@ from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from typing import Any
 
-from digitalkin.grpc_servers.exceptions import ServerError
+from digitalkin.grpc_servers.exceptions import PermissionDeniedError, ServerError
 from digitalkin.logger import logger
 
 
@@ -28,6 +28,7 @@ class GrpcErrorHandlerMixin:
             Context for the operation.
 
         Raises:
+            PermissionDeniedError: Re-raised as-is so the authz status is never masked.
             ServerError: For gRPC-related errors.
             service_error_class: For service-specific errors if provided.
         """
@@ -36,6 +37,8 @@ class GrpcErrorHandlerMixin:
 
         try:
             yield
+        except PermissionDeniedError:
+            raise
         except service_error_class as e:
             # Re-raise service-specific errors as-is
             msg = f"{service_error_class.__name__} in {operation}: {e}"

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/models/settings/gateway.py

@@ -11,7 +11,10 @@ class GatewayStreamSettings(BaseSettings):
 
     model_config = SettingsConfigDict(env_prefix="DIGITALKIN_GATEWAY_STREAM_", case_sensitive=False)
 
-    redis_stream_ttl: int = Field(default=60, description="Stream TTL in seconds after EOS")
+    redis_stream_ttl: int = Field(
+        default=360,
+        description="Stream TTL (s) after EOS; >= reconnect window so a completed stream survives a reboot",
+    )
     redis_stream_initial_ttl: int = Field(default=600, description="Stream TTL in seconds before EOS")
     redis_stream_maxlen: int = Field(default=1000, gt=0, description="Approximate max entries before trimming")
     redis_cursor_ttl: int = Field(default=360, description="Cursor key TTL in seconds")
@@ -113,6 +116,23 @@ class GatewayQueueSettings(BaseSettings):
     )
 
 
+class GatewayDialReconnectSettings(BaseSettings):
+    """Server-side dial-back auto-reconnect window and backoff."""
+
+    model_config = SettingsConfigDict(env_prefix="DIGITALKIN_GATEWAY_DIAL_BACK_RECONNECT_", case_sensitive=False)
+
+    window_s: float = Field(
+        default=300.0,
+        description=(
+            "Total time, measured from the first detected disconnect, that the gateway keeps "
+            "re-dialing a dead consumer so a client that lost internet or rebooted can re-attach "
+            "and resume from its cursor. Keep <= the post-EOS stream TTL."
+        ),
+    )
+    backoff_base_s: float = Field(default=3.0, description="Base delay between re-dial attempts (full-jittered).")
+    backoff_max_s: float = Field(default=5.0, description="Cap on the re-dial backoff delay.")
+
+
 class GatewaySettings(BaseSettings):
     """Top-level gateway configuration."""
 
@@ -149,6 +169,7 @@ class GatewaySettings(BaseSettings):
     stream: GatewayStreamSettings = Field(default_factory=GatewayStreamSettings)
     queue: GatewayQueueSettings = Field(default_factory=GatewayQueueSettings)
     m2m: GatewayM2MSettings = Field(default_factory=GatewayM2MSettings)
+    dial_reconnect: GatewayDialReconnectSettings = Field(default_factory=GatewayDialReconnectSettings)
 
 
 @lru_cache(maxsize=1)

{digitalkin-1.0.0.dev19 → digitalkin-1.0.0.dev21}/src/digitalkin/modules/_base_module.py

@@ -7,6 +7,7 @@ from abc import ABC, abstractmethod
 from collections.abc import Callable, Coroutine
 from typing import Any, ClassVar, Generic
 
+from digitalkin.grpc_servers.exceptions import PermissionDeniedError
 from digitalkin.grpc_servers.utils.utility_schema_extender import UtilitySchemaExtender
 from digitalkin.logger import logger
 from digitalkin.models.module.module import ModuleCodeModel, ModuleStatus
@@ -548,6 +549,10 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
             self._status = ModuleStatus.CANCELLED
             logger.info("Module %s cancelled", self.name, extra=self.context.session.current_ids())
             raise
+        except PermissionDeniedError as e:
+            self._status = ModuleStatus.FAILED
+            logger.warning("Permission denied in module %s: %s", self.name, e, extra=self.context.session.current_ids())
+            await self._notify_permission_denied(self.context.callbacks.send_message, e)
         except Exception as e:
             self._status = ModuleStatus.FAILED
             logger.exception("Error inside module %s", self.name, extra=self.context.session.current_ids())
@@ -564,6 +569,28 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
         else:
             self._status = ModuleStatus.STOPPING
 
+    async def _notify_permission_denied(
+        self,
+        callback: Callable[..., Coroutine[Any, Any, None]],
+        e: PermissionDeniedError,
+    ) -> None:
+        """Send a PermissionDenied notification to the user via the callback.
+
+        Args:
+            callback: The output callback installed on the module context.
+            e: The permission-denied error raised by a service call.
+        """
+        try:
+            await callback(
+                ModuleCodeModel(
+                    code="PermissionDenied",
+                    short_description="Permission denied",
+                    message=str(e),
+                )
+            )
+        except Exception:
+            logger.exception("Failed to send permission-denied callback", extra=self.context.session.current_ids())
+
     async def prepare(
         self,
         setup_data: SetupModelT,
@@ -623,6 +650,14 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
         try:
             await self.prepare(setup_data, callback)
             timer.mark("prepare")
+        except PermissionDeniedError as e:
+            self._status = ModuleStatus.FAILED
+            logger.warning("Permission denied initializing module: %s", e, extra=self.context.session.current_ids())
+            await self._notify_permission_denied(callback, e)
+            if done_callback is not None:
+                await done_callback(None)
+            await self.stop()
+            return
         except Exception as e:
             self._status = ModuleStatus.FAILED
             short_description = "Error initializing module"