digitalkin 1.0.0.dev23__tar.gz → 1.0.0.dev25__tar.gz

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalkin
-Version: 1.0.0.dev23
+Version: 1.0.0.dev25
 Summary: SDK to build kin used in DigitalKin
 Author-email: "DigitalKin.ai" <contact@digitalkin.ai>
 License: Attribution-NonCommercial-ShareAlike 4.0 International

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/pyproject.toml

@@ -37,7 +37,7 @@
         "pydantic-settings>=2.14.1",
         "redis[hiredis]>=7.4.0,<9",
     ]
-    version = "1.0.0.dev23"
+    version = "1.0.0.dev25"
 
     [project.optional-dependencies]
         performance = [ "uvloop>=0.21" ]

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/__version__.py

@@ -5,4 +5,4 @@ from importlib.metadata import PackageNotFoundError, version
 try:
     __version__ = version("digitalkin")
 except PackageNotFoundError:
-    __version__ = "1.0.0.dev23"
+    __version__ = "1.0.0.dev25"

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/profiling/step_timer.py

@@ -51,9 +51,9 @@ class StepTimer:
         total = (self._last - self._t0) / 1e6
         parts.append(f"total={total:.2f}ms")
         if task_id:
-            logger.info("[lat-audit] %s: %s task_id=%s", prefix, " ".join(parts), task_id)
+            logger.debug("[perf] %s: %s task_id=%s", prefix, " ".join(parts), task_id)
         else:
-            logger.info("[lat-audit] %s: %s", prefix, " ".join(parts))
+            logger.debug("[perf] %s: %s", prefix, " ".join(parts))
 
     def total_ms(self) -> float:
         """Total elapsed time across all marks in milliseconds.

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/local_task_manager.py

@@ -91,10 +91,6 @@ class LocalTaskManager(BaseTaskManager):
             else:
                 # H2: this call never registered a session (e.g. duplicate task_id) —
                 # undo only THIS call's admission; never touch the live task.
-                logger.info(
-                    "[VALIDATE H2] create rejected without touching live task: '%s'",
-                    task_id,
-                )  # TODO(validate): remove after prod validation
                 self._release_admission()
             logger.exception(
                 "Failed to create local task: '%s'",

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/module_runner.py

@@ -113,12 +113,6 @@ class ModuleRunner:
                     await self._redis_client.xadd(stream_key, {"eos": b"true"})
                     await self._redis_client.expire(stream_key, stream_settings.redis_stream_ttl)
                     t_eos_write_end = time.perf_counter_ns()
-                    logger.info(
-                        "[VALIDATE M4] proto stream wrote %d seq'd entries (maxlen=%d): task_id=%s",
-                        seq,
-                        stream_maxlen,
-                        task_id,
-                    )  # TODO(validate): remove after prod validation
                     logger.info(
                         "[close-debug] producer_eos_write: xadd_expire=%.2fms t_done_ns=%d task_id=%s",
                         (t_eos_write_end - t_eos_write_start) / 1e6,
@@ -137,8 +131,8 @@ class ModuleRunner:
                 # Arm a TTL on first XADD; final EXPIRE on stream.end shortens it.
                 if not first_logged:
                     elapsed_ms = (time.perf_counter_ns() - runner_start_ns) / 1e6
-                    logger.info(
-                        "[lat-audit] producer_first_byte_to_redis: %.1fms task_id=%s",
+                    logger.debug(
+                        "[perf] producer_first_byte_to_redis: %.1fms task_id=%s",
                         elapsed_ms,
                         task_id,
                         extra=log_extra,
@@ -218,11 +212,6 @@ class ModuleRunner:
             logger.exception("ModuleRunner: backpressure timeout", extra=log_extra)
             await on_fatal(StreamErrorCode.BACKPRESSURE_TIMEOUT.value, str(exc))
         except RedisError as exc:
-            logger.error(
-                "[VALIDATE R2] ModuleRunner: Redis unavailable writing output: task_id=%s",
-                task_id,
-                extra=log_extra,
-            )  # TODO(validate): remove after prod validation
             await on_fatal(
                 StreamErrorCode.REDIS_UNAVAILABLE.value,
                 f"redis unavailable: {type(exc).__name__}: {exc}",

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/redis/proto_streams.py

@@ -150,12 +150,6 @@ class ProtoStreamReader:
                         s.ParseFromString(pb_bytes)
                     except DecodeError:
                         # M6: poison entry (truncated/corrupt pb) — drop it, keep the stream alive.
-                        logger.warning(
-                            "[VALIDATE M6] dropped corrupt proto stream entry: task_id=%s seq=%d bytes=%d",
-                            self._task_id,
-                            seq,
-                            len(pb_bytes),
-                        )  # TODO(validate): remove after prod validation
                         continue
                     yield s
 

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/redis/redis_signal.py

@@ -108,8 +108,8 @@ class SharedRedisListener:
             self._pubsub = self._redis_client.pubsub()
             await self._pubsub.psubscribe("signal_ch:*")
             psub_ms = (time.perf_counter_ns() - psub_t0) / 1e6
-            logger.info(
-                "[lat-audit] signal_psubscribe: psubscribe_ms=%.2f pattern=signal_ch:* phase=boot origin=%s",
+            logger.debug(
+                "[perf] signal_psubscribe: psubscribe_ms=%.2f pattern=signal_ch:* phase=boot origin=%s",
                 psub_ms,
                 SharedRedisListener.PROCESS_ID,
             )
@@ -142,8 +142,8 @@ class SharedRedisListener:
         task.add_done_callback(lambda _: self.unregister(task_id))
 
         self._counters["subscribed"] += 1
-        logger.info(
-            "[lat-audit] signal_subscribe: register_ms=%.2f active_subs=%d task_id=%s origin=%s",
+        logger.debug(
+            "[perf] signal_subscribe: register_ms=%.2f active_subs=%d task_id=%s origin=%s",
             (time.perf_counter_ns() - reg_t0) / 1e6,
             len(self._task_refs),
             task_id,
@@ -176,14 +176,11 @@ class SharedRedisListener:
         if action.startswith("invalidate_"):
             origin = data.get("origin")
             if origin is not None and origin == SharedRedisListener.PROCESS_ID:
-                logger.debug(
-                    "[VALIDATE D3b] skipped self-originated invalidate action=%s", action
-                )  # TODO(validate): remove after prod validation
                 return True
             setup_id = data.get("setup_id", "")
             self._counters["invalidated"] += 1
-            logger.info(
-                "[lat-audit] signal_invalidate: e2e_ms=%.2f action=%s setup_id=%s",
+            logger.debug(
+                "[perf] signal_invalidate: e2e_ms=%.2f action=%s setup_id=%s",
                 e2e_ms,
                 action,
                 setup_id,
@@ -196,8 +193,8 @@ class SharedRedisListener:
                 inv_task.add_done_callback(log_unhandled)
             return True
 
-        logger.info(
-            "[lat-audit] signal_dispatch: e2e_ms=%.2f dispatch_ms=%.2f action=%s task_id=%s",
+        logger.debug(
+            "[perf] signal_dispatch: e2e_ms=%.2f dispatch_ms=%.2f action=%s task_id=%s",
             e2e_ms,
             (time.perf_counter_ns() - dispatch_t0) / 1e6,
             action,
@@ -253,8 +250,8 @@ class SharedRedisListener:
                     psub_t0 = time.perf_counter_ns()
                     await self._pubsub.psubscribe("signal_ch:*")
                     psub_ms = (time.perf_counter_ns() - psub_t0) / 1e6
-                    logger.info(
-                        "[lat-audit] signal_psubscribe: psubscribe_ms=%.2f pattern=signal_ch:* phase=loop",
+                    logger.debug(
+                        "[perf] signal_psubscribe: psubscribe_ms=%.2f pattern=signal_ch:* phase=loop",
                         psub_ms,
                     )
                 msg = await self._pubsub.get_message(ignore_subscribe_messages=True, timeout=0.5)
@@ -264,8 +261,8 @@ class SharedRedisListener:
                         route_task_id, data, raw_json = parsed
                         pub_ns = data.get("published_at_ns") or 0
                         e2e_ms = (time.time_ns() - pub_ns) / 1e6 if pub_ns else 0.0
-                        logger.info(
-                            "[lat-audit] signal_route: e2e_ms=%.2f action=%s task_id=%s",
+                        logger.debug(
+                            "[perf] signal_route: e2e_ms=%.2f action=%s task_id=%s",
                             e2e_ms,
                             data.get("action", ""),
                             route_task_id,
@@ -279,9 +276,6 @@ class SharedRedisListener:
                 if self._pubsub is not None:
                     with contextlib.suppress(Exception):
                         await self._pubsub.aclose()
-                    logger.info(
-                        "[VALIDATE D3a] closed stale pubsub before reconnect (backoff=%.1fs)", backoff
-                    )  # TODO(validate): remove after prod validation
                 self._pubsub = None
                 logger.exception("SharedRedisListener iteration error, retrying in %.1fs", backoff)
                 await asyncio.sleep(backoff)
@@ -290,8 +284,8 @@ class SharedRedisListener:
             now = time.monotonic()
             if now - self._last_counters_log >= 60.0:  # noqa: PLR2004
                 c = self._counters
-                logger.info(
-                    "[lat-audit] signal_counters: origin=%s received=%d deduped=%d evicted=%d "
+                logger.debug(
+                    "[perf] signal_counters: origin=%s received=%d deduped=%d evicted=%d "
                     "dropped=%d listener_restarts=%d active_subs=%d subscribed_total=%d "
                     "invalidated=%d",
                     SharedRedisListener.PROCESS_ID,

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/remote_task_manager.py

@@ -72,10 +72,6 @@ class RemoteTaskManager(BaseTaskManager):
             else:
                 # H2: this call never registered a session (e.g. duplicate task_id) —
                 # undo only THIS call's admission; never touch the live task.
-                logger.info(
-                    "[VALIDATE H2] remote create rejected without touching live task: '%s'",
-                    task_id,
-                )  # TODO(validate): remove after prod validation
                 self._release_admission()
             logger.exception(
                 "Failed to register remote task: '%s'",

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/core/task_manager/task_session.py

@@ -208,8 +208,8 @@ class TaskSession:
         pub_ns = self.last_signal_published_ns
         e2e_ms = (time.time_ns() - pub_ns) / 1e6 if pub_ns else 0.0
         self.last_signal_published_ns = 0
-        logger.info(
-            "[lat-audit] signal_handle: handler=cancel reason=%s e2e_ms=%.2f "
+        logger.debug(
+            "[perf] signal_handle: handler=cancel reason=%s e2e_ms=%.2f "
             "body_ms=%.2f ack_send_ms=%.2f ack_ok=%s task_id=%s",
             reason.value,
             e2e_ms,
@@ -263,8 +263,8 @@ class TaskSession:
         pub_ns = self.last_signal_published_ns
         e2e_ms = (time.time_ns() - pub_ns) / 1e6 if pub_ns else 0.0
         self.last_signal_published_ns = 0
-        logger.info(
-            "[lat-audit] signal_handle: handler=stop reason=%s e2e_ms=%.2f "
+        logger.debug(
+            "[perf] signal_handle: handler=stop reason=%s e2e_ms=%.2f "
             "body_ms=%.2f ack_send_ms=%.2f ack_ok=%s task_id=%s",
             CancellationReason.SIGNAL_SERVICE_STOP.value,
             e2e_ms,

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/gateway_servicer.py

@@ -7,7 +7,6 @@ import contextlib
 import json
 import random
 import time
-import uuid
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
@@ -159,44 +158,15 @@ class GatewayServicer:
         await self._m2m.stop()
         await self._registry.shutdown()
 
-    async def AssociateTask(
-        self,
-        request: Any,
-        context: grpc.aio.ServicerContext,  # noqa: ARG002
-    ) -> Any:
-        """Mint a sub-task id linked to a running parent task.
+    async def AssociateTask(self, request: Any, context: grpc.aio.ServicerContext) -> Any:  # noqa: ARG002, PLR6301
+        """Not served by the SDK — the backend mints sub-tasks.
 
-        Args:
-            request: AssociateTaskRequest proto.
-            context: gRPC service context.
-
-        Returns:
-            AssociateTaskResponse(task_id, parent_task_id); empty ``task_id`` on rejection.
+        Present only so the generated ``add_GatewayServiceServicer_to_server`` finds all
+        four RPCs; nothing dials the module for it. Callers use the backend endpoint.
         """
-        parent_task_id = request.parent_task_id
-        err = GatewayValidator.validate_id(parent_task_id, "parent_task_id")
-        if err is not None:
-            logger.warning("AssociateTask rejected: %s", err)
-            return gateway_pb2.AssociateTaskResponse(task_id="", parent_task_id=parent_task_id)
-
-        child_task_id = str(uuid.uuid4())
-        ttl = get_gateway_settings().stream.redis_stream_initial_ttl
-        try:
-            await self._redis_client.sadd(f"task:{parent_task_id}:children", child_task_id)
-            await self._redis_client.expire(f"task:{parent_task_id}:children", ttl)
-            await self._redis_client.set(f"task:{child_task_id}:parent", parent_task_id, ex=ttl)
-        except RedisError:
-            logger.warning(
-                "[VALIDATE AT1] AssociateTask persist failed parent=%s → task_id=''",
-                parent_task_id,
-                exc_info=True,
-            )  # TODO(validate): remove after prod validation
-            return gateway_pb2.AssociateTaskResponse(task_id="", parent_task_id=parent_task_id)
-
-        logger.info("AssociateTask: parent=%s child=%s", parent_task_id, child_task_id)
-        return gateway_pb2.AssociateTaskResponse(task_id=child_task_id, parent_task_id=parent_task_id)
+        await context.abort(grpc.StatusCode.UNIMPLEMENTED, "AssociateTask is served by the backend")
 
-    async def StartStream(  # noqa: PLR0911, PLR0915
+    async def StartStream(  # noqa: PLR0911
         self,
         request: Any,
         context: grpc.aio.ServicerContext,
@@ -247,11 +217,6 @@ class GatewayServicer:
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
 
         if self._registry.get(task_id) is not None:
-            logger.info(
-                "[VALIDATE RD1] StartStream refused: a dial is already live for task_id=%s",
-                task_id,
-                extra=log_extra,
-            )  # TODO(validate): remove after prod validation
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
         timer.mark("dedup_check")
 
@@ -262,20 +227,9 @@ class GatewayServicer:
         try:
             claim = await self._idempotency.claim(task_id, SharedRedisListener.PROCESS_ID)
         except RedisError:
-            logger.warning(
-                "[VALIDATE R2] StartStream claim Redis failure → accepted=False",
-                extra=log_extra,
-                exc_info=True,
-            )  # TODO(validate): remove after prod validation
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
         timer.mark("idempotency_claim")
         if claim is not ClaimResult.CLAIMED:
-            logger.info(
-                "[VALIDATE RD1] StartStream refused: task already claimed/running task_id=%s (%s)",
-                task_id,
-                claim.name,
-                extra=log_extra,
-            )  # TODO(validate): remove after prod validation
             return gateway_pb2.StartStreamResponse(accepted=False, task_id=task_id)
 
         session = StreamSession(task_id=task_id)
@@ -310,11 +264,6 @@ class GatewayServicer:
                 {"pb": start_info.SerializeToString(), "seq": "0"},
             )
         except RedisError:
-            logger.warning(
-                "[VALIDATE R2] StartStream xadd(stream.start) Redis failure → accepted=False",
-                extra=log_extra,
-                exc_info=True,
-            )  # TODO(validate): remove after prod validation
             # Claimed + registered but the stream couldn't be seeded: undo both so a retry re-runs.
             with contextlib.suppress(RedisError):
                 await self._idempotency.release(task_id)
@@ -571,8 +520,8 @@ class GatewayServicer:
                         extra=log_extra,
                         exc_info=True,
                     )
-                logger.info(
-                    "[lat-audit] SendSignal: %s path=cache total=%.2fms action=%s setup_id=%s",
+                logger.debug(
+                    "[perf] SendSignal: %s path=cache total=%.2fms action=%s setup_id=%s",
                     timer.format_steps(),
                     timer.total_ms(),
                     action_name,
@@ -618,8 +567,8 @@ class GatewayServicer:
             await self._redis_client.publish(f"signal_ch:{task_id}", payload)
             timer.mark("redis_publish")
             last_mark = "redis_publish"
-            logger.info(
-                "[lat-audit] SendSignal: %s path=redis total=%.2fms action=%s task_id=%s",
+            logger.debug(
+                "[perf] SendSignal: %s path=redis total=%.2fms action=%s task_id=%s",
                 timer.format_steps(),
                 timer.total_ms(),
                 action_name,
@@ -728,12 +677,6 @@ class GatewayServicer:
             except StopAsyncIteration:
                 return
             except asyncio.TimeoutError:
-                logger.warning(
-                    "[VALIDATE B1] stream read idle %.0fs with no EOS (producer likely died) — "
-                    "closing stream: task_id=%s",
-                    idle,
-                    task_id,
-                )  # TODO(validate): remove after prod validation
                 yield self._sentinel(
                     0,
                     task_id,
@@ -745,10 +688,6 @@ class GatewayServicer:
                 yield self._sentinel(0, task_id, "stream.end")
                 return
             except RedisError:
-                logger.warning(
-                    "[VALIDATE R2] stream read Redis failure — closing stream: task_id=%s",
-                    task_id,
-                )  # TODO(validate): remove after prod validation
                 yield self._sentinel(
                     0,
                     task_id,
@@ -829,21 +768,9 @@ class GatewayServicer:
                 if deadline is None:
                     deadline = now + reconnect.window_s
                 if now >= deadline:
-                    logger.info(
-                        "[VALIDATE RD1] dial-back gave up after %.0fs reconnect window: task_id=%s attempts=%d",
-                        reconnect.window_s,
-                        task_id,
-                        attempt,
-                    )  # TODO(validate): remove after prod validation
                     break
                 attempt += 1
                 delay = min(random.uniform(reconnect.backoff_base_s, reconnect.backoff_max_s), deadline - now)  # noqa: S311
-                logger.info(
-                    "[VALIDATE RD1] dial-back disconnected — re-dial attempt %d in %.1fs (resume): task_id=%s",
-                    attempt,
-                    delay,
-                    task_id,
-                )  # TODO(validate): remove after prod validation
                 await asyncio.sleep(max(0.0, delay))
                 resume = True
         finally:
@@ -1028,11 +955,6 @@ class GatewayServicer:
                         )
                         return
                     except RedisError:
-                        logger.warning(
-                            "[VALIDATE R2] dial-back stream read Redis failure — closing BiDi: task_id=%s",
-                            task_id,
-                            extra=log_extra,
-                        )  # TODO(validate): remove after prod validation
                         for sc in (
                             self._sentinel(
                                 0,
@@ -1101,14 +1023,6 @@ class GatewayServicer:
                 if first and resume:
                     limit = get_gateway_settings().stream.from_seq_limit
                     resume_cursor = min(upstream.from_seq, limit)
-                    if upstream.from_seq > limit:
-                        logger.warning(
-                            "[VALIDATE RESUME1] resume cursor %d exceeds limit %d, clamped: task_id=%s",
-                            upstream.from_seq,
-                            limit,
-                            task_id,
-                            extra=log_extra,
-                        )  # TODO(validate): remove after prod validation
                     logger.info(
                         "← Consumer resume cursor=%d received — resuming output (no re-run)",
                         resume_cursor,
@@ -1147,18 +1061,11 @@ class GatewayServicer:
                     first = False
                     continue
                 # Follow-up multi-turn input → task's input stream.
-                try:
+                with contextlib.suppress(RedisError):
                     await self._redis_client.xadd(
                         f"task:{task_id}:input",
                         {"pb": upstream.data.SerializeToString()},
                     )
-                except RedisError:
-                    logger.warning(
-                        "[VALIDATE R2] dropped follow-up input (redis unavailable): task_id=%s data=%s",
-                        task_id,
-                        str(upstream.data)[:512],
-                        extra=log_extra,
-                    )  # TODO(validate): remove after prod validation
         except grpc.aio.AioRpcError as exc:
             code_name = exc.code().name
             details = exc.details() or ""

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/m2m_call_registry.py

@@ -83,10 +83,7 @@ class M2MCallRegistry:
         )
         self._breakers[target_key] = breaker
         if len(self._breakers) > 256:  # noqa: PLR2004
-            evicted, _ = self._breakers.popitem(last=False)
-            logger.info(
-                "[VALIDATE D4] evicted LRU m2m breaker target=%s", evicted
-            )  # TODO(validate): remove after prod validation
+            self._breakers.popitem(last=False)
         return breaker
 
     async def acquire_slot(self) -> None:
@@ -222,10 +219,6 @@ class M2MCallRegistry:
                     is_fatal = fatal_field is not None and fatal_field.bool_value
                     if is_fatal:
                         # M2: breaker outcome recorded only in call_module (was double-counted here).
-                        logger.warning(
-                            "[VALIDATE M2] fatal dial-back; breaker recorded only in call_module: task_id=%s",
-                            task_id,
-                        )  # TODO(validate): remove after prod validation
                         return
         finally:
             with contextlib.suppress(asyncio.QueueFull):

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/module_server.py

@@ -197,9 +197,6 @@ class ModuleServer(BaseServer):
         from digitalkin.grpc_servers.utils.grpc_client_wrapper import GrpcClientWrapper
 
         # M9: close channels (resets ref_counts, clears stubs + breakers) instead of a bare cache clear.
-        logger.info(
-            "[VALIDATE M9] closing all cached gRPC channels on INVALIDATE_CHANNELS"
-        )  # TODO(validate): remove after prod validation
         await GrpcClientWrapper.close_all_cached_channels()
         Bulkhead.clear_all()
 
@@ -331,7 +328,6 @@ class ModuleServer(BaseServer):
         if self._gateway_redis_client is not None:
             # M8: this server created the gateway's Redis client, so it closes it (owner closes;
             # the gateway only borrows). Pools were leaked on every server stop.
-            logger.info("[VALIDATE M8] closing gateway Redis pools")  # TODO(validate): remove after prod validation
             try:
                 await self._gateway_redis_client.close()
             except Exception:

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/module_servicer.py

@@ -21,6 +21,7 @@ from digitalkin.core.job_manager.base_job_manager import BaseJobManager
 from digitalkin.core.job_manager.single_job_manager import SingleJobManager
 from digitalkin.core.task_manager.redis.redis_signal import SharedRedisListener
 from digitalkin.grpc_servers.exceptions import PermissionDeniedError, ServicerError
+from digitalkin.grpc_servers.interceptors.request_ids import RequestContext
 from digitalkin.logger import logger
 from digitalkin.models.module.module import ModuleCodeModel
 from digitalkin.models.module.setup_types import SetupModel
@@ -116,7 +117,6 @@ class ModuleServicer(module_service_pb2_grpc.ModuleServiceServicer, ArgParser):
             except Exception:
                 logger.exception("Error closing GrpcUserProfile channel")
         # M8: close the Redis connection pools (were leaked on every server stop).
-        logger.info("[VALIDATE M8] closing module servicer Redis pools")  # TODO(validate): remove after prod validation
         await self._redis_client.close()
 
         if isinstance(self._registry_cache, GrpcRegistry):
@@ -253,7 +253,12 @@ class ModuleServicer(module_service_pb2_grpc.ModuleServiceServicer, ArgParser):
 
         from digitalkin.services.communication.grpc_communication import GrpcCommunication
 
-        self._communication_cache = GrpcCommunication("", "", "", client_config)
+        gateway_backend_config = (self.module_class.services_config_params.get("user_profile") or {}).get(
+            "client_config"
+        )
+        self._communication_cache = GrpcCommunication(
+            "", "", "", client_config, gateway_backend_config=gateway_backend_config
+        )
         return self._communication_cache
 
     def _cache_setup(self, setup_id: str, version_data: SetupVersionData) -> None:
@@ -272,9 +277,17 @@ class ModuleServicer(module_service_pb2_grpc.ModuleServiceServicer, ArgParser):
         Raises:
             PermissionDeniedError: If access to the setup is denied.
         """
-        if not await self.user_profile.check_resource_access(user_profile_pb2.RESOURCE_TYPE_SETUP, setup_id):
+        allowed = await self.user_profile.check_resource_access(user_profile_pb2.RESOURCE_TYPE_SETUP, setup_id)
+        ids = RequestContext.current()
+        if not allowed:
+            logger.info(
+                "[VALIDATE AC1] setup access DENIED: setup_id=%s", setup_id, extra=ids
+            )  # TODO(validate): remove after prod validation
             msg = f"access denied to setup {setup_id}"
             raise PermissionDeniedError(msg)
+        logger.info(
+            "[VALIDATE AC1] setup access granted: setup_id=%s", setup_id, extra=ids
+        )  # TODO(validate): remove after prod validation
 
     async def resolve_setup(self, setup_id: str, mission_id: str) -> SetupVersionData:
         """Return setup version data from cache or remote service.
@@ -362,6 +375,9 @@ class ModuleServicer(module_service_pb2_grpc.ModuleServiceServicer, ArgParser):
         if not await self.user_profile.check_resource_access(
             user_profile_pb2.RESOURCE_TYPE_SETUP, setup_version.setup_id
         ):
+            logger.info(
+                "[VALIDATE AC1] setup config access DENIED: setup_id=%s", setup_version.setup_id
+            )  # TODO(validate): remove after prod validation
             context.set_code(grpc.StatusCode.PERMISSION_DENIED)
             context.set_details(f"access denied to setup {setup_version.setup_id}")
             return lifecycle_pb2.ConfigSetupModuleResponse(success=False)

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/stream_registry.py

@@ -76,11 +76,6 @@ class StreamRegistry:
         settings = get_gateway_settings()
         if len(self._local_cache) >= settings.max_streams:
             # H3: reject at capacity instead of evicting a live session.
-            logger.info(
-                "[VALIDATE H3] stream registry at capacity (%d), rejecting task_id=%s",
-                settings.max_streams,
-                session.task_id,
-            )  # TODO(validate): remove after prod validation
             return False
 
         self._local_cache[session.task_id] = session

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/grpc_servers/utils/grpc_client_wrapper.py

@@ -225,11 +225,12 @@ class GrpcClientWrapper:
         cls._stub_cache.clear()
         CircuitBreaker.clear_all()
 
-    async def exec_grpc_query(  # noqa: C901, PLR0912, PLR0914, PLR0915
+    async def exec_grpc_query(  # noqa: PLR0914
         self,
         query_endpoint: str,
         request: Any,
         timeout: float | None = None,
+        metadata: tuple[tuple[str, str], ...] | None = None,
     ) -> Any:
         """Execute a gRPC query with circuit breaker protection and retry.
 
@@ -245,6 +246,8 @@ class GrpcClientWrapper:
             query_endpoint: rpc query name (e.g., "GetSetup", "CreateSetupVersion")
             request: gRPC protobuf request object
             timeout: Per-call timeout in seconds. ``None`` applies no client-side deadline.
+            metadata: Optional gRPC metadata pairs (e.g. an idempotency key); the same
+                metadata is sent on every retry attempt.
 
         Returns:
             gRPC protobuf response object.
@@ -256,11 +259,6 @@ class GrpcClientWrapper:
         if rpc_method is None:
             # M3: validate the method before claiming the half-open probe lock,
             # so a missing method can't escape cb.check() and wedge the breaker.
-            logger.info(
-                "[VALIDATE M3] RPC method missing, raised before breaker probe: %s.%s",
-                self.service_name,
-                query_endpoint,
-            )  # TODO(validate): remove after prod validation
             msg = f"[gRPC-client:{self.service_name}] RPC method '{query_endpoint}' not found on stub"
             raise ServerError(msg)
 
@@ -273,13 +271,6 @@ class GrpcClientWrapper:
 
         grpc_settings = get_grpc_client_settings()
         eff_timeout = timeout if timeout is not None else grpc_settings.timeout
-        if timeout is None:
-            logger.info(
-                "[VALIDATE H1] default gRPC deadline applied: %.1fs (%s.%s)",
-                eff_timeout,
-                self.service_name,
-                query_endpoint,
-            )  # TODO(validate): remove after prod validation
         max_retries = grpc_settings.max_retries
         backoff_base_ms = grpc_settings.backoff_base_ms
         backoff_delays = tuple(backoff_base_ms / 1000 * (2**i) for i in range(max_retries))
@@ -291,7 +282,7 @@ class GrpcClientWrapper:
                     await asyncio.sleep(backoff_delays[attempt - 1])
 
                 try:
-                    response = await rpc_method(request, timeout=eff_timeout)
+                    response = await rpc_method(request, timeout=eff_timeout, metadata=metadata)
                 except grpc.RpcError as e:
                     last_error = e
                     if e.code() in self._RETRYABLE_CODES and attempt < max_retries:
@@ -342,9 +333,4 @@ class GrpcClientWrapper:
             raise ServerError(error_msg) from last_error
         finally:
             # Free a half-open probe the loop never resolved (e.g. CancelledError mid-call or backoff).
-            if cb.release_probe():
-                logger.info(
-                    "[VALIDATE R1] released abandoned circuit probe: %s.%s",
-                    self.service_name,
-                    query_endpoint,
-                )  # TODO(validate): remove after prod validation
+            cb.release_probe()

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/logger.py

@@ -197,9 +197,6 @@ class LoggerFactory:
             return
         if any(isinstance(h, RotatingFileHandler) for h in logger.handlers):
             # Low: idempotent — repeated setup_logger() calls must not stack handlers.
-            logger.debug(
-                "[VALIDATE Low-loghandler] file handler already attached, skipping"
-            )  # TODO(validate): remove after prod validation
             return
 
         log_file = settings.file or os.path.join(log_dir, f"{logger.name}.log")

{digitalkin-1.0.0.dev23 → digitalkin-1.0.0.dev25}/src/digitalkin/mixins/agui_mixin.py

@@ -30,7 +30,6 @@ from ag_ui.core.events import ToolCallEndEvent as AgUiToolCallEndEvent
 from ag_ui.core.events import ToolCallResultEvent as AgUiToolCallResultEvent
 from ag_ui.core.events import ToolCallStartEvent as AgUiToolCallStartEvent
 
-from digitalkin.logger import logger
 from digitalkin.models.events import (
     AgentRunEvent,
     BaseAgentRunEvent,
@@ -408,9 +407,6 @@ class AgUiMixin:
         event: CustomEvent,
     ) -> None:
         """Handle custom event - emit AG-UI CustomEvent."""
-        logger.info(
-            "[VALIDATE M15] CustomEvent dispatched via AG-UI: name=%s", event.name
-        )  # TODO(validate): remove after prod validation
         from ag_ui.core.events import CustomEvent as AgUiCustomEvent  # pylint: disable=C0415
 
         from digitalkin.models.module.ag_ui import AgUiCustomEventOutput  # pylint: disable=C0415