digitalkin 1.0.0.dev4__tar.gz → 1.0.0.dev6__tar.gz

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalkin
-Version: 1.0.0.dev4
+Version: 1.0.0.dev6
 Summary: SDK to build kin used in DigitalKin
 Author-email: "DigitalKin.ai" <contact@digitalkin.ai>
 License: Attribution-NonCommercial-ShareAlike 4.0 International
@@ -453,12 +453,12 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ag-ui-protocol>=0.1.18
-Requires-Dist: agentic-mesh-protocol==1.0.0.dev1
+Requires-Dist: agentic-mesh-protocol==1.0.0.dev2
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: grpcio-health-checking==1.80.0
 Requires-Dist: grpcio-reflection==1.80.0
 Requires-Dist: grpcio-status==1.80.0
-Requires-Dist: pydantic>=2.13.4
+Requires-Dist: pydantic>=2.12.4
 Requires-Dist: pydantic-settings>=2.14.0
 Requires-Dist: redis[hiredis]<8,>=7.4.0
 Provides-Extra: performance

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/pyproject.toml

@@ -28,16 +28,16 @@
 
     dependencies = [
         "ag-ui-protocol>=0.1.18",
-        "agentic-mesh-protocol==1.0.0.dev1",
+        "agentic-mesh-protocol==1.0.0.dev2",
         "anyio>=4.13.0",
         "grpcio-health-checking==1.80.0",
         "grpcio-reflection==1.80.0",
         "grpcio-status==1.80.0",
-        "pydantic>=2.13.4",
+        "pydantic>=2.12.4",
         "pydantic-settings>=2.14.0",
         "redis[hiredis]>=7.4.0,<8",
     ]
-    version = "1.0.0.dev4"
+    version = "1.0.0.dev6"
 
     [project.optional-dependencies]
         performance = [ "uvloop>=0.21" ]

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/__version__.py

@@ -5,4 +5,4 @@ from importlib.metadata import PackageNotFoundError, version
 try:
     __version__ = version("digitalkin")
 except PackageNotFoundError:
-    __version__ = "1.0.0.dev4"
+    __version__ = "1.0.0.dev6"

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/core/task_manager/module_runner.py

@@ -115,8 +115,16 @@ class ModuleRunner:
                 nonlocal first_logged
                 data = output_data.model_dump(mode="json")
                 if data.get("root", {}).get("protocol") == "stream.end":
+                    t_eos_write_start = time.perf_counter_ns()
                     await self._redis_client.xadd(stream_key, {"eos": b"true"})
                     await self._redis_client.expire(stream_key, 60)
+                    t_eos_write_end = time.perf_counter_ns()
+                    logger.info(
+                        "[close-debug] producer_eos_write: xadd_expire=%.2fms t_done_ns=%d task_id=%s",
+                        (t_eos_write_end - t_eos_write_start) / 1e6,
+                        t_eos_write_end,
+                        task_id,
+                    )
                     return
                 s = struct_pb2.Struct()
                 s.update(data)

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/core/task_manager/redis/proto_streams.py

@@ -369,11 +369,13 @@ class ProtoStreamReader:
         """
         entries_since_save = 0
         while True:
+            t_xread_start = time.perf_counter_ns()
             result = await self._redis_client.xread(
                 {self._stream_key: self._last_id},
                 count=count,
                 block=block_ms,
             )
+            t_xread_end = time.perf_counter_ns()
             if not result:
                 continue
 
@@ -383,6 +385,13 @@ class ProtoStreamReader:
 
                     eos = fields.get(b"eos", b"")
                     if eos == b"true":
+                        logger.info(
+                            "[close-debug] reader_saw_eos: last_xread_block=%.2fms "
+                            "t_seen_ns=%d task_id=%s",
+                            (t_xread_end - t_xread_start) / 1e6,
+                            t_xread_end,
+                            self._task_id,
+                        )
                         await self._save_cursor()
                         return
 

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/grpc_servers/gateway_servicer.py

@@ -78,7 +78,13 @@ class GatewayServicer:
 
     @staticmethod
     def _sentinel(seq: int, task_id: str, protocol: str, **fields: Any) -> Any:
-        """Build a StreamServer carrying a control sentinel.
+        """Build a StreamClient carrying a control sentinel (server→client wire).
+
+        Under dev2 of agentic-mesh-protocol the Stream RPC is
+        ``rpc Stream(stream StreamServer) returns (stream StreamClient)``,
+        so the server-side response type is ``StreamClient``. ``from_seq``
+        on StreamClient is the wire-equivalent of the old ``seq`` field
+        (same tag 1, same uint64).
 
         ``seq=0`` distinguishes gateway-emitted control entries (validation
         errors, late-consumer rejections) from Redis-replayed entries
@@ -92,11 +98,11 @@ class GatewayServicer:
             fields: Additional Struct fields under ``data.root``.
 
         Returns:
-            StreamServer proto.
+            StreamClient proto.
         """
         s = struct_pb2.Struct()
         s.update({"root": {"protocol": protocol, **fields}})
-        return gateway_pb2.StreamServer(seq=seq, task_id=task_id, data=s)
+        return gateway_pb2.StreamClient(from_seq=seq, task_id=task_id, data=s)
 
     async def _fatal_close(self, task_id: str, code: str, message: str) -> AsyncGenerator:
         """Yield ``stream.error(fatal=true)`` then ``stream.end`` and return.
@@ -342,27 +348,35 @@ class GatewayServicer:
 
     async def Stream(  # noqa: C901, PLR0912
         self,
-        request_iterator: AsyncIterator,
+        request_iterator: AsyncIterator[Any],
         context: grpc.aio.ServicerContext,  # noqa: ARG002
-    ) -> AsyncGenerator:
-        """BiDi: client sends StreamClient messages, server yields StreamServer.
-
-        First StreamClient identifies the task and carries the query as
-        ``data``; the data Struct is delivered to the SDK module as its
-        first input. Subsequent StreamClient messages provide additional
-        upstream input (optional).
-
-        Server emits StreamServer per Redis stream entry. First entry is
+    ) -> AsyncGenerator[Any, None]:
+        """BiDi: client sends StreamServer messages, server yields StreamClient.
+
+        Dev2 of agentic-mesh-protocol defines the RPC as
+        ``rpc Stream(stream StreamServer) returns (stream StreamClient)``.
+        The naming is inverted from intuition: ``StreamServer`` is the
+        message **sent to** the server (upstream input + resume cursor)
+        and ``StreamClient`` is the message **sent to** the client
+        (module output + lifecycle sentinels).
+
+        First StreamServer identifies the task (``task_id``), carries the
+        resume cursor in ``seq``, and carries the query in ``data``; the
+        Struct is delivered to the SDK module as its first input.
+        Subsequent StreamServer messages provide additional upstream input
+        (optional).
+
+        Server emits StreamClient per Redis stream entry. First entry is
         ``stream.start`` (seeded by StartStream); last is ``stream.end``.
         Errors are emitted as ``stream.error(fatal=true)`` followed by
         ``stream.end`` — never via ``context.abort``.
 
         Args:
-            request_iterator: BiDi stream of StreamClient from the client.
+            request_iterator: BiDi stream of StreamServer from the client.
             context: gRPC service context.
 
         Yields:
-            StreamServer — sentinels and SDK module output.
+            StreamClient — sentinels and SDK module output.
         """
         try:
             first_msg = await anext(request_iterator)
@@ -370,7 +384,7 @@ class GatewayServicer:
             return
 
         task_id = first_msg.task_id
-        from_seq = first_msg.from_seq
+        from_seq = first_msg.seq
 
         if validate_id(task_id, "task_id") is not None:
             async for out in self._fatal_close(task_id, "INVALID_ARGUMENT", "invalid task_id"):
@@ -378,7 +392,7 @@ class GatewayServicer:
             return
 
         if from_seq > MAX_FROM_SEQ:
-            async for out in self._fatal_close(task_id, "INVALID_ARGUMENT", "from_seq out of range"):
+            async for out in self._fatal_close(task_id, "INVALID_ARGUMENT", "seq out of range"):
                 yield out
             return
 
@@ -516,17 +530,23 @@ class GatewayServicer:
         no dict conversion, no JSON parsing. ~0.1-0.5ms per message
         instead of ~3-8ms on the JSON path.
 
-        Yields bare ``StreamServer`` (seq + data). Lifecycle is encoded
-        in ``data.root.protocol`` sentinels written by the producer
-        (``stream.start`` already seeded by StartStream; ``stream.end``
-        written by the producer's end-of-stream emit).
+        Yields ``StreamClient`` messages (the dev2 server→client wire
+        type). The ``from_seq`` field carries the monotonic sequence
+        number — same tag as the legacy ``StreamServer.seq``. Lifecycle
+        is encoded in ``data.root.protocol`` sentinels written by the
+        producer (``stream.start`` already seeded by StartStream;
+        ``stream.end`` emitted explicitly below on reader EOS).
+
+        Callers that need a ``StreamServer`` (the dial-back outbound
+        direction) re-wrap each yielded message; both messages share
+        identical field tags so the conversion is a field rename.
 
         Args:
             task_id: Task reference ID.
             from_seq: Resume point.
 
         Yields:
-            StreamServer messages.
+            StreamClient messages.
         """
         t0 = time.perf_counter_ns()
         reader = ProtoStreamReader(task_id, self._redis_client)  # type: ignore[arg-type]
@@ -549,14 +569,22 @@ class GatewayServicer:
                 )
                 first = False
             seq += 1
-            yield gateway_pb2.StreamServer(seq=seq, task_id=task_id, data=struct_data)
+            yield gateway_pb2.StreamClient(from_seq=seq, task_id=task_id, data=struct_data)
 
         # Reader exited because EOS was hit (Redis `{"eos": "true"}` marker is
         # consumed silently by ProtoStreamReader). Emit an explicit stream.end
         # sentinel so the wire contract is uniform: every stream ends with
         # exactly one stream.end entry, regardless of how it concluded.
+        t_after_reader = time.perf_counter_ns()
         seq += 1
         yield self._sentinel(seq, task_id, "stream.end")
+        t_after_yield = time.perf_counter_ns()
+        logger.info(
+            "[close-debug] gateway_stream_end: reader_to_yield=%.2fms t_yielded_ns=%d task_id=%s",
+            (t_after_yield - t_after_reader) / 1e6,
+            t_after_yield,
+            task_id,
+        )
 
     async def _dial_consumer(  # noqa: C901, PLR0912, PLR0915
         self,
@@ -569,16 +597,16 @@ class GatewayServicer:
 
         Flow:
 
-        1. Send ``stream.init`` as the first ``StreamClient``.
+        1. Send ``stream.init`` as the first ``StreamServer``.
         2. Read the consumer's first ``StreamServer`` — that's the query.
         3. Push the query onto ``session.input_queue`` so the dispatcher
            unblocks (matches the M2M client-initiated path).
         4. Concurrently:
 
-           - forward subsequent ``StreamServer`` messages from the
+           - forward subsequent ``StreamClient`` messages from the
              consumer → ``session.input_queue`` (multi-turn input).
            - pull module outputs from ``_consume_from_redis`` and push
-             each as a ``StreamClient(task_id, from_seq=<seq>, data=…)``
+             each as a ``StreamServer(task_id, from_seq=<seq>, data=…)``
              to the consumer.
 
         5. End cleanly when ``stream.end`` flows through.
@@ -646,12 +674,11 @@ class GatewayServicer:
             try:
                 state = chan.get_state(try_to_connect=False)
                 return getattr(state, "name", str(state))
-            except Exception as exc:  # noqa: BLE001
+            except Exception as exc:
                 return f"err:{type(exc).__name__}"
 
         logger.info(
-            "[dial-debug] channel_ready dt_init=%.3fms ch_state=%s "
-            "channel_id=%s ref_count=%d cache_keys=%d",
+            "[dial-debug] channel_ready dt_init=%.3fms ch_state=%s channel_id=%s ref_count=%d cache_keys=%d",
             (t_stub - t_dial0) / 1e6,
             _ch_state(comm._channel),
             id(comm._channel),
@@ -669,19 +696,28 @@ class GatewayServicer:
             await release()
             return
 
-        # Build the init StreamClient using the existing _sentinel helper —
-        # we extract the data Struct from a StreamServer-shaped sentinel.
-        init_server = self._sentinel(0, task_id, "stream.init")
-        init_client = gateway_pb2.StreamClient(task_id=task_id, from_seq=0, data=init_server.data)
+        # Dial-back contract under dev2's RPC signature
+        # ``Stream(stream StreamServer) returns (stream StreamClient)``:
+        # - gateway is the gRPC client and sends ``StreamServer``
+        #   (module output + sentinels) on the request stream.
+        # - gateway receives ``StreamClient`` (query + follow-up upstream)
+        #   on the response stream.
+        # ``_consume_from_redis`` yields ``StreamClient`` for the regular
+        # consumer-facing path; here we re-wrap each one into the
+        # ``StreamServer`` the dial-back wire expects (field tags are
+        # identical so it's a rename: ``from_seq`` → ``seq``).
+        init_struct = struct_pb2.Struct()
+        init_struct.update({"root": {"protocol": "stream.init"}})
+        init_server = gateway_pb2.StreamServer(seq=0, task_id=task_id, data=init_struct)
 
         # Gate the output drain on the first upstream message arriving:
         # the dispatcher can't produce anything until the query lands on
         # session.input_queue, which happens when we receive the consumer's
-        # first StreamServer reply.
+        # first StreamClient reply (carrying the query).
         output_started = asyncio.Event()
 
         async def _outgoing() -> AsyncGenerator:
-            yield init_client
+            yield init_server
             logger.info(
                 "→ stream.init sent, waiting for consumer query before draining outputs",
                 extra={"task_id": task_id, "mission_id": mission_id, "setup_id": setup_id},
@@ -691,11 +727,11 @@ class GatewayServicer:
                 "✓ Output drain started — streaming module outputs to consumer",
                 extra={"task_id": task_id, "mission_id": mission_id, "setup_id": setup_id},
             )
-            async for srv_msg in self._consume_from_redis(task_id, from_seq=0):
-                yield gateway_pb2.StreamClient(
+            async for cli_msg in self._consume_from_redis(task_id, from_seq=0):
+                yield gateway_pb2.StreamServer(
                     task_id=task_id,
-                    from_seq=srv_msg.seq,
-                    data=srv_msg.data,
+                    seq=cli_msg.from_seq,
+                    data=cli_msg.data,
                 )
 
         async def _runner_fatal(code: str, message: str) -> None:
@@ -721,6 +757,12 @@ class GatewayServicer:
             )
             responses = stub.Stream(_outgoing(), timeout=DIAL_BACK_BIDI_TIMEOUT_S)
             first = True
+            # `responses` items are deserialised by gRPC as the proto's
+            # response type (StreamServer), but per the dial-back contract
+            # the consumer is sending StreamClient bytes — the field tags
+            # match so reading `.data` / `.task_id` works either way. The
+            # consumer-emitted message is the query (first) or follow-up
+            # upstream input (subsequent).
             async for upstream in responses:
                 if not (upstream.data and len(upstream.data.fields) > 0):
                     continue
@@ -781,8 +823,7 @@ class GatewayServicer:
         except _GrpcUsageError:
             t_fail = time.perf_counter_ns()
             logger.warning(
-                "[dial-debug] UsageError raised dt_total=%.3fms dt_pre_to_call=%.3fms "
-                "ch_state=%s addr=%s",
+                "[dial-debug] UsageError raised dt_total=%.3fms dt_pre_to_call=%.3fms ch_state=%s addr=%s",
                 (t_fail - t_dial0) / 1e6,
                 (t_fail - t_pre_stream) / 1e6,
                 _ch_state(comm._channel),
@@ -832,5 +873,5 @@ class GatewayServicer:
                 removed = await self._registry.unregister(task_id)
                 if removed is not None:
                     await removed.teardown()
-            except Exception:  # noqa: BLE001 — finally must not raise out
+            except Exception:
                 logger.exception("end-of-stream unregister failed", extra=log_extra)

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/modules/_base_module.py

@@ -618,7 +618,7 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
         timer.mark("init_handlers")
 
         self._prepared = True
-        timer.log("module.prepare", task_id=self.context.session.current_ids().get("task_id", ""))
+        timer.log("module.prepare", task_id=self.context.session.current_ids().get("job_id", ""))
 
     async def start(
         self,
@@ -658,7 +658,7 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
             self._status = ModuleStatus.FAILED
             logger.exception("Error during module lifecyle")
         finally:
-            timer.log("module.start", task_id=self.context.session.current_ids().get("task_id", ""))
+            timer.log("module.start", task_id=self.context.session.current_ids().get("job_id", ""))
             await self.stop()
 
     async def stop(self) -> None:
@@ -694,12 +694,17 @@ class BaseModule(  # Module SDK base class requires many public methods # noqa:
                 )
             t3 = _t.perf_counter_ns()
             self._status = ModuleStatus.STOPPED
+            ids = self.context.session.current_ids()
             logger.info(
-                "module.stop: cleanup=%.1fms flush=%.1fms eos=%.1fms total=%.1fms",
+                "[close-debug] module.stop: cleanup=%.2fms flush=%.2fms eos=%.2fms "
+                "total=%.2fms t_done_ns=%d task_id=%s mission_id=%s",
                 (t1 - t0) / 1e6,
                 (t2 - t1) / 1e6,
                 (t3 - t2) / 1e6,
                 (t3 - t0) / 1e6,
+                t3,
+                ids.get("job_id", ""),
+                ids.get("mission_id", ""),
             )
         except Exception:
             self._status = ModuleStatus.FAILED

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin/services/communication/gateway_consumer.py

@@ -3,9 +3,15 @@
 A *consumer* (UI, dev tool, eval harness, M2M caller) calls
 ``StartStream`` on the gateway with ``x-client-address`` metadata, then
 serves a local ``GatewayService.Stream`` server that the gateway dials
-back into. The gateway delivers ``stream.init`` first, the consumer
-replies with the query as the first ``StreamServer``, and the gateway
-then forwards module outputs as subsequent ``StreamClient`` messages.
+back into. Dial-back contract (inverts the standard Stream RPC roles):
+
+- gateway → consumer: ``StreamServer`` (``stream.init`` first, then
+  module outputs + lifecycle sentinels).
+- consumer → gateway: ``StreamClient`` (the query as the first reply,
+  then any follow-up upstream input).
+
+Wire-compatible with ``Stream(StreamClient) → StreamServer`` because both
+messages share identical proto field tags.
 
 Two construction modes:
 
@@ -132,10 +138,17 @@ class _TaskHandle:
 class _DialBackServicer(gateway_service_pb2_grpc.GatewayServiceServicer):
     """Serves the gateway-initiated ``Stream`` BiDi.
 
-    Reads the first ``StreamClient`` (always ``stream.init``), looks up
+    Dial-back contract (inverts the standard Stream RPC roles): the
+    gateway emits ``StreamServer`` (module output + sentinels) and the
+    consumer (this servicer) replies with ``StreamClient`` (query +
+    follow-up upstream input). Wire-compatible with the proto's
+    ``Stream(StreamClient) → StreamServer`` signature because both
+    messages share identical field tags.
+
+    Reads the first ``StreamServer`` (always ``stream.init``), looks up
     the matching task, replies with the query as the first
-    ``StreamServer``, then forwards every subsequent ``StreamClient``
-    payload onto the per-task output queue.
+    ``StreamClient``, then forwards every subsequent ``StreamServer``
+    payload (module output) onto the per-task output queue.
     """
 
     def __init__(self, registry: dict[str, _TaskHandle]) -> None:
@@ -143,9 +156,9 @@ class _DialBackServicer(gateway_service_pb2_grpc.GatewayServiceServicer):
 
     async def Stream(
         self,
-        request_iterator: AsyncIterator[gateway_pb2.StreamClient],
+        request_iterator: AsyncIterator[gateway_pb2.StreamServer],
         context: grpc.aio.ServicerContext,  # noqa: ARG002
-    ) -> AsyncGenerator[gateway_pb2.StreamServer, None]:
+    ) -> AsyncGenerator[gateway_pb2.StreamClient, None]:
         handle: _TaskHandle | None = None
         try:
             async for upstream in request_iterator:
@@ -157,7 +170,7 @@ class _DialBackServicer(gateway_service_pb2_grpc.GatewayServiceServicer):
                             extra={"task_id": upstream.task_id},
                         )
                         return
-                    yield gateway_pb2.StreamServer(task_id=handle.task_id, seq=0, data=handle.query)
+                    yield gateway_pb2.StreamClient(task_id=handle.task_id, from_seq=0, data=handle.query)
                     continue
                 if upstream.data and len(upstream.data.fields) > 0:
                     await handle.output_queue.put(upstream.data)

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalkin
-Version: 1.0.0.dev4
+Version: 1.0.0.dev6
 Summary: SDK to build kin used in DigitalKin
 Author-email: "DigitalKin.ai" <contact@digitalkin.ai>
 License: Attribution-NonCommercial-ShareAlike 4.0 International
@@ -453,12 +453,12 @@ Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: ag-ui-protocol>=0.1.18
-Requires-Dist: agentic-mesh-protocol==1.0.0.dev1
+Requires-Dist: agentic-mesh-protocol==1.0.0.dev2
 Requires-Dist: anyio>=4.13.0
 Requires-Dist: grpcio-health-checking==1.80.0
 Requires-Dist: grpcio-reflection==1.80.0
 Requires-Dist: grpcio-status==1.80.0
-Requires-Dist: pydantic>=2.13.4
+Requires-Dist: pydantic>=2.12.4
 Requires-Dist: pydantic-settings>=2.14.0
 Requires-Dist: redis[hiredis]<8,>=7.4.0
 Provides-Extra: performance

{digitalkin-1.0.0.dev4 → digitalkin-1.0.0.dev6}/src/digitalkin.egg-info/requires.txt

@@ -1,10 +1,10 @@
 ag-ui-protocol>=0.1.18
-agentic-mesh-protocol==1.0.0.dev1
+agentic-mesh-protocol==1.0.0.dev2
 anyio>=4.13.0
 grpcio-health-checking==1.80.0
 grpcio-reflection==1.80.0
 grpcio-status==1.80.0
-pydantic>=2.13.4
+pydantic>=2.12.4
 pydantic-settings>=2.14.0
 redis[hiredis]<8,>=7.4.0