digitalkin 1.0.0.dev17__tar.gz → 1.0.0.dev19__tar.gz

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalkin
-Version: 1.0.0.dev17
+Version: 1.0.0.dev19
 Summary: SDK to build kin used in DigitalKin
 Author-email: "DigitalKin.ai" <contact@digitalkin.ai>
 License: Attribution-NonCommercial-ShareAlike 4.0 International
@@ -460,7 +460,7 @@ Requires-Dist: grpcio-reflection==1.81.0
 Requires-Dist: grpcio-status==1.81.0
 Requires-Dist: pydantic>=2.12.4
 Requires-Dist: pydantic-settings>=2.14.1
-Requires-Dist: redis[hiredis]<8,>=7.4.0
+Requires-Dist: redis[hiredis]<9,>=7.4.0
 Provides-Extra: performance
 Requires-Dist: uvloop>=0.21; extra == "performance"
 Provides-Extra: profiling

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/pyproject.toml

@@ -35,9 +35,9 @@
         "grpcio-status==1.81.0",
         "pydantic>=2.12.4",
         "pydantic-settings>=2.14.1",
-        "redis[hiredis]>=7.4.0,<8",
+        "redis[hiredis]>=7.4.0,<9",
     ]
-    version = "1.0.0.dev17"
+    version = "1.0.0.dev19"
 
     [project.optional-dependencies]
         performance = [ "uvloop>=0.21" ]
@@ -62,14 +62,14 @@
         "cryptography>=48.0.0",
         "mypy>=2.1.0",
         "pre-commit>=4.6.0",
-        "pyright>=1.1.409",
-        "ruff>=0.15.13",
+        "pyright>=1.1.411",
+        "ruff>=0.15.20",
         "twine>=6.2.0",
         "types-grpcio-health-checking>=1.0.0.20260518",
         "types-grpcio-reflection>=1.0.0.20260508",
         "types-grpcio>=1.0.0.20260518",
         "types-protobuf>=7.34.1.20260518",
-        "typos>=1.46.2",
+        "typos>=1.48.0",
     ]
     docs = [
         "griffe-inherited-docstrings>=1.1.3",

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/__version__.py

@@ -5,4 +5,4 @@ from importlib.metadata import PackageNotFoundError, version
 try:
     __version__ = version("digitalkin")
 except PackageNotFoundError:
-    __version__ = "1.0.0.dev17"
+    __version__ = "1.0.0.dev19"

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/core/task_manager/module_runner.py

@@ -8,10 +8,12 @@ from typing import TYPE_CHECKING, Any
 
 from google.protobuf import json_format, struct_pb2
 from pydantic import ValidationError
+from redis.exceptions import RedisError
 
 from digitalkin.core.exceptions import BackpressureTimeoutError
 from digitalkin.core.profiling.step_timer import StepTimer
 from digitalkin.core.profiling.task_profiler import TaskProfiler
+from digitalkin.grpc_servers.interceptors.request_ids import RequestContext
 from digitalkin.logger import logger
 from digitalkin.models.grpc_servers.stream_error_codes import StreamErrorCode
 from digitalkin.models.settings.gateway import get_gateway_settings
@@ -40,7 +42,7 @@ class ModuleRunner:
         self._redis_client = redis_client
         self._servicer = servicer
 
-    async def run(  # noqa: PLR0914, PLR0915
+    async def run(  # noqa: C901, PLR0914, PLR0915
         self,
         query: struct_pb2.Struct,
         *,
@@ -62,6 +64,9 @@ class ModuleRunner:
         log_extra = {"task_id": task_id, "setup_id": setup_id, "mission_id": mission_id}
         stream_key = f"task:{task_id}:stream"
         timer = StepTimer()
+        # Bind IDs so every downstream gRPC call (registry/storage/cost/...) and
+        # log record made during this task carries them via the interceptors/filter.
+        ctx_token = RequestContext.bind(task_id=task_id, setup_id=setup_id, mission_id=mission_id)
 
         # Construct profiler outside try so finally can always stop it.
         profiling = get_profiling_settings()
@@ -211,6 +216,16 @@ class ModuleRunner:
         except BackpressureTimeoutError as exc:
             logger.exception("ModuleRunner: backpressure timeout", extra=log_extra)
             await on_fatal(StreamErrorCode.BACKPRESSURE_TIMEOUT.value, str(exc))
+        except RedisError as exc:
+            logger.error(
+                "[VALIDATE R2] ModuleRunner: Redis unavailable writing output: task_id=%s",
+                task_id,
+                extra=log_extra,
+            )  # TODO(validate): remove after prod validation
+            await on_fatal(
+                StreamErrorCode.REDIS_UNAVAILABLE.value,
+                f"redis unavailable: {type(exc).__name__}: {exc}",
+            )
         except Exception as exc:
             logger.exception("ModuleRunner: module job failed", extra=log_extra)
             await on_fatal(
@@ -219,3 +234,4 @@ class ModuleRunner:
             )
         finally:
             profiler.stop()
+            RequestContext.reset(ctx_token)

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/core/task_manager/redis/proto_streams.py

@@ -73,6 +73,7 @@ class ProtoStreamReader:
         self,
         count: int = 50,
         cursor_save_interval: int = 100,
+        skip_to_seq: int | None = None,
     ) -> AsyncGenerator[struct_pb2.Struct, None]:
         """Read proto Structs from the stream until EOS.
 
@@ -87,6 +88,9 @@ class ProtoStreamReader:
         Args:
             count: Max entries per XREAD call.
             cursor_save_interval: Save cursor every N entries (default 100).
+            skip_to_seq: If set, entries with stored ``seq <= skip_to_seq``
+                are consumed (cursor and gap detection advance) but not
+                yielded, so a resumed reader starts past the consumer's cursor.
 
         Yields:
             Proto Struct objects from the stream.
@@ -134,6 +138,9 @@ class ProtoStreamReader:
                         )
                     self._last_seq = seq
 
+                    if skip_to_seq is not None and seq <= skip_to_seq:
+                        continue
+
                     pb_bytes = fields.get(b"pb", b"")
                     if not pb_bytes:
                         continue

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/core/task_manager/redis/redis_client.py

@@ -83,8 +83,8 @@ class RedisClient:  # noqa: PLR0904
         try:
             timeout = get_redis_settings().pool.health_check_timeout
             results = await asyncio.gather(
-                asyncio.wait_for(self._client.ping(), timeout=timeout),  # type: ignore[arg-type]
-                asyncio.wait_for(self._blocking_client.ping(), timeout=timeout),  # type: ignore[arg-type]
+                asyncio.wait_for(self._client.ping(), timeout=timeout),
+                asyncio.wait_for(self._blocking_client.ping(), timeout=timeout),
             )
             return all(results)
         except Exception:
@@ -107,7 +107,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Number of fields added (not updated).
         """
-        return await self._client.hset(name, mapping=mapping)  # type: ignore[misc]
+        return await self._client.hset(name, mapping=mapping)  # type: ignore[arg-type]
 
     async def hgetall(self, name: str) -> dict[bytes, bytes]:
         """Get all fields and values in a Redis hash.
@@ -118,7 +118,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             All field-value pairs as bytes.
         """
-        return await self._client.hgetall(name)  # type: ignore[misc]
+        return await self._client.hgetall(name)  # type: ignore[return-value]
 
     async def publish(self, channel: str, message: str | bytes) -> int:
         """Publish a message to a Redis pub/sub channel.
@@ -161,7 +161,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             True if Redis responds.
         """
-        return await self._client.ping()  # type: ignore[misc]
+        return await self._client.ping()
 
     def pubsub(self) -> aioredis.client.PubSub:
         """Return a PubSub object for subscribe operations.
@@ -200,7 +200,7 @@ class RedisClient:  # noqa: PLR0904
         if maxlen is not None:
             kwargs["maxlen"] = maxlen
             kwargs["approximate"] = True
-        return await self._client.xadd(name, fields, **kwargs)  # type: ignore[arg-type]
+        return await self._client.xadd(name, fields, **kwargs)  # type: ignore[arg-type, return-value]
 
     async def xread(
         self,
@@ -222,7 +222,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             List of [stream_key, [(entry_id, fields), ...]] pairs.
         """
-        return await self._blocking_client.xread(streams, count=count, block=block)  # type: ignore[arg-type]
+        return await self._blocking_client.xread(streams, count=count, block=block)  # type: ignore[arg-type, return-value]
 
     async def xlen(self, name: str) -> int:
         """Get the number of entries in a Redis Stream.
@@ -253,7 +253,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             List of (entry_id, fields) tuples, newest first.
         """
-        return await self._client.xrevrange(name, max=max_id, min=min_id, count=count)
+        return await self._client.xrevrange(name, max=max_id, min=min_id, count=count)  # type: ignore[return-value]
 
     async def zadd(self, name: str, mapping: dict[str, float]) -> int:
         """Add members to a sorted set with scores.
@@ -265,7 +265,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Number of members added.
         """
-        return await self._client.zadd(name, mapping)
+        return await self._client.zadd(name, mapping)  # type: ignore[return-value]
 
     async def zrangebyscore(
         self,
@@ -283,7 +283,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             List of member values.
         """
-        return await self._client.zrangebyscore(name, min_score, max_score)
+        return await self._client.zrangebyscore(name, min_score, max_score)  # type: ignore[return-value]
 
     async def zrem(self, name: str, *members: str) -> int:
         """Remove members from a sorted set.
@@ -319,7 +319,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Script return value.
         """
-        return await self._client.eval(script, len(keys), *keys, *args)  # type: ignore[misc]
+        return await self._client.eval(script, len(keys), *keys, *args)
 
     async def get(self, name: str) -> bytes | None:
         """Get the value of a key.
@@ -330,7 +330,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Value as bytes, or None if key does not exist.
         """
-        return await self._client.get(name)
+        return await self._client.get(name)  # type: ignore[return-value]
 
     async def set(
         self,
@@ -349,7 +349,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             True if set successfully.
         """
-        return await self._client.set(name, value, ex=ex)
+        return await self._client.set(name, value, ex=ex)  # type: ignore[return-value]
 
     async def sadd(self, name: str, *values: str) -> int:
         """Add members to a Redis set.
@@ -361,7 +361,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Number of members added.
         """
-        return await self._client.sadd(name, *values)  # type: ignore[misc]
+        return await self._client.sadd(name, *values)
 
     async def srem(self, name: str, *values: str) -> int:
         """Remove members from a Redis set.
@@ -373,7 +373,7 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Number of members removed.
         """
-        return await self._client.srem(name, *values)  # type: ignore[misc]
+        return await self._client.srem(name, *values)
 
     async def smembers(self, name: str) -> builtins.set[bytes]:
         """Get all members of a Redis set.
@@ -384,4 +384,4 @@ class RedisClient:  # noqa: PLR0904
         Returns:
             Set of member values as bytes.
         """
-        return await self._client.smembers(name)  # type: ignore[misc]
+        return await self._client.smembers(name)  # type: ignore[return-value]

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/core/task_manager/redis/redis_state.py

@@ -59,7 +59,7 @@ class RedisStateManager:
                 mapping[k] = str(v)
         # Pipeline: HSET + EXPIRE in 1 round-trip instead of 2
         pipe = self._redis_client.pipeline()
-        pipe.hset(key, mapping=mapping)
+        pipe.hset(key, mapping=mapping)  # type: ignore[arg-type]
         pipe.expire(key, get_redis_settings().task_ttl)
         await pipe.execute()
         logger.debug("RedisStateManager.set_status: task_id=%s status=%s", task_id, status)
@@ -94,7 +94,7 @@ class RedisStateManager:
         if exception_traceback is not None:
             mapping["exception_traceback"] = exception_traceback
         pipe = self._redis_client.pipeline()
-        pipe.hset(key, mapping=mapping)
+        pipe.hset(key, mapping=mapping)  # type: ignore[arg-type]
         pipe.expire(key, get_redis_settings().task_ttl)
         await pipe.execute()
 

{digitalkin-1.0.0.dev17 → digitalkin-1.0.0.dev19}/src/digitalkin/grpc_servers/_base_server.py

@@ -28,6 +28,7 @@ from digitalkin.grpc_servers.exceptions import (
     ServerStateError,
     ServicerError,
 )
+from digitalkin.grpc_servers.interceptors.request_ids import RequestIdServerInterceptor
 from digitalkin.grpc_servers.utils.grpc_client_wrapper import GrpcClientWrapper
 from digitalkin.logger import logger
 from digitalkin.models.grpc_servers.types import GrpcServer, ServiceDescriptor, T
@@ -214,7 +215,7 @@ class BaseServer(abc.ABC):
                 server = grpc_aio.server(
                     options=get_server_settings().grpc.options,
                     compression=grpc_compression,
-                    interceptors=self._interceptors or None,
+                    interceptors=[RequestIdServerInterceptor(), *self._interceptors],
                     maximum_concurrent_rpcs=get_server_settings().max_concurrent_rpcs,
                     migration_thread_pool=futures.ThreadPoolExecutor(
                         max_workers=get_server_settings().thread_pool_workers