digitalkin 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

digitalkin/grpc_servers/registry_servicer.py

@@ -5,7 +5,6 @@ which handles registration, deregistration, discovery, and status management
 of DigitalKin modules.
 """
 
-import logging
 from collections.abc import Iterator
 from enum import Enum
 
@@ -20,7 +19,7 @@ from digitalkin_proto.digitalkin.module_registry.v2 import (
 from pydantic import BaseModel
 from typing_extensions import Self
 
-logger = logging.getLogger(__name__)
+from digitalkin.logger import logger
 
 
 class ExtendedEnum(Enum):
@@ -184,7 +183,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             registration_pb2.RegisterResponse: A response indicating success or failure.
         """
         module_id = request.module_id
-        logger.info("Registering module: %s", module_id)
+        logger.debug("Registering module: %s", module_id)
 
         # Check if module is already registered
         if module_id in self.registered_modules:
@@ -207,7 +206,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             message=None,
         )
 
-        logger.info("Module %s registered at %s:%d", module_id, request.address, request.port)
+        logger.debug("Module %s registered at %s:%d", module_id, request.address, request.port)
         return registration_pb2.RegisterResponse(success=True)
 
     def DeregisterModule(  # noqa: N802
@@ -228,7 +227,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             registration_pb2.DeregisterResponse: A response indicating success or failure.
         """
         module_id = request.module_id
-        logger.info("Deregistering module: %s", module_id)
+        logger.debug("Deregistering module: %s", module_id)
 
         # Check if module exists in registry
         if module_id not in self.registered_modules:
@@ -242,7 +241,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         # Remove the module
         del self.registered_modules[module_id]
 
-        logger.info("Module %s deregistered", module_id)
+        logger.debug("Module %s deregistered", module_id)
         return registration_pb2.DeregisterResponse(success=True)
 
     def DiscoverInfoModule(  # noqa: N802
@@ -261,7 +260,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         Returns:
             discover_pb2.DiscoverInfoResponse: A response containing the module's information.
         """
-        logger.info("Discovering module: %s", request.module_id)
+        logger.debug("Discovering module: %s", request.module_id)
 
         # Check if module exists in registry
         if request.module_id not in self.registered_modules:
@@ -289,24 +288,24 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         Returns:
             discover_pb2.DiscoverSearchResponse: A response containing matching modules.
         """
-        logger.info("Discovering modules with criteria:")
+        logger.debug("Discovering modules with criteria:")
 
         # Start with all modules
         results = list(self.registered_modules.values())
-        logger.info("%s", list(results))
+        logger.debug("%s", list(results))
         # Filter by name if specified
         if request.name:
-            logger.info("\tname %s", request.name)
+            logger.debug("\tname %s", request.name)
             results = [m for m in results if request.name in m.metadata.name]
 
         # Filter by type if specified
         if request.module_type:
-            logger.info("\tmodule_type %s", request.module_type)
+            logger.debug("\tmodule_type %s", request.module_type)
             results = [m for m in results if m.module_type == request.module_type]
 
         # Filter by tags if specified
         if request.tags:
-            logger.info("\ttags %s", request.tags)
+            logger.debug("\ttags %s", request.tags)
             results = [m for m in results if any(tag in m.metadata.tags for tag in request.tags)]
 
         # Filter by description if specified
@@ -315,7 +314,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             results = [m for m in results if request.description in m.metadata.description]
         """
 
-        logger.info("Found %d matching modules", len(results))
+        logger.debug("Found %d matching modules", len(results))
         return discover_pb2.DiscoverSearchResponse(modules=[r.to_proto() for r in results])
 
     def GetModuleStatus(  # noqa: N802
@@ -334,7 +333,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         Returns:
             status_pb2.ModuleStatusResponse: A response containing the module's status.
         """
-        logger.info("Getting status for module: %s", request.module_id)
+        logger.debug("Getting status for module: %s", request.module_id)
 
         # Check if module exists in registry
         if request.module_id not in self.registered_modules:
@@ -363,7 +362,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         Returns:
             status_pb2.ListModulesStatusResponse: A response containing a list of module statuses.
         """
-        logger.info(
+        logger.debug(
             "Getting registered modules with offset %d and limit %d",
             request.offset,
             request.list_size,
@@ -384,7 +383,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             for module in list(self.registered_modules.values())[request.offset : request.offset + list_size]
         ]
 
-        logger.info("Found %d registered modules", len(modules_statuses))
+        logger.debug("Found %d registered modules", len(modules_statuses))
         return status_pb2.ListModulesStatusResponse(
             list_size=len(modules_statuses),
             modules_statuses=modules_statuses,
@@ -406,7 +405,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         Yields:
             status_pb2.ModuleStatusResponse: Responses containing individual module statuses.
         """
-        logger.info("Streaming all %d registered modules", len(self.registered_modules))
+        logger.debug("Streaming all %d registered modules", len(self.registered_modules))
         for module in self.registered_modules.values():
             yield status_pb2.ModuleStatusResponse(
                 module_id=module.module_id,
@@ -431,7 +430,7 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
             status_pb2.UpdateStatusResponse: A response indicating success or failure.
         """
         module_id = request.module_id
-        logger.info("Updating status for module: %s to %s", module_id, request.status)
+        logger.debug("Updating status for module: %s to %s", module_id, request.status)
 
         # Check if module exists in registry
         if request.module_id not in self.registered_modules:
@@ -453,5 +452,5 @@ class RegistryServicer(module_registry_service_pb2_grpc.ModuleRegistryServiceSer
         module_info = self.registered_modules[module_id]
         module_info.status = ModuleStatus(request.status)
 
-        logger.info("Status for module %s updated to %s", module_id, request.status)
+        logger.debug("Status for module %s updated to %s", module_id, request.status)
         return status_pb2.UpdateStatusResponse(success=True)

digitalkin/grpc_servers/utils/grpc_client_wrapper.py

@@ -1,6 +1,5 @@
 """Client wrapper to ease channel creation with specific ServerConfig."""
 
-import logging
 from pathlib import Path
 from typing import Any
 
@@ -8,8 +7,7 @@ import grpc
 
 from digitalkin.grpc_servers.utils.exceptions import ServerError
 from digitalkin.grpc_servers.utils.models import ClientConfig, SecurityMode
-
-logger = logging.getLogger(__name__)
+from digitalkin.logger import logger
 
 
 class GrpcClientWrapper:
@@ -64,9 +62,9 @@ class GrpcClientWrapper:
         """
         try:
             # Call the register method
-            logger.warning("send request to %s", query_endpoint)
+            logger.debug("send request to %s", query_endpoint)
             response = getattr(self.stub, query_endpoint)(request)
-            logger.warning("recive response from request to registry: %s", response)
+            logger.debug("receive response from request to registry: %s", response)
         except grpc.RpcError:
             logger.exception("RPC error during registration:")
             raise ServerError

digitalkin/logger.py

@@ -2,11 +2,46 @@
 
 import logging
 import sys
+from typing import ClassVar
+
+
+class ColorFormatter(logging.Formatter):
+    """Color formatter for logging."""
+
+    grey = "\x1b[38;20m"
+    green = "\x1b[32;20m"
+    blue = "\x1b[34;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format: str = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"  # type: ignore
+
+    FORMATS: ClassVar[dict[int, str]] = {
+        logging.DEBUG: grey + format + reset + "\n",  # type: ignore
+        logging.INFO: green + format + reset + "\n",  # type: ignore
+        logging.WARNING: yellow + format + reset + "\n",  # type: ignore
+        logging.ERROR: red + format + reset + "\n",  # type: ignore
+        logging.CRITICAL: bold_red + format + reset + "\n",  # type: ignore
+    }
+
+    def format(self, record: logging.LogRecord) -> str:  # type: ignore
+        """Format the log record.
+
+        Args:
+            record: The log record to format.
+
+        Returns:
+            str: The formatted log record.
+        """
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
 
 logging.basicConfig(
     level=logging.DEBUG,
     stream=sys.stdout,
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
@@ -15,3 +50,12 @@ logging.getLogger("asyncio").setLevel(logging.DEBUG)
 
 
 logger = logging.getLogger("digitalkin")
+
+if not logger.handlers:
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.INFO)
+
+    ch.setFormatter(ColorFormatter())
+
+    logger.addHandler(ch)
+    logger.propagate = False

digitalkin/models/module/module.py

@@ -14,6 +14,7 @@ class ModuleStatus(Enum):
     STOPPING = auto()  # Module is stopping
     STOPPED = auto()  # Module stop successfuly
     FAILED = auto()  # Module stopped due to internal error
+    CANCELLED = auto()  # Module stopped due to internal error
     NOT_FOUND = auto()
 
 

digitalkin/modules/_base_module.py

@@ -7,6 +7,8 @@ from abc import ABC, abstractmethod
 from collections.abc import Callable, Coroutine
 from typing import Any, ClassVar, Generic
 
+from pydantic import BaseModel
+
 from digitalkin.logger import logger
 from digitalkin.models.module import InputModelT, ModuleStatus, OutputModelT, SecretModelT, SetupModelT
 from digitalkin.services.agent.agent_strategy import AgentStrategy
@@ -20,6 +22,14 @@ from digitalkin.services.storage.storage_strategy import StorageStrategy
 from digitalkin.utils.llm_ready_schema import llm_ready_schema
 
 
+class ModuleErrorModel(BaseModel):
+    """Typed error/code model."""
+
+    code: str
+    exception: str
+    short_description: str
+
+
 class BaseModule(ABC, Generic[InputModelT, OutputModelT, SetupModelT, SecretModelT]):
     """BaseModule is the abstract base for all modules in the DigitalKin SDK."""
 
@@ -48,17 +58,19 @@ class BaseModule(ABC, Generic[InputModelT, OutputModelT, SetupModelT, SecretMode
     def _init_strategies(self) -> None:
         """Initialize the services configuration."""
         for service_name in self.services_config.valid_strategy_names():
-            service = self.services_config.init_strategy(service_name, self.mission_id)
+            service = self.services_config.init_strategy(service_name, self.mission_id, self.setup_version_id)
             setattr(self, service_name, service)
 
     def __init__(
         self,
         job_id: str,
         mission_id: str,
+        setup_version_id: str,
     ) -> None:
         """Initialize the module."""
         self.job_id: str = job_id
         self.mission_id: str = mission_id
+        self.setup_version_id: str = setup_version_id
         self._status = ModuleStatus.CREATED
         self._task: asyncio.Task | None = None
         # Initialize services configuration
@@ -221,30 +233,59 @@ class BaseModule(ABC, Generic[InputModelT, OutputModelT, SetupModelT, SecretMode
             asyncio.CancelledError: If the module is cancelled
         """
         try:
+            logger.warning("Starting module %s", self.name)
             await self.run(input_data, setup_data, callback)
-            await self.stop()
+            logger.warning("Module %s finished", self.name)
         except asyncio.CancelledError:
-            logger.info(f"Module {self.name} cancelled")
+            self._status = ModuleStatus.CANCELLED
+            logger.error(f"Module {self.name} cancelled")
         except Exception:
             self._status = ModuleStatus.FAILED
             logger.exception("Error inside module %s", self.name)
         else:
             self._status = ModuleStatus.STOPPED
+        finally:
+            await self.stop()
 
     async def start(
         self,
         input_data: InputModelT,
         setup_data: SetupModelT,
-        callback: Callable[[OutputModelT], Coroutine[Any, Any, None]],
+        callback: Callable[[OutputModelT | ModuleErrorModel], Coroutine[Any, Any, None]],
+        done_callback: Callable | None = None,
     ) -> None:
         """Start the module."""
         try:
+            logger.info("Inititalize module")
             await self.initialize(setup_data=setup_data)
+        except Exception as e:
+            self._status = ModuleStatus.FAILED
+            short_description = "Error initializing module"
+            logger.exception("%s: %s", short_description, e)
+            await callback(
+                ModuleErrorModel(
+                    code=str(self._status),
+                    short_description=short_description,
+                    exception=str(e),
+                )
+            )
+            if done_callback is not None:
+                await done_callback(None)
+            await self.stop()
+            return
+
+        try:
+            logger.info("Run lifecycle")
             self._status = ModuleStatus.RUNNING
-            self._task = asyncio.create_task(self._run_lifecycle(input_data, setup_data, callback))
+            self._task = asyncio.create_task(
+                self._run_lifecycle(input_data, setup_data, callback),
+                name="module_lifecycle",
+            )
+            if done_callback is not None:
+                self._task.add_done_callback(done_callback)
         except Exception:
             self._status = ModuleStatus.FAILED
-            logger.exception("Error starting module")
+            logger.exception("Error during module lifecyle")
 
     async def stop(self) -> None:
         """Stop the module."""

digitalkin/modules/job_manager/base_job_manager.py

@@ -0,0 +1,139 @@
+"""Background module manager."""
+
+import abc
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Coroutine
+from contextlib import asynccontextmanager
+from typing import Any, Generic
+
+from digitalkin.models import ModuleStatus
+from digitalkin.models.module import InputModelT, OutputModelT, SetupModelT
+from digitalkin.modules._base_module import BaseModule
+from digitalkin.services.services_config import ServicesConfig
+from digitalkin.services.services_models import ServicesMode
+
+
+class BaseJobManager(abc.ABC, Generic[InputModelT, SetupModelT]):
+    """Abstract base class for managing background module jobs."""
+
+    async def _start(self) -> None:
+        """Start the job manager.
+
+        This method initializes any necessary resources or configurations
+        required for the job manager to function.
+        """
+
+    @staticmethod
+    async def job_specific_callback(
+        callback: Callable[[str, OutputModelT], Coroutine[Any, Any, None]], job_id: str
+    ) -> Callable[[OutputModelT], Coroutine[Any, Any, None]]:
+        """Generate a job-specific callback function.
+
+        Args:
+            callback: The callback function to be executed when the job completes.
+            job_id: The unique identifier of the job.
+
+        Returns:
+            Callable: A wrapped callback function that includes the job ID.
+        """
+
+        def callback_wrapper(output_data: OutputModelT) -> Coroutine[Any, Any, None]:
+            """Wrapper for the callback function.
+
+            Args:
+                output_data: The output data produced by the job.
+
+            Returns:
+                Coroutine: The wrapped callback function.
+            """
+            return callback(job_id, output_data)
+
+        return callback_wrapper
+
+    def __init__(
+        self,
+        module_class: type[BaseModule],
+        services_mode: ServicesMode,
+    ) -> None:
+        """Initialize the job manager.
+
+        Args:
+            module_class: The class of the module to be managed.
+            services_mode: The mode of operation for the services (e.g., ASYNC or SYNC).
+        """
+        self.module_class = module_class
+
+        services_config = ServicesConfig(
+            services_config_strategies=self.module_class.services_config_strategies,
+            services_config_params=self.module_class.services_config_params,
+            mode=services_mode,
+        )
+        setattr(self.module_class, "services_config", services_config)
+
+    @abc.abstractmethod  # type: ignore
+    @asynccontextmanager  # type: ignore
+    async def generate_stream_consumer(self, job_id: str) -> AsyncIterator[AsyncGenerator[dict[str, Any], None]]:
+        """Generate a stream consumer for the job's message stream.
+
+        Args:
+            job_id: The unique identifier of the job to filter messages for.
+
+        Yields:
+            dict[str, Any]: The messages from the associated module's stream.
+        """
+
+    @abc.abstractmethod
+    async def create_job(
+        self,
+        input_data: InputModelT,
+        setup_data: SetupModelT,
+        mission_id: str,
+        setup_version_id: str,
+    ) -> str:
+        """Create and start a new job for the module.
+
+        Args:
+            input_data: The input data required to start the job.
+            setup_data: The setup configuration for the module.
+            mission_id: The mission ID associated with the job.
+            setup_version_id: The setup ID associated with the module.
+
+        Returns:
+            str: The unique identifier (job ID) of the created job.
+        """
+
+    @abc.abstractmethod
+    async def stop_module(self, job_id: str) -> bool:
+        """Stop a running module job.
+
+        Args:
+            job_id: The unique identifier of the job to stop.
+
+        Returns:
+            bool: True if the job was successfully stopped, False if it does not exist.
+        """
+
+    @abc.abstractmethod
+    async def get_module_status(self, job_id: str) -> ModuleStatus | None:
+        """Retrieve the status of a module job.
+
+        Args:
+            job_id: The unique identifier of the job.
+
+        Returns:
+            ModuleStatus | None: The status of the job, or None if the job does not exist.
+        """
+
+    @abc.abstractmethod
+    async def stop_all_modules(self) -> None:
+        """Stop all currently running module jobs.
+
+        This method ensures that all active jobs are gracefully terminated.
+        """
+
+    @abc.abstractmethod
+    async def list_modules(self) -> dict[str, dict[str, Any]]:
+        """List all modules along with their statuses.
+
+        Returns:
+            dict[str, dict[str, Any]]: A dictionary containing information about all modules and their statuses.
+        """

digitalkin/modules/job_manager/job_manager_models.py

@@ -0,0 +1,44 @@
+"""Job manager models."""
+
+from enum import Enum
+
+from pydantic import BaseModel
+
+from digitalkin.modules.job_manager.base_job_manager import BaseJobManager
+
+
+class StreamCodeModel(BaseModel):
+    """Typed error/code model."""
+
+    code: str
+
+
+class JobManagerMode(Enum):
+    """Job manager mode."""
+
+    SINGLE = "single"
+    TASKIQ = "taskiq"
+
+    def __str__(self) -> str:
+        """Get the string representation of the job manager mode.
+
+        Returns:
+            str: job manager mode name.
+        """
+        return self.value
+
+    def get_manager_class(self) -> type[BaseJobManager]:
+        """Get the job manager class based on the mode.
+
+        Returns:
+            type: The job manager class.
+        """
+        match self:
+            case JobManagerMode.SINGLE:
+                from digitalkin.modules.job_manager.single_job_manager import SingleJobManager  # noqa: PLC0415
+
+                return SingleJobManager
+            case JobManagerMode.TASKIQ:
+                from digitalkin.modules.job_manager.taskiq_job_manager import TaskiqJobManager  # noqa: PLC0415
+
+                return TaskiqJobManager