digitalkin 0.3.1.dev1__py3-none-any.whl → 0.3.2a2__py3-none-any.whl

digitalkin/modules/_base_module.py

@@ -2,24 +2,28 @@
 
 import asyncio
 import json
+import os
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Coroutine
 from typing import Any, ClassVar, Generic
 
+from digitalkin.grpc_servers.utils.utility_schema_extender import UtilitySchemaExtender
 from digitalkin.logger import logger
-from digitalkin.models.module import (
+from digitalkin.models.module.module import ModuleCodeModel, ModuleStatus
+from digitalkin.models.module.module_context import ModuleContext
+from digitalkin.models.module.module_types import (
+    DataModel,
     InputModelT,
-    ModuleStatus,
     OutputModelT,
     SecretModelT,
     SetupModelT,
 )
-from digitalkin.models.module.module import ModuleCodeModel
-from digitalkin.models.module.module_context import ModuleContext
+from digitalkin.models.module.utility import EndOfStreamOutput, ModuleStartInfoOutput, UtilityProtocol
+from digitalkin.models.services.storage import BaseRole
 from digitalkin.modules.trigger_handler import TriggerHandler
 from digitalkin.services.services_config import ServicesConfig, ServicesStrategy
-from digitalkin.utils.llm_ready_schema import llm_ready_schema
 from digitalkin.utils.package_discover import ModuleDiscoverer
+from digitalkin.utils.schema_splitter import SchemaSplitter
 
 
 class BaseModule(  # noqa: PLR0904
@@ -46,10 +50,20 @@ class BaseModule(  # noqa: PLR0904
     triggers_discoverer: ClassVar[ModuleDiscoverer]
 
     # service config params
-    services_config_strategies: ClassVar[dict[str, ServicesStrategy | None]]
-    services_config_params: ClassVar[dict[str, dict[str, Any | None] | None]]
+    services_config_strategies: ClassVar[dict[str, ServicesStrategy | None]] = {}
+    services_config_params: ClassVar[dict[str, dict[str, Any | None] | None]] = {}
     services_config: ServicesConfig
 
+    @classmethod
+    def get_module_id(cls) -> str:
+        """Get the module ID from environment variable or metadata.
+
+        Returns:
+            The module_id from DIGITALKIN_MODULE_ID env var, or metadata module_id,
+            or "unknown" if neither exists.
+        """
+        return os.environ.get("DIGITALKIN_MODULE_ID") or cls.metadata.get("module_id", "unknown")
+
     def _init_strategies(self, mission_id: str, setup_id: str, setup_version_id: str) -> dict[str, Any]:
         """Initialize the services configuration.
 
@@ -62,6 +76,7 @@ class BaseModule(  # noqa: PLR0904
                 registry: RegistryStrategy
                 snapshot: SnapshotStrategy
                 storage: StorageStrategy
+                user_profile: UserProfileStrategy
         """
         logger.debug("Service initialisation: %s", self.services_config_strategies.keys())
         return {
@@ -107,95 +122,175 @@ class BaseModule(  # noqa: PLR0904
         return self._status
 
     @classmethod
-    def get_secret_format(cls, *, llm_format: bool) -> str:
+    async def get_secret_format(cls, *, llm_format: bool) -> str:
         """Get the JSON schema of the secret format model.
 
-        Raises:
-            NotImplementedError: If the `secret_format` is not defined.
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
 
         Returns:
-            The JSON schema of the secret format as a string.
+            The JSON schema of the secret format as a JSON string.
+
+        Raises:
+            NotImplementedError: If the `secret_format` class attribute is not defined.
         """
         if cls.secret_format is not None:
             if llm_format:
-                return json.dumps(llm_ready_schema(cls.secret_format), indent=2)
+                result_json, result_ui = SchemaSplitter.split(cls.secret_format.model_json_schema())
+                return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
             return json.dumps(cls.secret_format.model_json_schema(), indent=2)
         msg = f"{cls.__name__}' class does not define a 'secret_format'."
         raise NotImplementedError(msg)
 
     @classmethod
-    def get_input_format(cls, *, llm_format: bool) -> str:
+    async def get_input_format(cls, *, llm_format: bool) -> str:
         """Get the JSON schema of the input format model.
 
-        Raises:
-            NotImplementedError: If the `input_format` is not defined.
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
 
         Returns:
-            The JSON schema of the input format as a string.
+            The JSON schema of the input format as a JSON string.
+
+        Raises:
+            NotImplementedError: If the `input_format` class attribute is not defined.
         """
-        if cls.input_format is not None:
-            if llm_format:
-                return json.dumps(llm_ready_schema(cls.input_format), indent=2)
-            return json.dumps(cls.input_format.model_json_schema(), indent=2)
-        msg = f"{cls.__name__}' class does not define an 'input_format'."
-        raise NotImplementedError(msg)
+        if cls.input_format is None:
+            msg = f"{cls.__name__}' class does not define an 'input_format'."
+            raise NotImplementedError(msg)
+
+        extended_model = UtilitySchemaExtender.create_extended_input_model(cls.input_format)
+
+        if llm_format:
+            result_json, result_ui = SchemaSplitter.split(extended_model.model_json_schema())
+            return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
+        return json.dumps(extended_model.model_json_schema(), indent=2)
 
     @classmethod
-    def get_output_format(cls, *, llm_format: bool) -> str:
+    async def get_output_format(cls, *, llm_format: bool) -> str:
         """Get the JSON schema of the output format model.
 
-        Raises:
-            NotImplementedError: If the `output_format` is not defined.
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
 
         Returns:
-            The JSON schema of the output format as a string.
+            The JSON schema of the output format as a JSON string.
+
+        Raises:
+            NotImplementedError: If the `output_format` class attribute is not defined.
         """
-        if cls.output_format is not None:
-            if llm_format:
-                return json.dumps(llm_ready_schema(cls.output_format), indent=2)
-            return json.dumps(cls.output_format.model_json_schema(), indent=2)
-        msg = "'%s' class does not define an 'output_format'."
-        raise NotImplementedError(msg)
+        if cls.output_format is None:
+            msg = f"'{cls.__name__}' class does not define an 'output_format'."
+            raise NotImplementedError(msg)
+
+        extended_model = UtilitySchemaExtender.create_extended_output_model(cls.output_format)
+
+        if llm_format:
+            result_json, result_ui = SchemaSplitter.split(extended_model.model_json_schema())
+            return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
+        return json.dumps(extended_model.model_json_schema(), indent=2)
 
     @classmethod
-    def get_config_setup_format(cls, *, llm_format: bool) -> str:
+    async def get_config_setup_format(cls, *, llm_format: bool) -> str:
         """Gets the JSON schema of the config setup format model.
 
-        The config setup format is used only to initialize the module with configuration data.
-        The setup format is used to initialize an run the module with setup data.
+        The config setup format is used only to initialize the module with configuration
+        data. It includes fields marked with `json_schema_extra={"config": True}` and
+        excludes hidden runtime fields.
 
-        Raises:
-            NotImplementedError: If the `setup_format` is not defined.
+        Dynamic schema fields are always resolved when generating the schema, as this
+        method is typically called during module discovery or schema generation where
+        fresh values are needed.
+
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
 
         Returns:
-            The JSON schema of the config setup format as a string.
+            The JSON schema of the config setup format as a JSON string.
+
+        Raises:
+            NotImplementedError: If the `setup_format` class attribute is not defined.
         """
         if cls.setup_format is not None:
-            setup_format = cls.setup_format.get_clean_model(config_fields=True, hidden_fields=False)
+            setup_format = await cls.setup_format.get_clean_model(config_fields=True, hidden_fields=False, force=True)
             if llm_format:
-                return json.dumps(llm_ready_schema(setup_format), indent=2)
+                result_json, result_ui = SchemaSplitter.split(setup_format.model_json_schema())
+                return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
             return json.dumps(setup_format.model_json_schema(), indent=2)
         msg = "'%s' class does not define an 'config_setup_format'."
         raise NotImplementedError(msg)
 
     @classmethod
-    def get_setup_format(cls, *, llm_format: bool) -> str:
+    async def get_setup_format(cls, *, llm_format: bool) -> str:
         """Gets the JSON schema of the setup format model.
 
-        Raises:
-            NotImplementedError: If the `setup_format` is not defined.
+        The setup format is used at runtime and includes hidden fields but excludes
+        config-only fields. This is the schema used when running the module.
+
+        Dynamic schema fields are always resolved when generating the schema, as this
+        method is typically called during module discovery or schema generation where
+        fresh values are needed.
+
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
 
         Returns:
-            The JSON schema of the setup format as a string.
+            The JSON schema of the setup format as a JSON string.
+
+        Raises:
+            NotImplementedError: If the `setup_format` class attribute is not defined.
         """
         if cls.setup_format is not None:
-            setup_format = cls.setup_format.get_clean_model(config_fields=False, hidden_fields=True)
+            setup_format = await cls.setup_format.get_clean_model(config_fields=False, hidden_fields=True, force=True)
             if llm_format:
-                return json.dumps(llm_ready_schema(setup_format), indent=2)
+                result_json, result_ui = SchemaSplitter.split(setup_format.model_json_schema())
+                return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
             return json.dumps(setup_format.model_json_schema(), indent=2)
         msg = "'%s' class does not define an 'setup_format'."
         raise NotImplementedError(msg)
 
+    @classmethod
+    async def get_cost_format(cls, *, llm_format: bool) -> str:
+        """Get the JSON schema of the cost configuration.
+
+        Extracts CostConfig from services_config_params["cost"]["config"]
+        and returns as JSON schema.
+
+        Args:
+            llm_format: If True, return LLM-optimized schema format with inlined
+                references and simplified structure.
+
+        Returns:
+            The JSON schema of the cost configuration as a JSON string.
+        """
+        cost_params = cls.services_config_params.get("cost", {})
+        config = cost_params.get("config", {}) if cost_params else {}
+
+        if not config:
+            return json.dumps({}, indent=2)
+
+        # Convert CostConfig objects to serializable dict
+        cost_schema = {
+            name: {
+                "name": cost_config.name,
+                "type": cost_config.type.value if hasattr(cost_config.type, "value") else cost_config.type,
+                "description": cost_config.description,
+                "unit": cost_config.unit,
+                "rate": cost_config.rate,
+            }
+            for name, cost_config in config.items()
+        }
+
+        if llm_format:
+            result_json, result_ui = SchemaSplitter.split({"costs": cost_schema})
+            return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
+        return json.dumps(cost_schema, indent=2)
+
     @classmethod
     def create_config_setup_model(cls, config_setup_data: dict[str, Any]) -> SetupModelT:
         """Create the setup model from the setup data.
@@ -221,17 +316,22 @@ class BaseModule(  # noqa: PLR0904
         return cls.input_format(**input_data)
 
     @classmethod
-    def create_setup_model(cls, setup_data: dict[str, Any], *, config_fields: bool = False) -> SetupModelT:
+    async def create_setup_model(cls, setup_data: dict[str, Any], *, config_fields: bool = False) -> SetupModelT:
         """Create the setup model from the setup data.
 
+        Creates a filtered setup model instance based on the provided data.
+        Uses `get_clean_model()` internally to get the appropriate model class
+        with field filtering applied.
+
         Args:
             setup_data: The setup data to create the model from.
             config_fields: If True, include only fields with json_schema_extra["config"] == True.
 
         Returns:
-            The setup model.
+            An instance of the setup model with the provided data.
         """
-        return cls.setup_format.get_clean_model(config_fields=config_fields, hidden_fields=True)(**setup_data)
+        model_cls = await cls.setup_format.get_clean_model(config_fields=config_fields, hidden_fields=True)
+        return model_cls(**setup_data)
 
     @classmethod
     def create_secret_model(cls, secret_data: dict[str, Any]) -> SecretModelT:
@@ -267,8 +367,18 @@ class BaseModule(  # noqa: PLR0904
         If a package is provided, all .py files within its path are imported; otherwise, the current
         working directory is searched. For each imported module, any class matching the criteria is
         registered via cls.register(). Errors during import are logged at debug level.
+
+        Built-in healthcheck handlers (ping, services, status) are automatically registered
+        to provide standard healthcheck functionality for all modules.
         """
+        from digitalkin.models.module.utility import UtilityRegistry  # noqa: PLC0415
+
         cls.triggers_discoverer.discover_modules()
+
+        # Auto-register built-in SDK triggers (healthcheck, etc.)
+        for trigger_cls in UtilityRegistry.get_builtin_triggers():
+            cls.triggers_discoverer.register_trigger(trigger_cls)
+
         logger.debug("discovered: %s", cls.triggers_discoverer)
 
     @classmethod
@@ -283,25 +393,6 @@ class BaseModule(  # noqa: PLR0904
         """
         return cls.triggers_discoverer.register_trigger(handler_cls)
 
-    async def run_config_setup(  # noqa: PLR6301
-        self,
-        context: ModuleContext,  # noqa: ARG002
-        config_setup_data: SetupModelT,
-    ) -> SetupModelT:
-        """Run config setup the module.
-
-        The config setup is used to initialize the setup with configuration data.
-        This method is typically used to set up the module with necessary configuration before running it,
-        especially for processing data like files.
-        The function needs to save the setup in the storage.
-        The module will be initialize with the setup and not the config setup.
-        This method is optional, the config setup and setup can be the same.
-
-        Returns:
-            The updated setup model after running the config setup.
-        """
-        return config_setup_data
-
     @abstractmethod
     async def initialize(self, context: ModuleContext, setup_data: SetupModelT) -> None:
         """Initialize the module."""
@@ -312,24 +403,22 @@ class BaseModule(  # noqa: PLR0904
         input_data: InputModelT,
         setup_data: SetupModelT,
     ) -> None:
-        """Run the module with the given input and setup data.
-
-        This method validates the input data, determines the protocol from the input,
-        and dispatches the request to the corresponding trigger handler. The trigger handler
-        is responsible for processing the input and invoking the callback with the result.
-
-        Triggers:
-            - The method is triggered when a module run is requested with specific input and setup data.
-            - The protocol specified in the input determines which trigger handler is invoked.
+        """Run the module by dispatching to the appropriate trigger handler.
 
         Args:
-            input_data (InputModelT): The input data to be processed by the module.
-            setup_data (SetupModelT): The setup or configuration data required for the module.
+            input_data: Input data to process.
+            setup_data: Configuration data for the module.
 
         Raises:
             ValueError: If no handler for the protocol is found.
         """
         input_instance = self.input_format.model_validate(input_data)
+
+        # Apply cost limits if present in input
+        cost_limits = getattr(input_instance, "cost_limits", None)
+        if cost_limits is not None and self.context.cost is not None:
+            self.context.cost.set_limits(cost_limits)
+
         handler_instance = self.triggers_discoverer.get_trigger(
             input_instance.root.protocol,
             input_instance.root,
@@ -346,6 +435,25 @@ class BaseModule(  # noqa: PLR0904
         """Run the module."""
         raise NotImplementedError
 
+    async def run_config_setup(  # noqa: PLR6301
+        self,
+        context: ModuleContext,  # noqa: ARG002
+        config_setup_data: SetupModelT,
+    ) -> SetupModelT:
+        """Run config setup the module.
+
+        The config setup is used to initialize the setup with configuration data.
+        This method is typically used to set up the module with necessary configuration before running it,
+        especially for processing data like files.
+        The function needs to save the setup in the storage.
+        The module will be initialize with the setup and not the config setup.
+        This method is optional, the config setup and setup can be the same.
+
+        Returns:
+            The updated setup model after running the config setup.
+        """
+        return config_setup_data
+
     async def _run_lifecycle(
         self,
         input_data: InputModelT,
@@ -373,13 +481,32 @@ class BaseModule(  # noqa: PLR0904
         self,
         input_data: InputModelT,
         setup_data: SetupModelT,
-        callback: Callable[[OutputModelT | ModuleCodeModel], Coroutine[Any, Any, None]],
+        callback: Callable[[OutputModelT | ModuleCodeModel | DataModel[UtilityProtocol]], Coroutine[Any, Any, None]],
         done_callback: Callable | None = None,
     ) -> None:
         """Start the module."""
         try:
             self.context.callbacks.send_message = callback
-            logger.info(f"Inititalize module {self.context.session.job_id}")
+
+            tool_cache = setup_data.build_tool_cache()
+            if tool_cache.entries:
+                self.context.tool_cache = tool_cache
+
+            await callback(
+                DataModel(
+                    root=ModuleStartInfoOutput(
+                        job_id=self.context.session.job_id,
+                        mission_id=self.context.session.mission_id,
+                        setup_id=self.context.session.setup_id,
+                        setup_version_id=self.context.session.setup_version_id,
+                        module_id=self.get_module_id(),
+                        module_name=self.name,
+                    ),
+                    annotations={"role": BaseRole.SYSTEM},
+                )
+            )
+
+            logger.info("Initialize module %s", self.context.session.job_id)
             await self.initialize(self.context, setup_data)
         except Exception as e:
             self._status = ModuleStatus.FAILED
@@ -400,7 +527,7 @@ class BaseModule(  # noqa: PLR0904
         try:
             logger.debug("Init the discovered input handlers.")
             self.triggers_discoverer.init_handlers(self.context)
-            logger.debug(f"Run lifecycle {self.context.session.job_id}")
+            logger.debug("Run lifecycle %s", self.context.session.job_id)
             await self._run_lifecycle(input_data, setup_data)
         except Exception:
             self._status = ModuleStatus.FAILED
@@ -415,30 +542,66 @@ class BaseModule(  # noqa: PLR0904
             self._status = ModuleStatus.STOPPING
             logger.debug("Module %s stopped", self.name)
             await self.cleanup()
-            await self.context.callbacks.send_message(ModuleCodeModel(code="__END_OF_STREAM__"))
+            await self.context.callbacks.send_message(
+                DataModel(
+                    root=EndOfStreamOutput(),
+                    annotations={"role": BaseRole.SYSTEM},
+                )
+            )
             self._status = ModuleStatus.STOPPED
             logger.debug("Module %s cleaned", self.name)
         except Exception:
             self._status = ModuleStatus.FAILED
             logger.exception("Error stopping module")
 
+    async def _resolve_tools(self, config_setup_data: SetupModelT) -> None:
+        """Resolve tool references and build cache.
+
+        Args:
+            config_setup_data: Setup data containing tool references.
+        """
+        logger.info("Starting tool resolution", extra=self.context.session.current_ids())
+        if self.context.registry is not None and self.context.communication is not None:
+            await config_setup_data.resolve_tool_references(self.context.registry, self.context.communication)
+            logger.info("Tool references resolved", extra=self.context.session.current_ids())
+        else:
+            logger.warning(
+                "No registry or communication available, skipping tool resolution",
+                extra=self.context.session.current_ids(),
+            )
+
+        tool_cache = config_setup_data.build_tool_cache()
+        self.context.tool_cache = tool_cache
+        logger.info(
+            "Tool cache built with %d entries: %s",
+            len(tool_cache.entries),
+            list(tool_cache.entries.keys()),
+            extra=self.context.session.current_ids(),
+        )
+
     async def start_config_setup(
         self,
         config_setup_data: SetupModelT,
         callback: Callable[[SetupModelT | ModuleCodeModel], Coroutine[Any, Any, None]],
     ) -> None:
-        """Start the module."""
+        """Run config setup lifecycle with tool resolution in parallel.
+
+        Args:
+            config_setup_data: Initial setup data to configure.
+            callback: Callback to send the configured setup model.
+        """
         try:
             logger.info("Run Config Setup lifecycle", extra=self.context.session.current_ids())
             self._status = ModuleStatus.RUNNING
             self.context.callbacks.set_config_setup = callback
-            content = await self.run_config_setup(self.context, config_setup_data)
 
-            wrapper = config_setup_data.model_dump()
-            wrapper["content"] = content.model_dump()
-            await callback(self.create_setup_model(wrapper))
+            # Resolve tools first to populate companion fields, then run config setup
+            await self._resolve_tools(config_setup_data)
+            updated_config = await self.run_config_setup(self.context, config_setup_data)
+
+            setup_model = await self.create_setup_model(updated_config.model_dump())
+            await callback(setup_model)
             self._status = ModuleStatus.STOPPING
         except Exception:
-            logger.error("Error during module lifecyle")
             self._status = ModuleStatus.FAILED
-            logger.exception("Error during module lifecyle", extra=self.context.session.current_ids())
+            logger.exception("Error during config setup lifecycle", extra=self.context.session.current_ids())

digitalkin/modules/archetype_module.py

@@ -2,7 +2,12 @@
 
 from abc import ABC
 
-from digitalkin.models.module import InputModelT, OutputModelT, SecretModelT, SetupModelT
+from digitalkin.models.module.module_types import (
+    InputModelT,
+    OutputModelT,
+    SecretModelT,
+    SetupModelT,
+)
 from digitalkin.modules._base_module import BaseModule
 
 

digitalkin/modules/tool_module.py

@@ -2,7 +2,12 @@
 
 from abc import ABC
 
-from digitalkin.models.module import InputModelT, OutputModelT, SecretModelT, SetupModelT
+from digitalkin.models.module.module_types import (
+    InputModelT,
+    OutputModelT,
+    SecretModelT,
+    SetupModelT,
+)
 from digitalkin.modules._base_module import BaseModule  # type: ignore
 
 

digitalkin/modules/triggers/__init__.py

@@ -0,0 +1,8 @@
+"""Built-in SDK triggers.
+
+These triggers are automatically registered without requiring discovery.
+They provide standard functionality available to all modules.
+
+Note: These are internal triggers. External code should not import them directly.
+Use UtilityRegistry.get_builtin_triggers() to access the trigger classes.
+"""

digitalkin/modules/triggers/healthcheck_ping_trigger.py

@@ -0,0 +1,45 @@
+"""Healthcheck ping trigger - simple alive check."""
+
+from datetime import datetime
+from typing import Any, ClassVar
+
+from digitalkin.mixins import BaseMixin
+from digitalkin.models.module.module_context import ModuleContext
+from digitalkin.models.module.utility import (
+    HealthcheckPingInput,
+    HealthcheckPingOutput,
+)
+from digitalkin.modules.trigger_handler import TriggerHandler
+
+
+class HealthcheckPingTrigger(TriggerHandler, BaseMixin):
+    """Handler for simple ping healthcheck.
+
+    Responds immediately with "pong" status to verify the module is responsive.
+    """
+
+    protocol: ClassVar[str] = "healthcheck_ping"
+    input_format = HealthcheckPingInput
+    _request_time: datetime
+
+    def __init__(self, context: ModuleContext) -> None:
+        """Initialize the handler."""
+        self._request_time = datetime.now(tz=context.session.timezone)
+
+    async def handle(
+        self,
+        input_data: HealthcheckPingInput,  # noqa: ARG002
+        setup_data: Any,  # noqa: ANN401, ARG002
+        context: ModuleContext,
+    ) -> None:
+        """Handle ping healthcheck request.
+
+        Args:
+            input_data: The input trigger data (unused for healthcheck).
+            setup_data: The setup configuration (unused for healthcheck).
+            context: The module context.
+        """
+        elapsed = datetime.now(tz=context.session.timezone) - self._request_time
+        latency_ms = elapsed.total_seconds() * 1000
+        output = HealthcheckPingOutput(latency_ms=latency_ms)
+        await self.send_message(context, output)