digitalkin 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

modules/storage_module.py

@@ -0,0 +1,187 @@
+"""Example module implementation to test ArchetypeModule functionality."""
+
+import asyncio
+import datetime
+from collections.abc import Callable
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from digitalkin.logger import logger
+from digitalkin.models.module import ModuleStatus
+from digitalkin.modules.archetype_module import ArchetypeModule
+from digitalkin.services.services_config import ServicesConfig
+from digitalkin.services.services_models import ServicesMode
+from digitalkin.services.storage.storage_strategy import DataType, StorageRecord
+
+
+class ExampleInput(BaseModel):
+    """Input model for example module."""
+
+    message: str = Field(description="Message to process")
+    number: int = Field(description="Number to process")
+
+
+class ExampleOutput(BaseModel):
+    """Output model for example module."""
+
+    processed_message: str = Field(description="The processed message")
+    processed_number: int = Field(description="The processed number")
+    timestamp: datetime.datetime = Field(description="When the processing was done")
+
+
+class ExampleSetup(BaseModel):
+    """Setup model for example module."""
+
+    processing_mode: str = Field(description="Mode to process data in", default="default")
+    multiply_factor: int = Field(description="Factor to multiply number by", default=1)
+
+
+class ExampleSecret(BaseModel):
+    """Secret model for example module."""
+
+    api_key: str = Field(description="API key for external service")
+
+
+class ExampleStorage(BaseModel):
+    """Secret model for example module."""
+
+    test_key: str = Field(description="Test value for storage")
+
+
+class ExampleModule(ArchetypeModule[ExampleInput, ExampleOutput, ExampleSetup, ExampleSecret]):
+    """Example module that demonstrates ArchetypeModule functionality."""
+
+    name = "ExampleModule"
+    description = "An example module for testing purposes"
+    input_format = ExampleInput
+    output_format = ExampleOutput
+    setup_format = ExampleSetup
+    secret_format = ExampleSecret
+    metadata = {"name": "ExampleModule", "description": "A module for testing ArchetypeModule functionality"}
+
+    # Define services_config_params with default values
+    services_config_strategies = {}
+    services_config_params = {"storage": {"config": {"example_outputs": ExampleOutput}}}
+
+    def __init__(self, job_id: str, mission_id: str) -> None:
+        """Initialize the example module.
+
+        Args:
+            job_id: Unique identifier for the job
+            name: Optional name for the module
+        """
+        # Initialize services configuration using the class attribute before the instance is created
+        self.services_config = ServicesConfig(
+            services_config_strategies=self.services_config_strategies,
+            services_config_params=self.services_config_params,
+            mode=ServicesMode.LOCAL,
+        )
+
+        super().__init__(job_id, mission_id)
+
+    async def initialize(self, setup_data: ExampleSetup) -> None:
+        """Initialize the module.
+
+        Args:
+            setup_data: Setup data for the module
+        """
+        logger.info("Initializing ExampleModule with setup data: %s", setup_data)
+        self.setup = self.setup_format.model_validate(setup_data)
+        logger.info("Initialization complete, using processing mode: [%s]", self.setup.processing_mode)
+
+    async def run(
+        self,
+        input_data: dict[str, Any],
+        setup_data: ExampleSetup,
+        callback: Callable,
+    ) -> None:
+        """Run the module.
+
+        Args:
+            input_data: Input data for the module
+            setup_data: Setup data for the module
+            callback: Callback function to report progress
+        """
+        # Validate the input data
+        input_model = self.input_format.model_validate(input_data)
+        logger.info("Running with input data: %s", input_model)
+
+        # Process the data
+        processed_message = f"Processed: {input_model.message}"
+        processed_number = input_model.number * self.setup.multiply_factor
+
+        # Create output model
+        output_data = self.output_format(
+            processed_message=processed_message,
+            processed_number=processed_number,
+            timestamp=datetime.datetime.now(),
+        )
+
+        # Store the output data in storage
+        storage_id = self.storage.store("example_outputs", output_data.model_dump(), data_type=DataType.OUTPUT)
+
+        logger.info("Stored output data with ID: %s", storage_id)
+
+        # Call the callback with the output data
+        callback(output_data.model_dump())
+
+        # Wait a bit to simulate processing time
+        await asyncio.sleep(1)
+
+    async def cleanup(self) -> None:
+        """Clean up the module."""
+        logger.info("Cleaning up ExampleModule")
+        # Nothing to clean up in this example
+
+
+async def test_module() -> None:
+    """Test the example module."""
+    # Create the module
+    module = ExampleModule(job_id="test-job-123", mission_id="test-mission-123")
+
+    # Define input and setup data
+    input_data = ExampleInput(message="Hello, world!", number=42)
+
+    setup_data = ExampleSetup(processing_mode="test", multiply_factor=10)
+
+    # Define a callback function
+    def callback(result) -> None:
+        for key, value in result.items():
+            pass
+
+    # Start the module
+    await module.start(input_data, setup_data, callback)
+
+    # Wait for the module to complete
+    while module.status not in {ModuleStatus.STOPPED, ModuleStatus.FAILED}:
+        await asyncio.sleep(0.5)
+
+    # Check the storage
+    if module.status == ModuleStatus.STOPPED:
+        result: StorageRecord = module.storage.read("example_outputs")
+        if result:
+            pass
+
+
+def test_storage_directly() -> None:
+    """Test the storage service directly."""
+    # Initialize storage service
+    storage = ServicesConfig().storage(mission_id="test-mission", config={"test_table": ExampleStorage})
+
+    # Create a test record
+    storage.store("test_table", {"test_key": "test_value"}, DataType.OUTPUT)
+
+    # Retrieve the record
+    retrieved = storage.read("test_table")
+
+    if retrieved:
+        pass
+
+
+if __name__ == "__main__":
+    # Run the module test
+    asyncio.run(test_module())
+
+    # Test storage directly
+    test_storage_directly()

modules/text_transform_module.py

@@ -0,0 +1,201 @@
+"""Simple module example transforming a text."""
+
+import logging
+from collections.abc import Callable
+from typing import Any, ClassVar
+
+from pydantic import BaseModel
+
+from digitalkin.grpc_servers.utils.models import SecurityMode, ServerConfig, ServerMode
+from digitalkin.modules._base_module import BaseModule
+from digitalkin.services.setup.setup_strategy import SetupData
+from digitalkin.services.storage.storage_strategy import DataType, StorageRecord
+
+# Configure logging with clear formatting
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+
+# Define schema models using Pydantic
+class TextTransformInput(BaseModel):
+    """Input model defining what data the module expects."""
+
+    text: str
+    transform_count: int = 1  # Default to 1 transformation
+
+
+class TextTransformOutput(BaseModel):
+    """Output model defining what data the module produces."""
+
+    transformed_text: str
+    iteration: int  # Tracks which transformation this is
+
+
+class TextTransformSetup(BaseModel):
+    """Setup model defining module configuration parameters."""
+
+    shift_amount: int = 1  # Default Caesar shift by 1
+    uppercase: bool = False  # Whether to convert to uppercase
+
+
+class TextTransformSecret(BaseModel):
+    """Secret model defining module configuration parameters."""
+
+
+class TextTransformStorage(BaseModel):
+    """Secret model defining module configuration parameters."""
+
+    module: str = "Text_Transform_Module"
+    user: str = "user"
+    consumption: int = 0
+    ended: bool = False
+
+
+server_config = ServerConfig(
+    host="[::]",
+    port=50151,
+    mode=ServerMode.ASYNC,
+    security=SecurityMode.INSECURE,
+    max_workers=10,
+    credentials=None,
+)
+
+
+class TextTransformModule(BaseModule[TextTransformInput, TextTransformOutput, TextTransformSetup, TextTransformSecret]):
+    """A text transformation module that demonstrates streaming capabilities.
+
+    This module takes text input and performs multiple transformations on it,
+    sending back each transformation as a separate output message.
+    """
+
+    # Define the schema formats for the module
+    name = "Text_Transform_Module"
+    input_format = TextTransformInput
+    output_format = TextTransformOutput
+    setup_format = TextTransformSetup
+    secret_format = TextTransformSecret
+
+    # Define module metadata for discovery
+    metadata: ClassVar[dict[str, Any]] = {
+        "name": "Text_Transform_Module",
+        "description": "Transforms input text using Caesar cipher with streaming output",
+        "version": "1.0.0",
+        "tags": ["text", "transformation", "encryption", "streaming"],
+    }
+
+    # Define services_config_params with default values
+    services_config_strategies = {}
+    services_config_params = {
+        "storage": {
+            "config": {"monitor": TextTransformStorage, "setups": TextTransformStorage},
+            "server_config": server_config,
+        }
+    }
+
+    async def initialize(self, setup_data: SetupData) -> None:
+        """Initialize the module capabilities.
+
+        This method is called when the module is loaded by the server.
+        Use it to set up module-specific resources or configurations.
+
+        Raises:
+            Exception: If initialization fails.
+        """
+        # Define what capabilities this module provides
+        self.capabilities = ["text-processing", "streaming", "transformation"]
+        logger.info(
+            "Module %s initialized with capabilities: %s",
+            self.metadata["name"],
+            self.capabilities,
+        )
+
+        self.db_id = self.storage.store(
+            "monitor",
+            {
+                "module": self.metadata["name"],
+                "user": f"xxxx+{self.job_id}",
+                "consumption": 0,
+                "ended": False,
+            },
+            data_type=DataType.VIEW,
+        )
+
+    async def run(
+        self,
+        input_data: dict[str, Any],
+        setup_data: SetupData,
+        callback: Callable,
+    ) -> None:
+        """Process input text and stream transformation results.
+
+        This method implements a Caesar cipher transformation on input text.
+        It demonstrates streaming capability by sending multiple outputs through
+        the callback for each transformation iteration.
+
+        Args:
+            input_data: Contains the text to transform and number of iterations.
+            setup_data: Contains shift amount and uppercase flags.
+            callback: Function to send output data back to the client.
+        """
+        text = input_data["text"]
+        transform_count = int(input_data["transform_count"])
+        logger.info("%s  | %s", setup_data, type(setup_data))
+        shift_amount = int(setup_data.current_setup_version.content["shift_amount"])
+        uppercase = setup_data.current_setup_version.content["uppercase"]
+
+        logger.info(
+            "Running job %s with text: '%s', iterations: %s",
+            self.job_id,
+            text,
+            transform_count,
+        )
+
+        # Process the text for each iteration
+        for i in range(transform_count):
+            # Apply Caesar cipher (shift each character by specified amount)
+            transformed = "".join([chr(ord(char) + shift_amount) if char.isalpha() else char for char in text])
+
+            # Apply uppercase transformation if configured
+            if uppercase:
+                transformed = transformed.upper()
+
+            output_data = TextTransformOutput(transformed_text=transformed, iteration=i + 1)
+
+            logger.info(
+                "Sending transformation %s/%s: '%s'",
+                i + 1,
+                transform_count,
+                transformed,
+            )
+
+            monitor_obj: StorageRecord | None = self.storage.read("monitor")
+            if monitor_obj is None:
+                logger.error("Monitor object not found in storage.")
+                break
+            monitor_obj.data.consumption += 1
+            updated_monitor_obj: StorageRecord | None = self.storage.modify("monitor", monitor_obj.data.model_dump())
+            self.db_id = updated_monitor_obj.name if updated_monitor_obj else "monitor"
+
+            # Send results through callback and wait for acknowledgment
+            await callback(job_id=self.job_id, output_data=output_data.model_dump())
+            text = transformed
+
+        logger.info("Job %s completed with %s transformations", self.job_id, transform_count)
+
+    async def cleanup(self) -> None:
+        """Clean up any resources when the module is stopped.
+
+        This method is called when the module is being shut down.
+        Use it to close connections, free resources, etc.
+        """
+        logger.info(f"Cleaning up module {self.metadata['name']}")
+        monitor_obj = self.storage.read("monitor")
+        if monitor_obj is None:
+            logger.error("Monitor object not found in storage.")
+            return
+        monitor_obj.data.ended = True
+        updated_monitor_obj: StorageRecord | None = self.storage.modify("monitor", monitor_obj.data.model_dump())
+        self.db_id = updated_monitor_obj.name if updated_monitor_obj else "monitor"

digitalkin/services/default_service.py

@@ -1,13 +0,0 @@
-"""Default entrypoint for a DigitalKin module's services."""
-
-from digitalkin.services.service_provider import ServiceProvider
-from digitalkin.services.storage.default_storage import DefaultStorage
-
-
-class DefaultServiceProvider(ServiceProvider):
-    """Service Instance used as default service in a Module.
-
-    Currently only allow the default (local) database service.
-    """
-
-    storage = DefaultStorage()

digitalkin/services/development_service.py

@@ -1,10 +0,0 @@
-"""Dev entrypoint for a DigitalKin module's services."""
-
-from digitalkin.services.service_provider import ServiceProvider
-from digitalkin.services.storage.grpc_storage import GrpcStorage
-
-
-class DevelopmentServiceProvider(ServiceProvider):
-    """Service Instance used as a development service in a Module."""
-
-    storage = GrpcStorage()

digitalkin/services/service_provider.py

@@ -1,27 +0,0 @@
-"""Service Provider definitions."""
-
-from typing import ClassVar
-
-from pydantic import BaseModel
-
-from digitalkin.services.agent.agent_strategy import AgentStrategy
-from digitalkin.services.cost.cost_strategy import CostStrategy
-from digitalkin.services.filesystem.filesystem_strategy import FilesystemStrategy
-from digitalkin.services.identity.identity_strategy import IdentityStrategy
-from digitalkin.services.registry.registry_strategy import RegistryStrategy
-from digitalkin.services.snapshot.snapshot_strategy import SnapshotStrategy
-from digitalkin.services.storage.storage_strategy import StorageStrategy
-
-
-class ServiceProvider(BaseModel):
-    """Service class describing the available services in a Module."""
-
-    storage: ClassVar[StorageStrategy]
-    cost: ClassVar[CostStrategy]
-    snapshot: ClassVar[SnapshotStrategy]
-    registry: ClassVar[RegistryStrategy]
-    filesystem: ClassVar[FilesystemStrategy]
-    agent: ClassVar[AgentStrategy]
-    identity: ClassVar[IdentityStrategy]
-
-    model_config = {"arbitrary_types_allowed": True}

digitalkin-0.1.1.dist-info/RECORD

@@ -1,59 +0,0 @@
-digitalkin/__init__.py,sha256=-lXi1_j09LqoGKb5G5yYE1Px-GCv3e01GVdlR_HBYGU,425
-digitalkin/__version__.py,sha256=ufsk7-Hv_LuYOeJWhJ5_b03iJSh5GN2IT8SJDyFWjXM,274
-digitalkin/logger.py,sha256=9cDgyJV2QXXT8F--xRODFlZyDgjuTTXNdpCU3GdqCsk,382
-digitalkin/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-digitalkin/grpc/__init__.py,sha256=K4RLuuZmB4sGVHmpH7MyYvx0UTaEz0C70xUVX-zYS48,1050
-digitalkin/grpc/_base_server.py,sha256=Y3JYo_bVqXhYr3ZRcgkD43_7sG0WmLZ5_g2GeXAYNr4,18625
-digitalkin/grpc/module_server.py,sha256=nUizlFI03bs6OTy4dHsV9GbPhDSquyxRABQbtkUw-yA,8795
-digitalkin/grpc/module_servicer.py,sha256=ocdNI1fng4DgPd_LhQg2VtBLfXwR34Txd7oQ2nwjR0Q,10918
-digitalkin/grpc/registry_server.py,sha256=HGm3bd43Q9bMN_FJY37IhOTSEH9iVeZZTlfkGx7296M,2087
-digitalkin/grpc/registry_servicer.py,sha256=9wtdWNsQzn3QOWC_ZkHb-09Tzzd-4tZkc5JrFhwBafc,16397
-digitalkin/grpc/utils/exceptions.py,sha256=I00OM8p8up20He4dU1fiHsvdLj1DymjR_UmoeUm2MSA,785
-digitalkin/grpc/utils/factory.py,sha256=RC8zh-y-Yq54G27Kn2UfgYalBeR0l_asYHPpU5yzoG8,5845
-digitalkin/grpc/utils/models.py,sha256=N9yK3rHUCllh9uHkz17iv0SwF3bjOJ4RDPYkxCBQph0,5475
-digitalkin/grpc/utils/types.py,sha256=rQ78s4nAet2jy-NIDj_PUWriT0kuGHr_w6ELjmjgBao,539
-digitalkin/models/__init__.py,sha256=-j1DI6VklZXYn3gv7lFrR7HKxLKM3FGlqlE7Y4JVHrw,229
-digitalkin/models/module/__init__.py,sha256=8Nd43spo0O3p3GSnWxzhnL9HOjHNY_6C-tKLv6MZMP4,135
-digitalkin/models/module/module.py,sha256=vlIeWmH61NVEiJpRx0Crk4iB8hAUV4mG30likSyRFP4,686
-digitalkin/models/services/__init__.py,sha256=xtKP4fYoQmowqWUgF5a35Dx1UPDwBm75C9PreYfXTLE,160
-digitalkin/models/services/cost.py,sha256=QTEuFD6xz62nob0z4ksE-INJWcZ-iFiuNW5mvXhpFes,1599
-digitalkin/models/services/storage.py,sha256=cYTVIriGKiprF9OerhSxmc_jM6fUTVwmeon1yQCinkE,143
-digitalkin/modules/__init__.py,sha256=H-OwxX72K7cCkNVsCZtJc8Qx0rwPywXN8DUpUXCzCsM,224
-digitalkin/modules/_base_module.py,sha256=S6wtWDXllF51wmYAKXNFuEMbisupKu_MPAgEGdYjd1o,5791
-digitalkin/modules/archetype_module.py,sha256=pMa_NaQHd4L_oKCYczM_BIPby5VwSOKkdG6_L09qN8g,487
-digitalkin/modules/job_manager.py,sha256=CPaoueyHHW6c3lkl6M1XGSXWdvndqj4PawxE0nwC1HQ,5408
-digitalkin/modules/tool_module.py,sha256=Dt8ZoDmMZ2Rxf3ZVaWH7i74JUPdLRstMcvW0VA5LgOo,412
-digitalkin/modules/trigger_module.py,sha256=sUWQdrS3LlnOUu_Rfmr7Jvt50AHWKeFyxgWYWTuEMNY,424
-digitalkin/services/__init__.py,sha256=xr58TS7eCqPKIAZr1K71whvYy2tpiCwFKnXcOyQtX4Q,846
-digitalkin/services/default_service.py,sha256=YOU8R4O5kKGBCQ0lUpeS8Ih14kpfAois_Isw6Q1YbTY,412
-digitalkin/services/development_service.py,sha256=IaBTUXrcAPJvN8CSoCxyd2wrHY7MuKvuoNQGfE5NdXU,340
-digitalkin/services/service_provider.py,sha256=ggHR_4uJN9MpI1JJOyeh7UIH0Sz4kMizwR3jtvgv7_A,1049
-digitalkin/services/agent/__init__.py,sha256=m6NyxKWG0-EQkjSyU1VgUsOlPbfj26KG6hpsSsRXJLs,194
-digitalkin/services/agent/agent_strategy.py,sha256=aQ5VJ9SOQnAqPGLraT0jjLHBsF10hc9FHosGCLzWhec,550
-digitalkin/services/agent/default_agent.py,sha256=epqdZB0Tad-pYtyx_Fn2fqGNA_JnvGlhIP9EukqU1T0,397
-digitalkin/services/cost/__init__.py,sha256=4I9cNPIN7Bj6b0vifKu3DYu_O--FpH5e9H1UMJjbypI,187
-digitalkin/services/cost/cost_strategy.py,sha256=hWxQr46fbP67mcNEqQpaT0h4qN9p-o_lCgu02UTb70I,399
-digitalkin/services/cost/default_cost.py,sha256=2DUnDu7_MlaMX1Zwn2MCUpWdKXh7L7npVYC159Eceew,301
-digitalkin/services/filesystem/__init__.py,sha256=vfcC_Aox48qhGg7dfA0hN8YXzOD2nTBGL9fDnfi5sOs,229
-digitalkin/services/filesystem/default_filesystem.py,sha256=EWR8y1MzJNdEOxXF9mvcfSMjUUVAYjthC017VkwKU_A,879
-digitalkin/services/filesystem/filesystem_strategy.py,sha256=zSSe3is3DCCWQGu_SswEt0HGh-JfWtgsAdB3xgxh-y8,927
-digitalkin/services/identity/__init__.py,sha256=D2VBvwkXx0JDEOvVH1dbDxv9G7RUkfbaUxhwAxN4_aA,214
-digitalkin/services/identity/default_identity.py,sha256=cxUb0WKAtmlKlr5edpeMBty7CBctNQcZizi4nnr8YMg,358
-digitalkin/services/identity/identity_strategy.py,sha256=FLZvBvcdfFs1lAwb0SB9AOTX-G1Sa0CfHhXQBlC7vaI,355
-digitalkin/services/registry/__init__.py,sha256=RVwQ7M2Vx1xJpCh5-pAlePlfB5MU1pj3DSpaOtx4A6g,214
-digitalkin/services/registry/default_registry.py,sha256=fF2CVOW52mMFhOA7q4Yb9c9P9GjB6lyEwpezORYPgdE,330
-digitalkin/services/registry/registry_strategy.py,sha256=idSRg_95_fPrPbBT9g7fWby4_9KBPws9l-8ReXj1714,484
-digitalkin/services/snapshot/__init__.py,sha256=awlR_9S64qO4SePQVYHclgAvzHEWyes9jpmFOTw382g,214
-digitalkin/services/snapshot/default_snapshot.py,sha256=XrI3aaOaiVG9WuENVmbCn9w-oRDjOV5Pj8g0YBesVxk,1030
-digitalkin/services/snapshot/snapshot_strategy.py,sha256=C7cPsV7eEqeGhAwjOJTIF3eyFJ00ADVgBBXhaBS6GMM,905
-digitalkin/services/storage/__init__.py,sha256=mb6KS-8lPhh7Go65eVO3KuqugGR5QgRrBynOhfqvWUI,207
-digitalkin/services/storage/default_storage.py,sha256=Brlpmjfie6KVieV-Gv3f0xJ8zRvRiPArkdZ7KWqGXNc,2855
-digitalkin/services/storage/grpc_storage.py,sha256=wK6a9l1Fmc8WVAdKjd21XELm1KqLkGrtt4ziGcNR8sk,6893
-digitalkin/services/storage/storage_strategy.py,sha256=yfEwMV_eeuk3r80-lXHqHOWGW8CR0IOjNJNV0FQlz8o,1300
-digitalkin/utils/__init__.py,sha256=sJnY-ZUgsjMfojAjONC1VN14mhgIDnzyOlGkw21rRnM,28
-digitalkin/utils/arg_parser.py,sha256=RGVXE2L0HwYCqA8ju5KjyOvOHvLvXS4psp8ha-LyLIs,4607
-digitalkin-0.1.1.dist-info/licenses/LICENSE,sha256=Ies4HFv2r2hzDRakJYxk3Y60uDFLiG-orIgeTpstnIo,20327
-digitalkin-0.1.1.dist-info/METADATA,sha256=Swl_Epq2e8Jl4A6TRnVwR381DSA8PUfvNgb-drHNUws,29047
-digitalkin-0.1.1.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-digitalkin-0.1.1.dist-info/top_level.txt,sha256=Fq3vZIKuGrYjfdNOqbKcbVQ4ac-HflvQzVadMhCXEMc,11
-digitalkin-0.1.1.dist-info/RECORD,,

digitalkin-0.1.1.dist-info/top_level.txt

@@ -1 +0,0 @@
-digitalkin