digitalkin 0.3.1.dev2__py3-none-any.whl → 0.3.2.dev14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- base_server/server_async_insecure.py +6 -5
- base_server/server_async_secure.py +6 -5
- base_server/server_sync_insecure.py +5 -4
- base_server/server_sync_secure.py +5 -4
- digitalkin/__version__.py +1 -1
- digitalkin/core/job_manager/base_job_manager.py +1 -1
- digitalkin/core/job_manager/single_job_manager.py +28 -9
- digitalkin/core/job_manager/taskiq_broker.py +7 -6
- digitalkin/core/job_manager/taskiq_job_manager.py +1 -1
- digitalkin/core/task_manager/surrealdb_repository.py +7 -7
- digitalkin/core/task_manager/task_session.py +60 -98
- digitalkin/grpc_servers/module_server.py +109 -168
- digitalkin/grpc_servers/module_servicer.py +38 -16
- digitalkin/grpc_servers/utils/grpc_client_wrapper.py +24 -8
- digitalkin/grpc_servers/utils/utility_schema_extender.py +100 -0
- digitalkin/models/__init__.py +1 -1
- digitalkin/models/core/job_manager_models.py +0 -8
- digitalkin/models/core/task_monitor.py +4 -0
- digitalkin/models/grpc_servers/models.py +91 -6
- digitalkin/models/module/__init__.py +18 -13
- digitalkin/models/module/base_types.py +61 -0
- digitalkin/models/module/module_context.py +173 -13
- digitalkin/models/module/module_types.py +28 -392
- digitalkin/models/module/setup_types.py +490 -0
- digitalkin/models/module/tool_cache.py +68 -0
- digitalkin/models/module/tool_reference.py +117 -0
- digitalkin/models/module/utility.py +167 -0
- digitalkin/models/services/registry.py +35 -0
- digitalkin/modules/__init__.py +5 -1
- digitalkin/modules/_base_module.py +154 -61
- digitalkin/modules/archetype_module.py +6 -1
- digitalkin/modules/tool_module.py +6 -1
- digitalkin/modules/triggers/__init__.py +8 -0
- digitalkin/modules/triggers/healthcheck_ping_trigger.py +45 -0
- digitalkin/modules/triggers/healthcheck_services_trigger.py +63 -0
- digitalkin/modules/triggers/healthcheck_status_trigger.py +52 -0
- digitalkin/services/__init__.py +4 -0
- digitalkin/services/communication/__init__.py +7 -0
- digitalkin/services/communication/communication_strategy.py +76 -0
- digitalkin/services/communication/default_communication.py +101 -0
- digitalkin/services/communication/grpc_communication.py +234 -0
- digitalkin/services/cost/grpc_cost.py +1 -1
- digitalkin/services/filesystem/grpc_filesystem.py +1 -1
- digitalkin/services/registry/__init__.py +22 -1
- digitalkin/services/registry/default_registry.py +135 -4
- digitalkin/services/registry/exceptions.py +47 -0
- digitalkin/services/registry/grpc_registry.py +306 -0
- digitalkin/services/registry/registry_models.py +15 -0
- digitalkin/services/registry/registry_strategy.py +88 -4
- digitalkin/services/services_config.py +25 -3
- digitalkin/services/services_models.py +5 -1
- digitalkin/services/setup/default_setup.py +1 -1
- digitalkin/services/setup/grpc_setup.py +1 -1
- digitalkin/services/storage/grpc_storage.py +1 -1
- digitalkin/services/user_profile/__init__.py +11 -0
- digitalkin/services/user_profile/grpc_user_profile.py +2 -2
- digitalkin/services/user_profile/user_profile_strategy.py +0 -15
- digitalkin/utils/schema_splitter.py +207 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/METADATA +5 -5
- digitalkin-0.3.2.dev14.dist-info/RECORD +143 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/top_level.txt +1 -0
- modules/archetype_with_tools_module.py +244 -0
- modules/cpu_intensive_module.py +1 -1
- modules/dynamic_setup_module.py +5 -29
- modules/minimal_llm_module.py +1 -1
- modules/text_transform_module.py +1 -1
- monitoring/digitalkin_observability/__init__.py +46 -0
- monitoring/digitalkin_observability/http_server.py +150 -0
- monitoring/digitalkin_observability/interceptors.py +176 -0
- monitoring/digitalkin_observability/metrics.py +201 -0
- monitoring/digitalkin_observability/prometheus.py +137 -0
- monitoring/tests/test_metrics.py +172 -0
- services/filesystem_module.py +7 -5
- services/storage_module.py +4 -2
- digitalkin/grpc_servers/registry_server.py +0 -65
- digitalkin/grpc_servers/registry_servicer.py +0 -456
- digitalkin-0.3.1.dev2.dist-info/RECORD +0 -119
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/WHEEL +0 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""Utility protocols for SDK-provided functionality.
|
|
2
|
+
|
|
3
|
+
These protocols are automatically available to all modules and don't need to be
|
|
4
|
+
explicitly included in module output unions.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from typing import Any, ClassVar, Literal
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
from digitalkin.models.module.base_types import DataTrigger
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class UtilityProtocol(DataTrigger):
|
|
16
|
+
"""Base class for SDK-provided utility protocols.
|
|
17
|
+
|
|
18
|
+
All SDK utility protocols inherit from this class to enable:
|
|
19
|
+
- Easy identification of SDK vs user-defined protocols
|
|
20
|
+
- Auto-injection capability
|
|
21
|
+
- Consistent behavior across the SDK
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class EndOfStreamOutput(UtilityProtocol):
|
|
26
|
+
"""Signal that the stream has ended."""
|
|
27
|
+
|
|
28
|
+
protocol: Literal["end_of_stream"] = "end_of_stream" # type: ignore
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class ModuleStartInfoOutput(UtilityProtocol):
|
|
32
|
+
"""Output sent when module starts with execution context.
|
|
33
|
+
|
|
34
|
+
This protocol is sent as the first message when a module starts,
|
|
35
|
+
providing the client with essential execution context information.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
protocol: Literal["module_start_info"] = "module_start_info" # type: ignore
|
|
39
|
+
job_id: str = Field(..., description="Unique job identifier")
|
|
40
|
+
mission_id: str = Field(..., description="Mission identifier")
|
|
41
|
+
setup_id: str = Field(..., description="Setup identifier")
|
|
42
|
+
setup_version_id: str = Field(..., description="Setup version identifier")
|
|
43
|
+
module_id: str = Field(..., description="Module identifier")
|
|
44
|
+
module_name: str = Field(..., description="Human-readable module name")
|
|
45
|
+
started_at: str = Field(
|
|
46
|
+
default_factory=lambda: datetime.now(tz=timezone.utc).isoformat(),
|
|
47
|
+
description="ISO timestamp when module started",
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class HealthcheckPingInput(UtilityProtocol):
|
|
52
|
+
"""Input for healthcheck ping request."""
|
|
53
|
+
|
|
54
|
+
protocol: Literal["healthcheck_ping"] = "healthcheck_ping" # type: ignore
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class HealthcheckPingOutput(UtilityProtocol):
|
|
58
|
+
"""Output for healthcheck ping response.
|
|
59
|
+
|
|
60
|
+
Simple alive check that returns "pong" status.
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
protocol: Literal["healthcheck_ping"] = "healthcheck_ping" # type: ignore
|
|
64
|
+
status: Literal["pong"] = "pong"
|
|
65
|
+
latency_ms: float | None = Field(
|
|
66
|
+
default=None,
|
|
67
|
+
description="Round-trip latency in milliseconds",
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class ServiceHealthStatus(BaseModel):
|
|
72
|
+
"""Health status of a single service."""
|
|
73
|
+
|
|
74
|
+
name: str = Field(..., description="Name of the service")
|
|
75
|
+
status: Literal["healthy", "unhealthy", "unknown"] = Field(
|
|
76
|
+
...,
|
|
77
|
+
description="Health status of the service",
|
|
78
|
+
)
|
|
79
|
+
message: str | None = Field(
|
|
80
|
+
default=None,
|
|
81
|
+
description="Optional message about the service status",
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class HealthcheckServicesInput(UtilityProtocol):
|
|
86
|
+
"""Input for healthcheck services request."""
|
|
87
|
+
|
|
88
|
+
protocol: Literal["healthcheck_services"] = "healthcheck_services" # type: ignore
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class HealthcheckServicesOutput(UtilityProtocol):
|
|
92
|
+
"""Output for healthcheck services response.
|
|
93
|
+
|
|
94
|
+
Reports the health status of all configured services.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
protocol: Literal["healthcheck_services"] = "healthcheck_services" # type: ignore
|
|
98
|
+
services: list[ServiceHealthStatus] = Field(
|
|
99
|
+
...,
|
|
100
|
+
description="List of service health statuses",
|
|
101
|
+
)
|
|
102
|
+
overall_status: Literal["healthy", "degraded", "unhealthy"] = Field(
|
|
103
|
+
...,
|
|
104
|
+
description="Overall health status based on all services",
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class HealthcheckStatusInput(UtilityProtocol):
|
|
109
|
+
"""Input for healthcheck status request."""
|
|
110
|
+
|
|
111
|
+
protocol: Literal["healthcheck_status"] = "healthcheck_status" # type: ignore
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class HealthcheckStatusOutput(UtilityProtocol):
|
|
115
|
+
"""Output for healthcheck status response.
|
|
116
|
+
|
|
117
|
+
Comprehensive module status including uptime, active jobs, and metadata.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
protocol: Literal["healthcheck_status"] = "healthcheck_status" # type: ignore
|
|
121
|
+
module_name: str = Field(..., description="Name of the module")
|
|
122
|
+
module_status: str = Field(..., description="Current status of the module")
|
|
123
|
+
uptime_seconds: float | None = Field(
|
|
124
|
+
default=None,
|
|
125
|
+
description="Module uptime in seconds",
|
|
126
|
+
)
|
|
127
|
+
active_jobs: int = Field(
|
|
128
|
+
default=0,
|
|
129
|
+
description="Number of currently active jobs",
|
|
130
|
+
)
|
|
131
|
+
metadata: dict[str, Any] = Field(
|
|
132
|
+
default_factory=dict,
|
|
133
|
+
description="Additional metadata about the module",
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class UtilityRegistry:
|
|
138
|
+
"""Registry for SDK-provided built-in triggers.
|
|
139
|
+
|
|
140
|
+
Example:
|
|
141
|
+
builtin_triggers = UtilityRegistry.get_builtin_triggers()
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
_builtin_triggers: ClassVar[tuple | None] = None
|
|
145
|
+
|
|
146
|
+
@classmethod
|
|
147
|
+
def get_builtin_triggers(cls) -> tuple:
|
|
148
|
+
"""Get all SDK-provided built-in trigger handlers.
|
|
149
|
+
|
|
150
|
+
Uses lazy loading to avoid circular imports with the modules package.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Tuple of TriggerHandler subclasses for built-in functionality.
|
|
154
|
+
"""
|
|
155
|
+
if cls._builtin_triggers is None:
|
|
156
|
+
from digitalkin.modules.triggers.healthcheck_ping_trigger import HealthcheckPingTrigger # noqa: PLC0415
|
|
157
|
+
from digitalkin.modules.triggers.healthcheck_services_trigger import ( # noqa: PLC0415
|
|
158
|
+
HealthcheckServicesTrigger,
|
|
159
|
+
)
|
|
160
|
+
from digitalkin.modules.triggers.healthcheck_status_trigger import HealthcheckStatusTrigger # noqa: PLC0415
|
|
161
|
+
|
|
162
|
+
cls._builtin_triggers = (
|
|
163
|
+
HealthcheckPingTrigger,
|
|
164
|
+
HealthcheckServicesTrigger,
|
|
165
|
+
HealthcheckStatusTrigger,
|
|
166
|
+
)
|
|
167
|
+
return cls._builtin_triggers
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""Registry data models."""
|
|
2
|
+
|
|
3
|
+
from enum import Enum
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class RegistryModuleStatus(str, Enum):
|
|
9
|
+
"""Module status in the registry."""
|
|
10
|
+
|
|
11
|
+
UNSPECIFIED = "unspecified"
|
|
12
|
+
READY = "ready"
|
|
13
|
+
ACTIVE = "active"
|
|
14
|
+
ARCHIVED = "archived"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class RegistryModuleType(str, Enum):
|
|
18
|
+
"""Module type in the registry."""
|
|
19
|
+
|
|
20
|
+
UNSPECIFIED = "unspecified"
|
|
21
|
+
ARCHETYPE = "archetype"
|
|
22
|
+
TOOL = "tool"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ModuleInfo(BaseModel):
|
|
26
|
+
"""Module information from registry."""
|
|
27
|
+
|
|
28
|
+
module_id: str
|
|
29
|
+
module_type: RegistryModuleType
|
|
30
|
+
address: str
|
|
31
|
+
port: int
|
|
32
|
+
version: str
|
|
33
|
+
name: str = ""
|
|
34
|
+
documentation: str | None = None
|
|
35
|
+
status: RegistryModuleStatus | None = None
|
digitalkin/modules/__init__.py
CHANGED
|
@@ -4,4 +4,8 @@ from digitalkin.modules.archetype_module import ArchetypeModule
|
|
|
4
4
|
from digitalkin.modules.tool_module import ToolModule
|
|
5
5
|
from digitalkin.modules.trigger_handler import TriggerHandler
|
|
6
6
|
|
|
7
|
-
__all__ = [
|
|
7
|
+
__all__ = [
|
|
8
|
+
"ArchetypeModule",
|
|
9
|
+
"ToolModule",
|
|
10
|
+
"TriggerHandler",
|
|
11
|
+
]
|
|
@@ -2,24 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import json
|
|
5
|
+
import os
|
|
5
6
|
from abc import ABC, abstractmethod
|
|
6
7
|
from collections.abc import Callable, Coroutine
|
|
7
8
|
from typing import Any, ClassVar, Generic
|
|
8
9
|
|
|
10
|
+
from digitalkin.grpc_servers.utils.utility_schema_extender import UtilitySchemaExtender
|
|
9
11
|
from digitalkin.logger import logger
|
|
10
|
-
from digitalkin.models.module import
|
|
12
|
+
from digitalkin.models.module.module import ModuleCodeModel, ModuleStatus
|
|
13
|
+
from digitalkin.models.module.module_context import ModuleContext
|
|
14
|
+
from digitalkin.models.module.module_types import (
|
|
15
|
+
DataModel,
|
|
11
16
|
InputModelT,
|
|
12
|
-
ModuleStatus,
|
|
13
17
|
OutputModelT,
|
|
14
18
|
SecretModelT,
|
|
15
19
|
SetupModelT,
|
|
16
20
|
)
|
|
17
|
-
from digitalkin.models.module.
|
|
18
|
-
from digitalkin.models.
|
|
21
|
+
from digitalkin.models.module.utility import EndOfStreamOutput, ModuleStartInfoOutput, UtilityProtocol
|
|
22
|
+
from digitalkin.models.services.storage import BaseRole
|
|
19
23
|
from digitalkin.modules.trigger_handler import TriggerHandler
|
|
20
24
|
from digitalkin.services.services_config import ServicesConfig, ServicesStrategy
|
|
21
|
-
from digitalkin.utils.llm_ready_schema import llm_ready_schema
|
|
22
25
|
from digitalkin.utils.package_discover import ModuleDiscoverer
|
|
26
|
+
from digitalkin.utils.schema_splitter import SchemaSplitter
|
|
23
27
|
|
|
24
28
|
|
|
25
29
|
class BaseModule( # noqa: PLR0904
|
|
@@ -46,10 +50,20 @@ class BaseModule( # noqa: PLR0904
|
|
|
46
50
|
triggers_discoverer: ClassVar[ModuleDiscoverer]
|
|
47
51
|
|
|
48
52
|
# service config params
|
|
49
|
-
services_config_strategies: ClassVar[dict[str, ServicesStrategy | None]]
|
|
50
|
-
services_config_params: ClassVar[dict[str, dict[str, Any | None] | None]]
|
|
53
|
+
services_config_strategies: ClassVar[dict[str, ServicesStrategy | None]] = {}
|
|
54
|
+
services_config_params: ClassVar[dict[str, dict[str, Any | None] | None]] = {}
|
|
51
55
|
services_config: ServicesConfig
|
|
52
56
|
|
|
57
|
+
@classmethod
|
|
58
|
+
def get_module_id(cls) -> str:
|
|
59
|
+
"""Get the module ID from environment variable or metadata.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
The module_id from DIGITALKIN_MODULE_ID env var, or metadata module_id,
|
|
63
|
+
or "unknown" if neither exists.
|
|
64
|
+
"""
|
|
65
|
+
return os.environ.get("DIGITALKIN_MODULE_ID") or cls.metadata.get("module_id", "unknown")
|
|
66
|
+
|
|
53
67
|
def _init_strategies(self, mission_id: str, setup_id: str, setup_version_id: str) -> dict[str, Any]:
|
|
54
68
|
"""Initialize the services configuration.
|
|
55
69
|
|
|
@@ -62,6 +76,7 @@ class BaseModule( # noqa: PLR0904
|
|
|
62
76
|
registry: RegistryStrategy
|
|
63
77
|
snapshot: SnapshotStrategy
|
|
64
78
|
storage: StorageStrategy
|
|
79
|
+
user_profile: UserProfileStrategy
|
|
65
80
|
"""
|
|
66
81
|
logger.debug("Service initialisation: %s", self.services_config_strategies.keys())
|
|
67
82
|
return {
|
|
@@ -122,7 +137,8 @@ class BaseModule( # noqa: PLR0904
|
|
|
122
137
|
"""
|
|
123
138
|
if cls.secret_format is not None:
|
|
124
139
|
if llm_format:
|
|
125
|
-
|
|
140
|
+
result_json, result_ui = SchemaSplitter.split(cls.secret_format.model_json_schema())
|
|
141
|
+
return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
|
|
126
142
|
return json.dumps(cls.secret_format.model_json_schema(), indent=2)
|
|
127
143
|
msg = f"{cls.__name__}' class does not define a 'secret_format'."
|
|
128
144
|
raise NotImplementedError(msg)
|
|
@@ -141,12 +157,16 @@ class BaseModule( # noqa: PLR0904
|
|
|
141
157
|
Raises:
|
|
142
158
|
NotImplementedError: If the `input_format` class attribute is not defined.
|
|
143
159
|
"""
|
|
144
|
-
if cls.input_format is
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
160
|
+
if cls.input_format is None:
|
|
161
|
+
msg = f"{cls.__name__}' class does not define an 'input_format'."
|
|
162
|
+
raise NotImplementedError(msg)
|
|
163
|
+
|
|
164
|
+
extended_model = UtilitySchemaExtender.create_extended_input_model(cls.input_format)
|
|
165
|
+
|
|
166
|
+
if llm_format:
|
|
167
|
+
result_json, result_ui = SchemaSplitter.split(extended_model.model_json_schema())
|
|
168
|
+
return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
|
|
169
|
+
return json.dumps(extended_model.model_json_schema(), indent=2)
|
|
150
170
|
|
|
151
171
|
@classmethod
|
|
152
172
|
async def get_output_format(cls, *, llm_format: bool) -> str:
|
|
@@ -162,12 +182,16 @@ class BaseModule( # noqa: PLR0904
|
|
|
162
182
|
Raises:
|
|
163
183
|
NotImplementedError: If the `output_format` class attribute is not defined.
|
|
164
184
|
"""
|
|
165
|
-
if cls.output_format is
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
185
|
+
if cls.output_format is None:
|
|
186
|
+
msg = f"'{cls.__name__}' class does not define an 'output_format'."
|
|
187
|
+
raise NotImplementedError(msg)
|
|
188
|
+
|
|
189
|
+
extended_model = UtilitySchemaExtender.create_extended_output_model(cls.output_format)
|
|
190
|
+
|
|
191
|
+
if llm_format:
|
|
192
|
+
result_json, result_ui = SchemaSplitter.split(extended_model.model_json_schema())
|
|
193
|
+
return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
|
|
194
|
+
return json.dumps(extended_model.model_json_schema(), indent=2)
|
|
171
195
|
|
|
172
196
|
@classmethod
|
|
173
197
|
async def get_config_setup_format(cls, *, llm_format: bool) -> str:
|
|
@@ -194,7 +218,8 @@ class BaseModule( # noqa: PLR0904
|
|
|
194
218
|
if cls.setup_format is not None:
|
|
195
219
|
setup_format = await cls.setup_format.get_clean_model(config_fields=True, hidden_fields=False, force=True)
|
|
196
220
|
if llm_format:
|
|
197
|
-
|
|
221
|
+
result_json, result_ui = SchemaSplitter.split(setup_format.model_json_schema())
|
|
222
|
+
return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
|
|
198
223
|
return json.dumps(setup_format.model_json_schema(), indent=2)
|
|
199
224
|
msg = "'%s' class does not define an 'config_setup_format'."
|
|
200
225
|
raise NotImplementedError(msg)
|
|
@@ -223,7 +248,8 @@ class BaseModule( # noqa: PLR0904
|
|
|
223
248
|
if cls.setup_format is not None:
|
|
224
249
|
setup_format = await cls.setup_format.get_clean_model(config_fields=False, hidden_fields=True, force=True)
|
|
225
250
|
if llm_format:
|
|
226
|
-
|
|
251
|
+
result_json, result_ui = SchemaSplitter.split(setup_format.model_json_schema())
|
|
252
|
+
return json.dumps({"json_schema": result_json, "ui_schema": result_ui}, indent=2)
|
|
227
253
|
return json.dumps(setup_format.model_json_schema(), indent=2)
|
|
228
254
|
msg = "'%s' class does not define an 'setup_format'."
|
|
229
255
|
raise NotImplementedError(msg)
|
|
@@ -304,8 +330,18 @@ class BaseModule( # noqa: PLR0904
|
|
|
304
330
|
If a package is provided, all .py files within its path are imported; otherwise, the current
|
|
305
331
|
working directory is searched. For each imported module, any class matching the criteria is
|
|
306
332
|
registered via cls.register(). Errors during import are logged at debug level.
|
|
333
|
+
|
|
334
|
+
Built-in healthcheck handlers (ping, services, status) are automatically registered
|
|
335
|
+
to provide standard healthcheck functionality for all modules.
|
|
307
336
|
"""
|
|
337
|
+
from digitalkin.models.module.utility import UtilityRegistry # noqa: PLC0415
|
|
338
|
+
|
|
308
339
|
cls.triggers_discoverer.discover_modules()
|
|
340
|
+
|
|
341
|
+
# Auto-register built-in SDK triggers (healthcheck, etc.)
|
|
342
|
+
for trigger_cls in UtilityRegistry.get_builtin_triggers():
|
|
343
|
+
cls.triggers_discoverer.register_trigger(trigger_cls)
|
|
344
|
+
|
|
309
345
|
logger.debug("discovered: %s", cls.triggers_discoverer)
|
|
310
346
|
|
|
311
347
|
@classmethod
|
|
@@ -320,25 +356,6 @@ class BaseModule( # noqa: PLR0904
|
|
|
320
356
|
"""
|
|
321
357
|
return cls.triggers_discoverer.register_trigger(handler_cls)
|
|
322
358
|
|
|
323
|
-
async def run_config_setup( # noqa: PLR6301
|
|
324
|
-
self,
|
|
325
|
-
context: ModuleContext, # noqa: ARG002
|
|
326
|
-
config_setup_data: SetupModelT,
|
|
327
|
-
) -> SetupModelT:
|
|
328
|
-
"""Run config setup the module.
|
|
329
|
-
|
|
330
|
-
The config setup is used to initialize the setup with configuration data.
|
|
331
|
-
This method is typically used to set up the module with necessary configuration before running it,
|
|
332
|
-
especially for processing data like files.
|
|
333
|
-
The function needs to save the setup in the storage.
|
|
334
|
-
The module will be initialize with the setup and not the config setup.
|
|
335
|
-
This method is optional, the config setup and setup can be the same.
|
|
336
|
-
|
|
337
|
-
Returns:
|
|
338
|
-
The updated setup model after running the config setup.
|
|
339
|
-
"""
|
|
340
|
-
return config_setup_data
|
|
341
|
-
|
|
342
359
|
@abstractmethod
|
|
343
360
|
async def initialize(self, context: ModuleContext, setup_data: SetupModelT) -> None:
|
|
344
361
|
"""Initialize the module."""
|
|
@@ -349,19 +366,11 @@ class BaseModule( # noqa: PLR0904
|
|
|
349
366
|
input_data: InputModelT,
|
|
350
367
|
setup_data: SetupModelT,
|
|
351
368
|
) -> None:
|
|
352
|
-
"""Run the module
|
|
353
|
-
|
|
354
|
-
This method validates the input data, determines the protocol from the input,
|
|
355
|
-
and dispatches the request to the corresponding trigger handler. The trigger handler
|
|
356
|
-
is responsible for processing the input and invoking the callback with the result.
|
|
357
|
-
|
|
358
|
-
Triggers:
|
|
359
|
-
- The method is triggered when a module run is requested with specific input and setup data.
|
|
360
|
-
- The protocol specified in the input determines which trigger handler is invoked.
|
|
369
|
+
"""Run the module by dispatching to the appropriate trigger handler.
|
|
361
370
|
|
|
362
371
|
Args:
|
|
363
|
-
input_data
|
|
364
|
-
setup_data
|
|
372
|
+
input_data: Input data to process.
|
|
373
|
+
setup_data: Configuration data for the module.
|
|
365
374
|
|
|
366
375
|
Raises:
|
|
367
376
|
ValueError: If no handler for the protocol is found.
|
|
@@ -383,6 +392,25 @@ class BaseModule( # noqa: PLR0904
|
|
|
383
392
|
"""Run the module."""
|
|
384
393
|
raise NotImplementedError
|
|
385
394
|
|
|
395
|
+
async def run_config_setup( # noqa: PLR6301
|
|
396
|
+
self,
|
|
397
|
+
context: ModuleContext, # noqa: ARG002
|
|
398
|
+
config_setup_data: SetupModelT,
|
|
399
|
+
) -> SetupModelT:
|
|
400
|
+
"""Run config setup the module.
|
|
401
|
+
|
|
402
|
+
The config setup is used to initialize the setup with configuration data.
|
|
403
|
+
This method is typically used to set up the module with necessary configuration before running it,
|
|
404
|
+
especially for processing data like files.
|
|
405
|
+
The function needs to save the setup in the storage.
|
|
406
|
+
The module will be initialize with the setup and not the config setup.
|
|
407
|
+
This method is optional, the config setup and setup can be the same.
|
|
408
|
+
|
|
409
|
+
Returns:
|
|
410
|
+
The updated setup model after running the config setup.
|
|
411
|
+
"""
|
|
412
|
+
return config_setup_data
|
|
413
|
+
|
|
386
414
|
async def _run_lifecycle(
|
|
387
415
|
self,
|
|
388
416
|
input_data: InputModelT,
|
|
@@ -410,13 +438,32 @@ class BaseModule( # noqa: PLR0904
|
|
|
410
438
|
self,
|
|
411
439
|
input_data: InputModelT,
|
|
412
440
|
setup_data: SetupModelT,
|
|
413
|
-
callback: Callable[[OutputModelT | ModuleCodeModel], Coroutine[Any, Any, None]],
|
|
441
|
+
callback: Callable[[OutputModelT | ModuleCodeModel | DataModel[UtilityProtocol]], Coroutine[Any, Any, None]],
|
|
414
442
|
done_callback: Callable | None = None,
|
|
415
443
|
) -> None:
|
|
416
444
|
"""Start the module."""
|
|
417
445
|
try:
|
|
418
446
|
self.context.callbacks.send_message = callback
|
|
419
|
-
|
|
447
|
+
|
|
448
|
+
tool_cache = setup_data.build_tool_cache()
|
|
449
|
+
if tool_cache.entries:
|
|
450
|
+
self.context.tool_cache = tool_cache
|
|
451
|
+
|
|
452
|
+
await callback(
|
|
453
|
+
DataModel(
|
|
454
|
+
root=ModuleStartInfoOutput(
|
|
455
|
+
job_id=self.context.session.job_id,
|
|
456
|
+
mission_id=self.context.session.mission_id,
|
|
457
|
+
setup_id=self.context.session.setup_id,
|
|
458
|
+
setup_version_id=self.context.session.setup_version_id,
|
|
459
|
+
module_id=self.get_module_id(),
|
|
460
|
+
module_name=self.name,
|
|
461
|
+
),
|
|
462
|
+
annotations={"role": BaseRole.SYSTEM},
|
|
463
|
+
)
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
logger.info("Initialize module %s", self.context.session.job_id)
|
|
420
467
|
await self.initialize(self.context, setup_data)
|
|
421
468
|
except Exception as e:
|
|
422
469
|
self._status = ModuleStatus.FAILED
|
|
@@ -437,7 +484,7 @@ class BaseModule( # noqa: PLR0904
|
|
|
437
484
|
try:
|
|
438
485
|
logger.debug("Init the discovered input handlers.")
|
|
439
486
|
self.triggers_discoverer.init_handlers(self.context)
|
|
440
|
-
logger.debug(
|
|
487
|
+
logger.debug("Run lifecycle %s", self.context.session.job_id)
|
|
441
488
|
await self._run_lifecycle(input_data, setup_data)
|
|
442
489
|
except Exception:
|
|
443
490
|
self._status = ModuleStatus.FAILED
|
|
@@ -452,31 +499,77 @@ class BaseModule( # noqa: PLR0904
|
|
|
452
499
|
self._status = ModuleStatus.STOPPING
|
|
453
500
|
logger.debug("Module %s stopped", self.name)
|
|
454
501
|
await self.cleanup()
|
|
455
|
-
await self.context.callbacks.send_message(
|
|
502
|
+
await self.context.callbacks.send_message(
|
|
503
|
+
DataModel(
|
|
504
|
+
root=EndOfStreamOutput(),
|
|
505
|
+
annotations={"role": BaseRole.SYSTEM},
|
|
506
|
+
)
|
|
507
|
+
)
|
|
456
508
|
self._status = ModuleStatus.STOPPED
|
|
457
509
|
logger.debug("Module %s cleaned", self.name)
|
|
458
510
|
except Exception:
|
|
459
511
|
self._status = ModuleStatus.FAILED
|
|
460
512
|
logger.exception("Error stopping module")
|
|
461
513
|
|
|
514
|
+
async def _resolve_tools(self, config_setup_data: SetupModelT) -> None:
|
|
515
|
+
"""Resolve tool references and build cache.
|
|
516
|
+
|
|
517
|
+
Args:
|
|
518
|
+
config_setup_data: Setup data containing tool references.
|
|
519
|
+
"""
|
|
520
|
+
logger.info("Starting tool resolution", extra=self.context.session.current_ids())
|
|
521
|
+
if self.context.registry is not None:
|
|
522
|
+
config_setup_data.resolve_tool_references(self.context.registry)
|
|
523
|
+
logger.info("Tool references resolved", extra=self.context.session.current_ids())
|
|
524
|
+
else:
|
|
525
|
+
logger.warning("No registry available, skipping tool resolution", extra=self.context.session.current_ids())
|
|
526
|
+
|
|
527
|
+
tool_cache = config_setup_data.build_tool_cache()
|
|
528
|
+
self.context.tool_cache = tool_cache
|
|
529
|
+
logger.info(
|
|
530
|
+
"Tool cache built with %d entries: %s",
|
|
531
|
+
len(tool_cache.entries),
|
|
532
|
+
list(tool_cache.entries.keys()),
|
|
533
|
+
extra=self.context.session.current_ids(),
|
|
534
|
+
)
|
|
535
|
+
|
|
462
536
|
async def start_config_setup(
|
|
463
537
|
self,
|
|
464
538
|
config_setup_data: SetupModelT,
|
|
465
539
|
callback: Callable[[SetupModelT | ModuleCodeModel], Coroutine[Any, Any, None]],
|
|
466
540
|
) -> None:
|
|
467
|
-
"""
|
|
541
|
+
"""Run config setup lifecycle with tool resolution in parallel.
|
|
542
|
+
|
|
543
|
+
Args:
|
|
544
|
+
config_setup_data: Initial setup data to configure.
|
|
545
|
+
callback: Callback to send the configured setup model.
|
|
546
|
+
"""
|
|
468
547
|
try:
|
|
469
548
|
logger.info("Run Config Setup lifecycle", extra=self.context.session.current_ids())
|
|
470
549
|
self._status = ModuleStatus.RUNNING
|
|
471
550
|
self.context.callbacks.set_config_setup = callback
|
|
472
|
-
content = await self.run_config_setup(self.context, config_setup_data)
|
|
473
551
|
|
|
552
|
+
# Resolve tools first to populate companion fields, then run config setup
|
|
553
|
+
await self._resolve_tools(config_setup_data)
|
|
554
|
+
updated_config = await self.run_config_setup(self.context, config_setup_data)
|
|
555
|
+
|
|
556
|
+
# Build wrapper: original structure with updated content
|
|
474
557
|
wrapper = config_setup_data.model_dump()
|
|
475
|
-
wrapper["content"] =
|
|
558
|
+
wrapper["content"] = updated_config.model_dump()
|
|
559
|
+
|
|
560
|
+
# Debug logging
|
|
561
|
+
content = wrapper.get("content", {})
|
|
562
|
+
logger.info(
|
|
563
|
+
"Config setup wrapper: keys=%s, content_keys=%s, tools_cache=%s",
|
|
564
|
+
list(wrapper.keys()),
|
|
565
|
+
list(content.keys()) if isinstance(content, dict) else "N/A",
|
|
566
|
+
content.get("tools_cache") if isinstance(content, dict) else "N/A",
|
|
567
|
+
extra=self.context.session.current_ids(),
|
|
568
|
+
)
|
|
569
|
+
|
|
476
570
|
setup_model = await self.create_setup_model(wrapper)
|
|
477
571
|
await callback(setup_model)
|
|
478
572
|
self._status = ModuleStatus.STOPPING
|
|
479
573
|
except Exception:
|
|
480
|
-
logger.error("Error during module lifecyle")
|
|
481
574
|
self._status = ModuleStatus.FAILED
|
|
482
|
-
logger.exception("Error during
|
|
575
|
+
logger.exception("Error during config setup lifecycle", extra=self.context.session.current_ids())
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
from abc import ABC
|
|
4
4
|
|
|
5
|
-
from digitalkin.models.module import
|
|
5
|
+
from digitalkin.models.module.module_types import (
|
|
6
|
+
InputModelT,
|
|
7
|
+
OutputModelT,
|
|
8
|
+
SecretModelT,
|
|
9
|
+
SetupModelT,
|
|
10
|
+
)
|
|
6
11
|
from digitalkin.modules._base_module import BaseModule
|
|
7
12
|
|
|
8
13
|
|
|
@@ -2,7 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
from abc import ABC
|
|
4
4
|
|
|
5
|
-
from digitalkin.models.module import
|
|
5
|
+
from digitalkin.models.module.module_types import (
|
|
6
|
+
InputModelT,
|
|
7
|
+
OutputModelT,
|
|
8
|
+
SecretModelT,
|
|
9
|
+
SetupModelT,
|
|
10
|
+
)
|
|
6
11
|
from digitalkin.modules._base_module import BaseModule # type: ignore
|
|
7
12
|
|
|
8
13
|
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Built-in SDK triggers.
|
|
2
|
+
|
|
3
|
+
These triggers are automatically registered without requiring discovery.
|
|
4
|
+
They provide standard functionality available to all modules.
|
|
5
|
+
|
|
6
|
+
Note: These are internal triggers. External code should not import them directly.
|
|
7
|
+
Use UtilityRegistry.get_builtin_triggers() to access the trigger classes.
|
|
8
|
+
"""
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Healthcheck ping trigger - simple alive check."""
|
|
2
|
+
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from typing import Any, ClassVar
|
|
5
|
+
|
|
6
|
+
from digitalkin.mixins import BaseMixin
|
|
7
|
+
from digitalkin.models.module.module_context import ModuleContext
|
|
8
|
+
from digitalkin.models.module.utility import (
|
|
9
|
+
HealthcheckPingInput,
|
|
10
|
+
HealthcheckPingOutput,
|
|
11
|
+
)
|
|
12
|
+
from digitalkin.modules.trigger_handler import TriggerHandler
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class HealthcheckPingTrigger(TriggerHandler, BaseMixin):
|
|
16
|
+
"""Handler for simple ping healthcheck.
|
|
17
|
+
|
|
18
|
+
Responds immediately with "pong" status to verify the module is responsive.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
protocol: ClassVar[str] = "healthcheck_ping"
|
|
22
|
+
input_format = HealthcheckPingInput
|
|
23
|
+
_request_time: datetime
|
|
24
|
+
|
|
25
|
+
def __init__(self, context: ModuleContext) -> None:
|
|
26
|
+
"""Initialize the handler."""
|
|
27
|
+
self._request_time = datetime.now(tz=context.session.timezone)
|
|
28
|
+
|
|
29
|
+
async def handle(
|
|
30
|
+
self,
|
|
31
|
+
input_data: HealthcheckPingInput, # noqa: ARG002
|
|
32
|
+
setup_data: Any, # noqa: ANN401, ARG002
|
|
33
|
+
context: ModuleContext,
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Handle ping healthcheck request.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
input_data: The input trigger data (unused for healthcheck).
|
|
39
|
+
setup_data: The setup configuration (unused for healthcheck).
|
|
40
|
+
context: The module context.
|
|
41
|
+
"""
|
|
42
|
+
elapsed = datetime.now(tz=context.session.timezone) - self._request_time
|
|
43
|
+
latency_ms = elapsed.total_seconds() * 1000
|
|
44
|
+
output = HealthcheckPingOutput(latency_ms=latency_ms)
|
|
45
|
+
await self.send_message(context, output)
|