digitalkin 0.3.1.dev2__py3-none-any.whl → 0.3.2.dev14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- base_server/server_async_insecure.py +6 -5
- base_server/server_async_secure.py +6 -5
- base_server/server_sync_insecure.py +5 -4
- base_server/server_sync_secure.py +5 -4
- digitalkin/__version__.py +1 -1
- digitalkin/core/job_manager/base_job_manager.py +1 -1
- digitalkin/core/job_manager/single_job_manager.py +28 -9
- digitalkin/core/job_manager/taskiq_broker.py +7 -6
- digitalkin/core/job_manager/taskiq_job_manager.py +1 -1
- digitalkin/core/task_manager/surrealdb_repository.py +7 -7
- digitalkin/core/task_manager/task_session.py +60 -98
- digitalkin/grpc_servers/module_server.py +109 -168
- digitalkin/grpc_servers/module_servicer.py +38 -16
- digitalkin/grpc_servers/utils/grpc_client_wrapper.py +24 -8
- digitalkin/grpc_servers/utils/utility_schema_extender.py +100 -0
- digitalkin/models/__init__.py +1 -1
- digitalkin/models/core/job_manager_models.py +0 -8
- digitalkin/models/core/task_monitor.py +4 -0
- digitalkin/models/grpc_servers/models.py +91 -6
- digitalkin/models/module/__init__.py +18 -13
- digitalkin/models/module/base_types.py +61 -0
- digitalkin/models/module/module_context.py +173 -13
- digitalkin/models/module/module_types.py +28 -392
- digitalkin/models/module/setup_types.py +490 -0
- digitalkin/models/module/tool_cache.py +68 -0
- digitalkin/models/module/tool_reference.py +117 -0
- digitalkin/models/module/utility.py +167 -0
- digitalkin/models/services/registry.py +35 -0
- digitalkin/modules/__init__.py +5 -1
- digitalkin/modules/_base_module.py +154 -61
- digitalkin/modules/archetype_module.py +6 -1
- digitalkin/modules/tool_module.py +6 -1
- digitalkin/modules/triggers/__init__.py +8 -0
- digitalkin/modules/triggers/healthcheck_ping_trigger.py +45 -0
- digitalkin/modules/triggers/healthcheck_services_trigger.py +63 -0
- digitalkin/modules/triggers/healthcheck_status_trigger.py +52 -0
- digitalkin/services/__init__.py +4 -0
- digitalkin/services/communication/__init__.py +7 -0
- digitalkin/services/communication/communication_strategy.py +76 -0
- digitalkin/services/communication/default_communication.py +101 -0
- digitalkin/services/communication/grpc_communication.py +234 -0
- digitalkin/services/cost/grpc_cost.py +1 -1
- digitalkin/services/filesystem/grpc_filesystem.py +1 -1
- digitalkin/services/registry/__init__.py +22 -1
- digitalkin/services/registry/default_registry.py +135 -4
- digitalkin/services/registry/exceptions.py +47 -0
- digitalkin/services/registry/grpc_registry.py +306 -0
- digitalkin/services/registry/registry_models.py +15 -0
- digitalkin/services/registry/registry_strategy.py +88 -4
- digitalkin/services/services_config.py +25 -3
- digitalkin/services/services_models.py +5 -1
- digitalkin/services/setup/default_setup.py +1 -1
- digitalkin/services/setup/grpc_setup.py +1 -1
- digitalkin/services/storage/grpc_storage.py +1 -1
- digitalkin/services/user_profile/__init__.py +11 -0
- digitalkin/services/user_profile/grpc_user_profile.py +2 -2
- digitalkin/services/user_profile/user_profile_strategy.py +0 -15
- digitalkin/utils/schema_splitter.py +207 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/METADATA +5 -5
- digitalkin-0.3.2.dev14.dist-info/RECORD +143 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/top_level.txt +1 -0
- modules/archetype_with_tools_module.py +244 -0
- modules/cpu_intensive_module.py +1 -1
- modules/dynamic_setup_module.py +5 -29
- modules/minimal_llm_module.py +1 -1
- modules/text_transform_module.py +1 -1
- monitoring/digitalkin_observability/__init__.py +46 -0
- monitoring/digitalkin_observability/http_server.py +150 -0
- monitoring/digitalkin_observability/interceptors.py +176 -0
- monitoring/digitalkin_observability/metrics.py +201 -0
- monitoring/digitalkin_observability/prometheus.py +137 -0
- monitoring/tests/test_metrics.py +172 -0
- services/filesystem_module.py +7 -5
- services/storage_module.py +4 -2
- digitalkin/grpc_servers/registry_server.py +0 -65
- digitalkin/grpc_servers/registry_servicer.py +0 -456
- digitalkin-0.3.1.dev2.dist-info/RECORD +0 -119
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/WHEEL +0 -0
- {digitalkin-0.3.1.dev2.dist-info → digitalkin-0.3.2.dev14.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
"""Define the module context used in the triggers."""
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
from collections.abc import AsyncGenerator, Callable, Coroutine
|
|
5
|
+
from datetime import tzinfo
|
|
3
6
|
from types import SimpleNamespace
|
|
4
7
|
from typing import Any
|
|
8
|
+
from zoneinfo import ZoneInfo
|
|
5
9
|
|
|
10
|
+
from digitalkin.logger import logger
|
|
11
|
+
from digitalkin.models.module.tool_cache import ToolCache
|
|
6
12
|
from digitalkin.services.agent.agent_strategy import AgentStrategy
|
|
13
|
+
from digitalkin.services.communication.communication_strategy import CommunicationStrategy
|
|
7
14
|
from digitalkin.services.cost.cost_strategy import CostStrategy
|
|
8
15
|
from digitalkin.services.filesystem.filesystem_strategy import FilesystemStrategy
|
|
9
16
|
from digitalkin.services.identity.identity_strategy import IdentityStrategy
|
|
10
17
|
from digitalkin.services.registry.registry_strategy import RegistryStrategy
|
|
11
18
|
from digitalkin.services.snapshot.snapshot_strategy import SnapshotStrategy
|
|
12
19
|
from digitalkin.services.storage.storage_strategy import StorageStrategy
|
|
20
|
+
from digitalkin.services.user_profile.user_profile_strategy import UserProfileStrategy
|
|
13
21
|
|
|
14
22
|
|
|
15
23
|
class Session(SimpleNamespace):
|
|
@@ -19,6 +27,7 @@ class Session(SimpleNamespace):
|
|
|
19
27
|
mission_id: str
|
|
20
28
|
setup_id: str
|
|
21
29
|
setup_version_id: str
|
|
30
|
+
timezone: tzinfo
|
|
22
31
|
|
|
23
32
|
def __init__(
|
|
24
33
|
self,
|
|
@@ -26,37 +35,32 @@ class Session(SimpleNamespace):
|
|
|
26
35
|
mission_id: str,
|
|
27
36
|
setup_id: str,
|
|
28
37
|
setup_version_id: str,
|
|
38
|
+
timezone: tzinfo | None = None,
|
|
29
39
|
**kwargs: dict[str, Any],
|
|
30
40
|
) -> None:
|
|
31
41
|
"""Init Module Session.
|
|
32
42
|
|
|
33
|
-
Args:
|
|
34
|
-
job_id: current job_id.
|
|
35
|
-
mission_id: current mission_id.
|
|
36
|
-
setup_id: used setup config.
|
|
37
|
-
setup_version_id: used setup config.
|
|
38
|
-
kwargs: user defined session variables.
|
|
39
|
-
|
|
40
43
|
Raises:
|
|
41
|
-
ValueError: If mandatory args are missing
|
|
44
|
+
ValueError: If mandatory args are missing.
|
|
42
45
|
"""
|
|
43
46
|
if not setup_id:
|
|
44
|
-
msg = "setup_id is mandatory
|
|
47
|
+
msg = "setup_id is mandatory"
|
|
45
48
|
raise ValueError(msg)
|
|
46
49
|
if not setup_version_id:
|
|
47
|
-
msg = "setup_version_id is mandatory
|
|
50
|
+
msg = "setup_version_id is mandatory"
|
|
48
51
|
raise ValueError(msg)
|
|
49
52
|
if not mission_id:
|
|
50
|
-
msg = "mission_id is mandatory
|
|
53
|
+
msg = "mission_id is mandatory"
|
|
51
54
|
raise ValueError(msg)
|
|
52
55
|
if not job_id:
|
|
53
|
-
msg = "job_id is mandatory
|
|
56
|
+
msg = "job_id is mandatory"
|
|
54
57
|
raise ValueError(msg)
|
|
55
58
|
|
|
56
59
|
self.job_id = job_id
|
|
57
60
|
self.mission_id = mission_id
|
|
58
61
|
self.setup_id = setup_id
|
|
59
62
|
self.setup_version_id = setup_version_id
|
|
63
|
+
self.timezone = timezone or ZoneInfo(os.environ.get("DIGITALKIN_TIMEZONE", "Europe/Paris"))
|
|
60
64
|
|
|
61
65
|
super().__init__(**kwargs)
|
|
62
66
|
|
|
@@ -83,58 +87,214 @@ class ModuleContext:
|
|
|
83
87
|
|
|
84
88
|
# services list
|
|
85
89
|
agent: AgentStrategy
|
|
90
|
+
communication: CommunicationStrategy
|
|
86
91
|
cost: CostStrategy
|
|
87
92
|
filesystem: FilesystemStrategy
|
|
88
93
|
identity: IdentityStrategy
|
|
89
94
|
registry: RegistryStrategy
|
|
90
95
|
snapshot: SnapshotStrategy
|
|
91
96
|
storage: StorageStrategy
|
|
97
|
+
user_profile: UserProfileStrategy
|
|
92
98
|
|
|
93
99
|
session: Session
|
|
94
100
|
callbacks: SimpleNamespace
|
|
95
101
|
metadata: SimpleNamespace
|
|
96
102
|
helpers: SimpleNamespace
|
|
97
103
|
state: SimpleNamespace = SimpleNamespace()
|
|
104
|
+
tool_cache: ToolCache
|
|
98
105
|
|
|
99
106
|
def __init__( # noqa: PLR0913, PLR0917
|
|
100
107
|
self,
|
|
101
108
|
agent: AgentStrategy,
|
|
109
|
+
communication: CommunicationStrategy,
|
|
102
110
|
cost: CostStrategy,
|
|
103
111
|
filesystem: FilesystemStrategy,
|
|
104
112
|
identity: IdentityStrategy,
|
|
105
113
|
registry: RegistryStrategy,
|
|
106
114
|
snapshot: SnapshotStrategy,
|
|
107
115
|
storage: StorageStrategy,
|
|
116
|
+
user_profile: UserProfileStrategy,
|
|
108
117
|
session: dict[str, Any],
|
|
109
118
|
metadata: dict[str, Any] = {},
|
|
110
119
|
helpers: dict[str, Any] = {},
|
|
111
120
|
callbacks: dict[str, Any] = {},
|
|
121
|
+
tool_cache: ToolCache | None = None,
|
|
112
122
|
) -> None:
|
|
113
123
|
"""Register mandatory services, session, metadata and callbacks.
|
|
114
124
|
|
|
115
125
|
Args:
|
|
116
126
|
agent: AgentStrategy.
|
|
127
|
+
communication: CommunicationStrategy.
|
|
117
128
|
cost: CostStrategy.
|
|
118
129
|
filesystem: FilesystemStrategy.
|
|
119
130
|
identity: IdentityStrategy.
|
|
120
131
|
registry: RegistryStrategy.
|
|
121
132
|
snapshot: SnapshotStrategy.
|
|
122
133
|
storage: StorageStrategy.
|
|
134
|
+
user_profile: UserProfileStrategy.
|
|
123
135
|
metadata: dict defining differents Module metadata.
|
|
124
136
|
helpers: dict different user defined helpers.
|
|
125
137
|
session: dict referring the session IDs or informations.
|
|
126
138
|
callbacks: Functions allowing user to agent interaction.
|
|
139
|
+
tool_cache: ToolCache with pre-resolved tool references from setup.
|
|
127
140
|
"""
|
|
128
|
-
# Core services
|
|
129
141
|
self.agent = agent
|
|
142
|
+
self.communication = communication
|
|
130
143
|
self.cost = cost
|
|
131
144
|
self.filesystem = filesystem
|
|
132
145
|
self.identity = identity
|
|
133
146
|
self.registry = registry
|
|
134
147
|
self.snapshot = snapshot
|
|
135
148
|
self.storage = storage
|
|
149
|
+
self.user_profile = user_profile
|
|
136
150
|
|
|
137
151
|
self.metadata = SimpleNamespace(**metadata)
|
|
138
152
|
self.session = Session(**session)
|
|
139
153
|
self.helpers = SimpleNamespace(**helpers)
|
|
140
154
|
self.callbacks = SimpleNamespace(**callbacks)
|
|
155
|
+
self.tool_cache = tool_cache or ToolCache()
|
|
156
|
+
|
|
157
|
+
async def call_module_by_id(
|
|
158
|
+
self,
|
|
159
|
+
module_id: str,
|
|
160
|
+
input_data: dict,
|
|
161
|
+
setup_id: str,
|
|
162
|
+
mission_id: str,
|
|
163
|
+
callback: Callable[[dict], Coroutine[Any, Any, None]] | None = None,
|
|
164
|
+
) -> AsyncGenerator[dict, None]:
|
|
165
|
+
"""Call a module by ID, discovering address/port from registry.
|
|
166
|
+
|
|
167
|
+
Args:
|
|
168
|
+
module_id: Module identifier to look up in registry.
|
|
169
|
+
input_data: Input data as dictionary.
|
|
170
|
+
setup_id: Setup configuration ID.
|
|
171
|
+
mission_id: Mission context ID.
|
|
172
|
+
callback: Optional callback for each response.
|
|
173
|
+
|
|
174
|
+
Yields:
|
|
175
|
+
Streaming responses from module as dictionaries.
|
|
176
|
+
"""
|
|
177
|
+
module_info = self.registry.discover_by_id(module_id)
|
|
178
|
+
|
|
179
|
+
logger.debug(
|
|
180
|
+
"Calling module by ID",
|
|
181
|
+
extra={
|
|
182
|
+
"module_id": module_id,
|
|
183
|
+
"address": module_info.address,
|
|
184
|
+
"port": module_info.port,
|
|
185
|
+
},
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
async for response in self.communication.call_module(
|
|
189
|
+
module_address=module_info.address,
|
|
190
|
+
module_port=module_info.port,
|
|
191
|
+
input_data=input_data,
|
|
192
|
+
setup_id=setup_id,
|
|
193
|
+
mission_id=mission_id,
|
|
194
|
+
callback=callback,
|
|
195
|
+
):
|
|
196
|
+
yield response
|
|
197
|
+
|
|
198
|
+
async def get_module_schemas_by_id(
|
|
199
|
+
self,
|
|
200
|
+
module_id: str,
|
|
201
|
+
*,
|
|
202
|
+
llm_format: bool = False,
|
|
203
|
+
) -> dict[str, dict]:
|
|
204
|
+
"""Get module schemas by ID, discovering address/port from registry.
|
|
205
|
+
|
|
206
|
+
Args:
|
|
207
|
+
module_id: Module identifier to look up in registry.
|
|
208
|
+
llm_format: If True, return LLM-optimized schema format.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
Dictionary containing schemas: {"input": ..., "output": ..., "setup": ..., "secret": ...}
|
|
212
|
+
"""
|
|
213
|
+
module_info = self.registry.discover_by_id(module_id)
|
|
214
|
+
|
|
215
|
+
logger.debug(
|
|
216
|
+
"Getting module schemas by ID",
|
|
217
|
+
extra={
|
|
218
|
+
"module_id": module_id,
|
|
219
|
+
"address": module_info.address,
|
|
220
|
+
"port": module_info.port,
|
|
221
|
+
},
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
return await self.communication.get_module_schemas(
|
|
225
|
+
module_address=module_info.address,
|
|
226
|
+
module_port=module_info.port,
|
|
227
|
+
llm_format=llm_format,
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
async def create_openai_style_tool(self, tool_name: str) -> dict[str, Any] | None:
|
|
231
|
+
"""Create OpenAI-style function calling schema for a tool.
|
|
232
|
+
|
|
233
|
+
Uses tool cache (fast path) with registry fallback. Fetches the tool's
|
|
234
|
+
input schema and wraps it in OpenAI function calling format.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
tool_name: Module ID to look up (checks cache first, then registry).
|
|
238
|
+
|
|
239
|
+
Returns:
|
|
240
|
+
OpenAI-style tool schema if found, None otherwise.
|
|
241
|
+
"""
|
|
242
|
+
module_info = self.tool_cache.get(tool_name, registry=self.registry)
|
|
243
|
+
if not module_info:
|
|
244
|
+
return None
|
|
245
|
+
|
|
246
|
+
schemas = await self.communication.get_module_schemas(
|
|
247
|
+
module_address=module_info.address,
|
|
248
|
+
module_port=module_info.port,
|
|
249
|
+
llm_format=True,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
return {
|
|
253
|
+
"type": "function",
|
|
254
|
+
"function": {
|
|
255
|
+
"module_id": module_info.module_id,
|
|
256
|
+
"name": module_info.name or "undefined",
|
|
257
|
+
"description": module_info.documentation or "",
|
|
258
|
+
"parameters": schemas["input"],
|
|
259
|
+
},
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
def create_tool_function(
|
|
263
|
+
self,
|
|
264
|
+
module_id: str,
|
|
265
|
+
) -> Callable[..., AsyncGenerator[dict, None]] | None:
|
|
266
|
+
"""Create async generator function for a tool.
|
|
267
|
+
|
|
268
|
+
Returns an async generator that calls the remote tool module via gRPC
|
|
269
|
+
and yields each response as it arrives until end_of_stream or gRPC ends.
|
|
270
|
+
|
|
271
|
+
Args:
|
|
272
|
+
module_id: Module ID to look up (checks cache first, then registry).
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
Async generator function if tool found, None otherwise.
|
|
276
|
+
"""
|
|
277
|
+
module_info = self.tool_cache.get(module_id, registry=self.registry)
|
|
278
|
+
if not module_info:
|
|
279
|
+
return None
|
|
280
|
+
|
|
281
|
+
communication = self.communication
|
|
282
|
+
session = self.session
|
|
283
|
+
address = module_info.address
|
|
284
|
+
port = module_info.port
|
|
285
|
+
|
|
286
|
+
async def tool_function(**kwargs: Any) -> AsyncGenerator[dict, None]: # noqa: ANN401
|
|
287
|
+
wrapped_input = {"root": kwargs}
|
|
288
|
+
async for response in communication.call_module(
|
|
289
|
+
module_address=address,
|
|
290
|
+
module_port=port,
|
|
291
|
+
input_data=wrapped_input,
|
|
292
|
+
setup_id=session.setup_id,
|
|
293
|
+
mission_id=session.mission_id,
|
|
294
|
+
):
|
|
295
|
+
yield response
|
|
296
|
+
|
|
297
|
+
tool_function.__name__ = module_info.name or module_info.module_id
|
|
298
|
+
tool_function.__doc__ = module_info.documentation or ""
|
|
299
|
+
|
|
300
|
+
return tool_function
|