digitalkin 0.3.1.dev1__py3-none-any.whl → 0.3.2a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. base_server/server_async_insecure.py +6 -5
  2. base_server/server_async_secure.py +6 -5
  3. base_server/server_sync_insecure.py +5 -4
  4. base_server/server_sync_secure.py +5 -4
  5. digitalkin/__version__.py +1 -1
  6. digitalkin/core/job_manager/base_job_manager.py +1 -1
  7. digitalkin/core/job_manager/single_job_manager.py +78 -36
  8. digitalkin/core/job_manager/taskiq_broker.py +8 -7
  9. digitalkin/core/job_manager/taskiq_job_manager.py +9 -5
  10. digitalkin/core/task_manager/base_task_manager.py +3 -1
  11. digitalkin/core/task_manager/surrealdb_repository.py +13 -7
  12. digitalkin/core/task_manager/task_executor.py +27 -10
  13. digitalkin/core/task_manager/task_session.py +133 -101
  14. digitalkin/grpc_servers/module_server.py +95 -171
  15. digitalkin/grpc_servers/module_servicer.py +133 -27
  16. digitalkin/grpc_servers/utils/grpc_client_wrapper.py +36 -10
  17. digitalkin/grpc_servers/utils/utility_schema_extender.py +106 -0
  18. digitalkin/models/__init__.py +1 -1
  19. digitalkin/models/core/job_manager_models.py +0 -8
  20. digitalkin/models/core/task_monitor.py +23 -1
  21. digitalkin/models/grpc_servers/models.py +95 -8
  22. digitalkin/models/module/__init__.py +26 -13
  23. digitalkin/models/module/base_types.py +61 -0
  24. digitalkin/models/module/module_context.py +279 -13
  25. digitalkin/models/module/module_types.py +29 -109
  26. digitalkin/models/module/setup_types.py +547 -0
  27. digitalkin/models/module/tool_cache.py +230 -0
  28. digitalkin/models/module/tool_reference.py +160 -0
  29. digitalkin/models/module/utility.py +167 -0
  30. digitalkin/models/services/cost.py +22 -1
  31. digitalkin/models/services/registry.py +77 -0
  32. digitalkin/modules/__init__.py +5 -1
  33. digitalkin/modules/_base_module.py +253 -90
  34. digitalkin/modules/archetype_module.py +6 -1
  35. digitalkin/modules/tool_module.py +6 -1
  36. digitalkin/modules/triggers/__init__.py +8 -0
  37. digitalkin/modules/triggers/healthcheck_ping_trigger.py +45 -0
  38. digitalkin/modules/triggers/healthcheck_services_trigger.py +63 -0
  39. digitalkin/modules/triggers/healthcheck_status_trigger.py +52 -0
  40. digitalkin/services/__init__.py +4 -0
  41. digitalkin/services/communication/__init__.py +7 -0
  42. digitalkin/services/communication/communication_strategy.py +87 -0
  43. digitalkin/services/communication/default_communication.py +104 -0
  44. digitalkin/services/communication/grpc_communication.py +264 -0
  45. digitalkin/services/cost/cost_strategy.py +36 -14
  46. digitalkin/services/cost/default_cost.py +61 -1
  47. digitalkin/services/cost/grpc_cost.py +98 -2
  48. digitalkin/services/filesystem/grpc_filesystem.py +9 -2
  49. digitalkin/services/registry/__init__.py +22 -1
  50. digitalkin/services/registry/default_registry.py +156 -4
  51. digitalkin/services/registry/exceptions.py +47 -0
  52. digitalkin/services/registry/grpc_registry.py +382 -0
  53. digitalkin/services/registry/registry_models.py +15 -0
  54. digitalkin/services/registry/registry_strategy.py +106 -4
  55. digitalkin/services/services_config.py +25 -3
  56. digitalkin/services/services_models.py +5 -1
  57. digitalkin/services/setup/default_setup.py +1 -1
  58. digitalkin/services/setup/grpc_setup.py +1 -1
  59. digitalkin/services/storage/grpc_storage.py +1 -1
  60. digitalkin/services/user_profile/__init__.py +11 -0
  61. digitalkin/services/user_profile/grpc_user_profile.py +2 -2
  62. digitalkin/services/user_profile/user_profile_strategy.py +0 -15
  63. digitalkin/utils/__init__.py +40 -0
  64. digitalkin/utils/conditional_schema.py +260 -0
  65. digitalkin/utils/dynamic_schema.py +487 -0
  66. digitalkin/utils/schema_splitter.py +290 -0
  67. {digitalkin-0.3.1.dev1.dist-info → digitalkin-0.3.2a2.dist-info}/METADATA +13 -13
  68. digitalkin-0.3.2a2.dist-info/RECORD +144 -0
  69. {digitalkin-0.3.1.dev1.dist-info → digitalkin-0.3.2a2.dist-info}/WHEEL +1 -1
  70. {digitalkin-0.3.1.dev1.dist-info → digitalkin-0.3.2a2.dist-info}/top_level.txt +1 -0
  71. modules/archetype_with_tools_module.py +232 -0
  72. modules/cpu_intensive_module.py +1 -1
  73. modules/dynamic_setup_module.py +338 -0
  74. modules/minimal_llm_module.py +1 -1
  75. modules/text_transform_module.py +1 -1
  76. monitoring/digitalkin_observability/__init__.py +46 -0
  77. monitoring/digitalkin_observability/http_server.py +150 -0
  78. monitoring/digitalkin_observability/interceptors.py +176 -0
  79. monitoring/digitalkin_observability/metrics.py +201 -0
  80. monitoring/digitalkin_observability/prometheus.py +137 -0
  81. monitoring/tests/test_metrics.py +172 -0
  82. services/filesystem_module.py +7 -5
  83. services/storage_module.py +4 -2
  84. digitalkin/grpc_servers/registry_server.py +0 -65
  85. digitalkin/grpc_servers/registry_servicer.py +0 -456
  86. digitalkin-0.3.1.dev1.dist-info/RECORD +0 -117
  87. {digitalkin-0.3.1.dev1.dist-info → digitalkin-0.3.2a2.dist-info}/licenses/LICENSE +0 -0
@@ -3,6 +3,7 @@
3
3
  from typing import Literal
4
4
 
5
5
  from digitalkin.logger import logger
6
+ from digitalkin.models.services.cost import AmountLimit, QuantityLimit
6
7
  from digitalkin.services.cost.cost_strategy import (
7
8
  CostConfig,
8
9
  CostData,
@@ -24,8 +25,46 @@ class DefaultCost(CostStrategy):
24
25
  setup_version_id: The ID of the setup version this strategy is associated with
25
26
  config: The configuration dictionary for the cost
26
27
  """
27
- super().__init__(mission_id=mission_id, setup_id=setup_id, setup_version_id=setup_version_id, config=config)
28
+ super().__init__(mission_id=mission_id, setup_id=setup_id, setup_version_id=setup_version_id)
29
+ self.config = config
28
30
  self.db: dict[str, list[CostData]] = {}
31
+ self._limits: dict[str, QuantityLimit | AmountLimit] = {}
32
+ self._accumulated: dict[str, float] = {}
33
+
34
+ def set_limits(self, limits: list[QuantityLimit | AmountLimit]) -> None:
35
+ """Set cost limits for this session.
36
+
37
+ Args:
38
+ limits: List of CostLimit objects to enforce.
39
+ """
40
+ self._limits = {limit.name: limit for limit in limits}
41
+ self._accumulated = {}
42
+
43
+ def check_limit(self, cost_config_name: str, quantity: float) -> bool:
44
+ """Check if adding this cost would exceed any limits.
45
+
46
+ Args:
47
+ cost_config_name: Name of the cost config.
48
+ quantity: Quantity to add.
49
+
50
+ Returns:
51
+ True if within limits, False if would exceed.
52
+ """
53
+ limit = self._limits.get(cost_config_name)
54
+ if limit is None:
55
+ return True
56
+
57
+ cost_config = self.config.get(cost_config_name)
58
+ if cost_config is None:
59
+ return True
60
+
61
+ if limit.limit_type == "quantity":
62
+ current = self._accumulated.get(f"{cost_config_name}_quantity", 0)
63
+ return current + quantity <= limit.max_value
64
+
65
+ current = self._accumulated.get(f"{cost_config_name}_amount", 0)
66
+ projected = cost_config.rate * quantity
67
+ return current + projected <= limit.max_value
29
68
 
30
69
  def add(
31
70
  self,
@@ -112,3 +151,24 @@ class DefaultCost(CostStrategy):
112
151
  for cost in self.db[self.mission_id]
113
152
  if (names and cost.name in names) or (cost_types and cost.cost_type in cost_types)
114
153
  ]
154
+
155
+ def get_cost_config(self) -> list[CostConfig]:
156
+ """Get cost configuration from in-memory config.
157
+
158
+ Returns:
159
+ List of CostConfig objects from the config dictionary.
160
+ """
161
+ return list(self.config.values())
162
+
163
+ def set_cost_config(self, configs: list[CostConfig]) -> bool:
164
+ """Store cost configuration in memory.
165
+
166
+ Args:
167
+ configs: List of CostConfig objects to store.
168
+
169
+ Returns:
170
+ True if successfully stored.
171
+ """
172
+ self.config = {config.cost_name: config for config in configs}
173
+ logger.debug("Cost configs stored in memory: %s", self.config)
174
+ return True
@@ -2,13 +2,14 @@
2
2
 
3
3
  from typing import Literal
4
4
 
5
- from digitalkin_proto.agentic_mesh_protocol.cost.v1 import cost_pb2, cost_service_pb2_grpc
5
+ from agentic_mesh_protocol.cost.v1 import cost_pb2, cost_service_pb2_grpc
6
6
  from google.protobuf import json_format
7
7
 
8
8
  from digitalkin.grpc_servers.utils.grpc_client_wrapper import GrpcClientWrapper
9
9
  from digitalkin.grpc_servers.utils.grpc_error_handler import GrpcErrorHandlerMixin
10
10
  from digitalkin.logger import logger
11
11
  from digitalkin.models.grpc_servers.models import ClientConfig
12
+ from digitalkin.models.services.cost import AmountLimit, QuantityLimit
12
13
  from digitalkin.services.cost.cost_strategy import (
13
14
  CostConfig,
14
15
  CostData,
@@ -30,11 +31,49 @@ class GrpcCost(CostStrategy, GrpcClientWrapper, GrpcErrorHandlerMixin):
30
31
  client_config: ClientConfig,
31
32
  ) -> None:
32
33
  """Initialize the cost."""
33
- super().__init__(mission_id=mission_id, setup_id=setup_id, setup_version_id=setup_version_id, config=config)
34
+ super().__init__(mission_id=mission_id, setup_id=setup_id, setup_version_id=setup_version_id)
35
+ self.config = config
36
+ self._limits: dict[str, QuantityLimit | AmountLimit] = {}
37
+ self._accumulated: dict[str, float] = {}
34
38
  channel = self._init_channel(client_config)
35
39
  self.stub = cost_service_pb2_grpc.CostServiceStub(channel)
36
40
  logger.debug("Channel client 'Cost' initialized successfully")
37
41
 
42
+ def set_limits(self, limits: list[QuantityLimit | AmountLimit]) -> None:
43
+ """Set cost limits for this session.
44
+
45
+ Args:
46
+ limits: List of CostLimit objects to enforce.
47
+ """
48
+ self._limits = {limit.name: limit for limit in limits}
49
+ self._accumulated = {}
50
+
51
+ def check_limit(self, cost_config_name: str, quantity: float) -> bool:
52
+ """Check if adding this cost would exceed any limits.
53
+
54
+ Args:
55
+ cost_config_name: Name of the cost config.
56
+ quantity: Quantity to add.
57
+
58
+ Returns:
59
+ True if within limits, False if would exceed.
60
+ """
61
+ limit = self._limits.get(cost_config_name)
62
+ if limit is None:
63
+ return True
64
+
65
+ cost_config = self.config.get(cost_config_name)
66
+ if cost_config is None:
67
+ return True
68
+
69
+ if limit.limit_type == "quantity":
70
+ current = self._accumulated.get(f"{cost_config_name}_quantity", 0)
71
+ return current + quantity <= limit.max_value
72
+
73
+ current = self._accumulated.get(f"{cost_config_name}_amount", 0)
74
+ projected = cost_config.rate * quantity
75
+ return current + projected <= limit.max_value
76
+
38
77
  def add(
39
78
  self,
40
79
  name: str,
@@ -136,3 +175,60 @@ class GrpcCost(CostStrategy, GrpcClientWrapper, GrpcErrorHandlerMixin):
136
175
  ]
137
176
  logger.debug("Filtered costs retrieved with cost_dict: %s", cost_data_list)
138
177
  return [CostData.model_validate(cost_data) for cost_data in cost_data_list]
178
+
179
+ def get_cost_config(self) -> list[CostConfig]:
180
+ """Get cost configuration from the database.
181
+
182
+ Returns:
183
+ List of CostConfig objects from the database.
184
+ """
185
+ with self.handle_grpc_errors("GetCostConfig", CostServiceError):
186
+ request = cost_pb2.GetCostConfigRequest(setup_version_id=self.setup_version_id)
187
+ response: cost_pb2.GetCostConfigResponse = self.exec_grpc_query("GetCostConfig", request)
188
+ config_list = []
189
+ for config in response.configs:
190
+ config_dict = json_format.MessageToDict(
191
+ config,
192
+ preserving_proto_field_name=True,
193
+ always_print_fields_with_no_presence=True,
194
+ )
195
+ # Map proto field names to CostConfig field names
196
+ config_list.append(
197
+ CostConfig(
198
+ cost_name=config_dict.get("name", ""),
199
+ cost_type=config_dict.get("cost_type", "OTHER"),
200
+ description=config_dict.get("description"),
201
+ unit=config_dict.get("unit", ""),
202
+ rate=config_dict.get("rate", 0.0),
203
+ )
204
+ )
205
+ logger.debug("Cost configs retrieved: %s", config_list)
206
+ return config_list
207
+
208
+ def set_cost_config(self, configs: list[CostConfig]) -> bool:
209
+ """Store cost configuration in the database.
210
+
211
+ Args:
212
+ configs: List of CostConfig objects to store.
213
+
214
+ Returns:
215
+ True if successfully stored.
216
+ """
217
+ with self.handle_grpc_errors("SetCostConfig", CostServiceError):
218
+ proto_configs = [
219
+ cost_pb2.CostConfig(
220
+ name=config.cost_name,
221
+ cost_type=config.cost_type,
222
+ description=config.description or "",
223
+ unit=config.unit,
224
+ rate=config.rate,
225
+ )
226
+ for config in configs
227
+ ]
228
+ request = cost_pb2.SetCostConfigRequest(
229
+ setup_version_id=self.setup_version_id,
230
+ configs=proto_configs,
231
+ )
232
+ response: cost_pb2.SetCostConfigResponse = self.exec_grpc_query("SetCostConfig", request)
233
+ logger.debug("Cost configs stored, success: %s", response.success)
234
+ return response.success
@@ -2,7 +2,7 @@
2
2
 
3
3
  from typing import Any, Literal
4
4
 
5
- from digitalkin_proto.agentic_mesh_protocol.filesystem.v1 import filesystem_pb2, filesystem_service_pb2_grpc
5
+ from agentic_mesh_protocol.filesystem.v1 import filesystem_pb2, filesystem_service_pb2_grpc
6
6
  from google.protobuf import struct_pb2
7
7
  from google.protobuf.json_format import MessageToDict
8
8
 
@@ -90,12 +90,19 @@ class GrpcFilesystem(FilesystemStrategy, GrpcClientWrapper, GrpcErrorHandlerMixi
90
90
  Returns:
91
91
  filesystem_pb2.FileFilter: The converted FileFilter proto message
92
92
  """
93
+ context_id = "unknown"
94
+ match filters.context:
95
+ case "setup":
96
+ context_id = self.setup_id
97
+ case "mission":
98
+ context_id = self.mission_id
93
99
  return filesystem_pb2.FileFilter(
94
- **filters.model_dump(exclude={"file_types", "status"}),
100
+ **filters.model_dump(exclude={"file_types", "status", "context"}),
95
101
  file_types=[self._file_type_to_enum(file_type) for file_type in filters.file_types]
96
102
  if filters.file_types
97
103
  else None,
98
104
  status=self._file_status_to_enum(filters.status) if filters.status else None,
105
+ context=context_id,
99
106
  )
100
107
 
101
108
  def __init__(
@@ -1,6 +1,27 @@
1
1
  """This module is responsible for handling the registry service."""
2
2
 
3
+ from digitalkin.models.services.registry import (
4
+ ModuleInfo,
5
+ RegistryModuleStatus,
6
+ RegistryModuleType,
7
+ )
3
8
  from digitalkin.services.registry.default_registry import DefaultRegistry
9
+ from digitalkin.services.registry.exceptions import (
10
+ RegistryModuleNotFoundError,
11
+ RegistryServiceError,
12
+ )
13
+ from digitalkin.services.registry.grpc_registry import GrpcRegistry
14
+ from digitalkin.services.registry.registry_models import ModuleStatusInfo
4
15
  from digitalkin.services.registry.registry_strategy import RegistryStrategy
5
16
 
6
- __all__ = ["DefaultRegistry", "RegistryStrategy"]
17
+ __all__ = [
18
+ "DefaultRegistry",
19
+ "GrpcRegistry",
20
+ "ModuleInfo",
21
+ "ModuleStatusInfo",
22
+ "RegistryModuleNotFoundError",
23
+ "RegistryModuleStatus",
24
+ "RegistryModuleType",
25
+ "RegistryServiceError",
26
+ "RegistryStrategy",
27
+ ]
@@ -1,10 +1,162 @@
1
- """Default registry."""
1
+ """Default registry implementation."""
2
2
 
3
+ from typing import ClassVar
4
+
5
+ from digitalkin.models.services.registry import (
6
+ ModuleInfo,
7
+ RegistryModuleStatus,
8
+ RegistryModuleType,
9
+ )
10
+ from digitalkin.services.registry.exceptions import RegistryModuleNotFoundError
11
+ from digitalkin.services.registry.registry_models import ModuleStatusInfo
3
12
  from digitalkin.services.registry.registry_strategy import RegistryStrategy
4
13
 
5
14
 
6
15
  class DefaultRegistry(RegistryStrategy):
7
- """Default registry strategy."""
16
+ """Default registry strategy using in-memory storage."""
17
+
18
+ _modules: ClassVar[dict[str, ModuleInfo]] = {}
19
+
20
+ def discover_by_id(self, module_id: str) -> ModuleInfo:
21
+ """Get module info by ID.
22
+
23
+ Args:
24
+ module_id: The module identifier.
25
+
26
+ Returns:
27
+ ModuleInfo with module details.
28
+
29
+ Raises:
30
+ RegistryModuleNotFoundError: If module not found.
31
+ """
32
+ if module_id not in self._modules:
33
+ raise RegistryModuleNotFoundError(module_id)
34
+ return self._modules[module_id]
35
+
36
+ def search(
37
+ self,
38
+ name: str | None = None,
39
+ module_type: str | None = None,
40
+ organization_id: str | None = None, # noqa: ARG002
41
+ ) -> list[ModuleInfo]:
42
+ """Search for modules by criteria.
43
+
44
+ Args:
45
+ name: Filter by name (partial match).
46
+ module_type: Filter by type (archetype, tool).
47
+ organization_id: Filter by organization (not used in local storage).
48
+
49
+ Returns:
50
+ List of matching modules.
51
+ """
52
+ results = list(self._modules.values())
53
+
54
+ if name:
55
+ results = [m for m in results if name in m.name]
56
+
57
+ if module_type:
58
+ results = [m for m in results if m.module_type == module_type]
59
+
60
+ return results
61
+
62
+ def get_status(self, module_id: str) -> ModuleStatusInfo:
63
+ """Get module status.
64
+
65
+ Args:
66
+ module_id: The module identifier.
67
+
68
+ Returns:
69
+ ModuleStatusInfo with current status.
70
+
71
+ Raises:
72
+ RegistryModuleNotFoundError: If module not found.
73
+ """
74
+ if module_id not in self._modules:
75
+ raise RegistryModuleNotFoundError(module_id)
76
+
77
+ module = self._modules[module_id]
78
+ return ModuleStatusInfo(
79
+ module_id=module_id,
80
+ status=module.status or RegistryModuleStatus.UNSPECIFIED,
81
+ )
82
+
83
+ def register(
84
+ self,
85
+ module_id: str,
86
+ address: str,
87
+ port: int,
88
+ version: str,
89
+ ) -> ModuleInfo | None:
90
+ """Register a module with the registry.
91
+
92
+ Note: Updates existing module or creates new one in local storage.
93
+
94
+ Args:
95
+ module_id: Unique module identifier.
96
+ address: Network address.
97
+ port: Network port.
98
+ version: Module version.
99
+
100
+ Returns:
101
+ ModuleInfo if successful, None otherwise.
102
+ """
103
+ existing = self._modules.get(module_id)
104
+ self._modules[module_id] = ModuleInfo(
105
+ module_id=module_id,
106
+ module_type=existing.module_type if existing else RegistryModuleType.UNSPECIFIED,
107
+ address=address,
108
+ port=port,
109
+ version=version,
110
+ name=existing.name if existing else module_id,
111
+ status=RegistryModuleStatus.ACTIVE,
112
+ )
113
+ return self._modules[module_id]
114
+
115
+ def heartbeat(self, module_id: str) -> RegistryModuleStatus:
116
+ """Send heartbeat to keep module active.
117
+
118
+ Args:
119
+ module_id: The module identifier.
120
+
121
+ Returns:
122
+ Current module status after heartbeat.
123
+
124
+ Raises:
125
+ RegistryModuleNotFoundError: If module not found.
126
+ """
127
+ if module_id not in self._modules:
128
+ raise RegistryModuleNotFoundError(module_id)
129
+
130
+ module = self._modules[module_id]
131
+ # Update status to ACTIVE on heartbeat
132
+ self._modules[module_id] = ModuleInfo(
133
+ module_id=module.module_id,
134
+ module_type=module.module_type,
135
+ address=module.address,
136
+ port=module.port,
137
+ version=module.version,
138
+ name=module.name,
139
+ status=RegistryModuleStatus.ACTIVE,
140
+ )
141
+ return RegistryModuleStatus.ACTIVE
142
+
143
+ async def deregister(self, module_id: str) -> bool:
144
+ """Deregister a module from the registry.
145
+
146
+ Args:
147
+ module_id: The module identifier to deregister.
148
+
149
+ Returns:
150
+ True if module was removed, False if not found.
151
+ """
152
+ if module_id in self._modules:
153
+ del self._modules[module_id]
154
+ return True
155
+ return False
156
+
157
+ def get_setup(self, setup_id: str) -> None:
158
+ """Get setup info (not supported in default registry).
8
159
 
9
- def get_by_id(self, module_id: str) -> None:
10
- """Get services from the registry."""
160
+ Args:
161
+ setup_id: The setup identifier.
162
+ """
@@ -0,0 +1,47 @@
1
+ """Registry-specific exceptions.
2
+
3
+ This module contains custom exceptions for registry service operations.
4
+ """
5
+
6
+
7
+ class RegistryServiceError(Exception):
8
+ """Base exception for registry service errors."""
9
+
10
+
11
+ class RegistryModuleNotFoundError(RegistryServiceError):
12
+ """Raised when a module is not found in the registry."""
13
+
14
+ def __init__(self, module_id: str) -> None:
15
+ """Initialize the exception.
16
+
17
+ Args:
18
+ module_id: The ID of the module that was not found.
19
+ """
20
+ self.module_id = module_id
21
+ super().__init__(f"Module '{module_id}' not found in registry")
22
+
23
+
24
+ class ModuleAlreadyExistsError(RegistryServiceError):
25
+ """Raised when attempting to register an already-registered module."""
26
+
27
+ def __init__(self, module_id: str) -> None:
28
+ """Initialize the exception.
29
+
30
+ Args:
31
+ module_id: The ID of the module that already exists.
32
+ """
33
+ self.module_id = module_id
34
+ super().__init__(f"Module '{module_id}' already registered")
35
+
36
+
37
+ class InvalidStatusError(RegistryServiceError):
38
+ """Raised when an invalid status is provided."""
39
+
40
+ def __init__(self, status: int) -> None:
41
+ """Initialize the exception.
42
+
43
+ Args:
44
+ status: The invalid status value.
45
+ """
46
+ self.status = status
47
+ super().__init__(f"Invalid module status: {status}")