sinapsis_core 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. sinapsis_core/__init__.py +0 -0
  2. sinapsis_core/agent/__init__.py +5 -0
  3. sinapsis_core/agent/agent.py +362 -0
  4. sinapsis_core/agent/agent_helpers.py +112 -0
  5. sinapsis_core/agent/base_models.py +361 -0
  6. sinapsis_core/agent/builder.py +112 -0
  7. sinapsis_core/agent/profiler_agent.py +117 -0
  8. sinapsis_core/cli/__init__.py +0 -0
  9. sinapsis_core/cli/arg_parser.py +29 -0
  10. sinapsis_core/cli/display_template_info.py +202 -0
  11. sinapsis_core/cli/run_agent_from_config.py +40 -0
  12. sinapsis_core/context_managers/__init__.py +0 -0
  13. sinapsis_core/context_managers/env_var_override_context_manager.py +39 -0
  14. sinapsis_core/context_managers/working_dir_context_manager.py +57 -0
  15. sinapsis_core/data_containers/__init__.py +0 -0
  16. sinapsis_core/data_containers/annotations.py +152 -0
  17. sinapsis_core/data_containers/data_packet.py +364 -0
  18. sinapsis_core/data_containers/model_types/__init__.py +38 -0
  19. sinapsis_core/data_containers/model_types/_asr_base_models.py +39 -0
  20. sinapsis_core/data_containers/model_types/llm_types.py +189 -0
  21. sinapsis_core/exceptions/agent_exceptions.py +2 -0
  22. sinapsis_core/hooks/__init__.py +15 -0
  23. sinapsis_core/hooks/pre_init_hook_registry.py +225 -0
  24. sinapsis_core/template_base/__init__.py +10 -0
  25. sinapsis_core/template_base/base_models.py +59 -0
  26. sinapsis_core/template_base/copy_data_container.py +46 -0
  27. sinapsis_core/template_base/dynamic_template.py +169 -0
  28. sinapsis_core/template_base/dynamic_template_factory.py +454 -0
  29. sinapsis_core/template_base/dynamic_template_helpers/__init__.py +0 -0
  30. sinapsis_core/template_base/dynamic_template_helpers/builder_methods.py +92 -0
  31. sinapsis_core/template_base/dynamic_template_helpers/signature_parser.py +625 -0
  32. sinapsis_core/template_base/empty_data_container.py +23 -0
  33. sinapsis_core/template_base/input_template.py +8 -0
  34. sinapsis_core/template_base/merge_data_flow.py +33 -0
  35. sinapsis_core/template_base/multi_execute_template.py +65 -0
  36. sinapsis_core/template_base/output_template.py +8 -0
  37. sinapsis_core/template_base/split_data_flow.py +50 -0
  38. sinapsis_core/template_base/template.py +292 -0
  39. sinapsis_core/template_base/transfer_data_container.py +13 -0
  40. sinapsis_core/utils/__init__.py +0 -0
  41. sinapsis_core/utils/base_model_updater.py +148 -0
  42. sinapsis_core/utils/doc_string_generator.py +20 -0
  43. sinapsis_core/utils/env_var_keys.py +249 -0
  44. sinapsis_core/utils/env_var_utils.py +6 -0
  45. sinapsis_core/utils/logging_utils.py +53 -0
  46. sinapsis_core-1.0.0.dist-info/METADATA +239 -0
  47. sinapsis_core-1.0.0.dist-info/RECORD +50 -0
  48. sinapsis_core-1.0.0.dist-info/WHEEL +4 -0
  49. sinapsis_core-1.0.0.dist-info/licenses/LICENSE-APACHE +202 -0
  50. sinapsis_core-1.0.0.dist-info/licenses/LICENSE-MIT +21 -0
File without changes
@@ -0,0 +1,5 @@
1
+ # -*- coding: utf-8 -*-
2
+ #
3
+ from .agent import Agent
4
+
5
+ __all__ = ["Agent"]
@@ -0,0 +1,362 @@
1
+ # -*- coding: utf-8 -*-
2
+ """
3
+ Agent defines the template execution and returns DataContainer
4
+ """
5
+
6
+ from typing import Any, Iterator
7
+
8
+ from pydantic import BaseModel, ConfigDict, Field
9
+
10
+ from sinapsis_core.agent.base_models import (
11
+ AgentAttributes,
12
+ AgentDefinitionType,
13
+ AgentMode,
14
+ AgentSchema,
15
+ SinapsisStatusCodes,
16
+ TemplateSchema,
17
+ TemplateTypes,
18
+ )
19
+ from sinapsis_core.agent.builder import generic_agent_schema_builder, instantiate_template
20
+ from sinapsis_core.context_managers.env_var_override_context_manager import (
21
+ AgentEnvVarContextManager,
22
+ )
23
+ from sinapsis_core.data_containers.data_packet import DataContainer
24
+ from sinapsis_core.exceptions.agent_exceptions import AgentCancelledException
25
+ from sinapsis_core.hooks import PreInitMetaclass
26
+ from sinapsis_core.template_base import Template
27
+ from sinapsis_core.template_base.base_models import TemplateMetadata
28
+ from sinapsis_core.utils.logging_utils import sinapsis_logger
29
+
30
+
31
+ class AgentTemplateEntry(BaseModel):
32
+ metadata: TemplateSchema
33
+ template_instance: Template
34
+ processed_output: DataContainer | list[DataContainer] = Field(default_factory=DataContainer)
35
+ model_config = ConfigDict(arbitrary_types_allowed=True)
36
+
37
+
38
+ class Agent(metaclass=PreInitMetaclass):
39
+ """The Agent establishes and manages the execution of templates
40
+ by defining a structure that includes the Agent attributes and the
41
+ list of templates to be executed. The agent already organizes the way
42
+ templates should be executed by inspecting the template_input field
43
+ and establishes how the data should be transported from template to template
44
+ """
45
+
46
+ PROFILER_ENABLED = False
47
+ LAZY_INITIALIZATION = False
48
+
49
+ def __init__(self, agent_definition: AgentDefinitionType) -> None:
50
+ """
51
+ Constructor method for the Agent.
52
+ Before executing the agent, the templates are sorted and validated
53
+ """
54
+ self.agent_schema: AgentSchema = self._make_agent_schema(agent_definition)
55
+ self.logger = sinapsis_logger.bind(agent_name=self.agent_name)
56
+
57
+ self.topological_sort: dict[str, AgentTemplateEntry] = {}
58
+
59
+ self.processed_containers: dict[str, DataContainer]
60
+ self.is_initialized: bool = False
61
+ self.context_manager = self._make_context_manager()
62
+
63
+ if not self.LAZY_INITIALIZATION:
64
+ self._lazy_init()
65
+ self.__num_templates: int = len(self.agent_schema.templates)
66
+ self.__ancestor_finished_status: list[bool] = [False] * self.num_templates
67
+ self.agent_status: SinapsisStatusCodes = SinapsisStatusCodes.READY
68
+
69
+ def _make_context_manager(self) -> AgentEnvVarContextManager:
70
+ return AgentEnvVarContextManager(self.agent_attributes)
71
+
72
+ def _make_agent_schema(self, agent_definition: AgentDefinitionType) -> AgentSchema:
73
+ return generic_agent_schema_builder(agent_definition, self.PROFILER_ENABLED)
74
+
75
+ def _lazy_init(self) -> None:
76
+ """Instantiate all the templates composing the Agent, initializing the Agent as well,
77
+ and sets the is_initialized flag to True.
78
+ Raise:
79
+ ValueError: If there is no topological sort, there are no templates in the Agent.
80
+
81
+ """
82
+ self.__instantiate_templates()
83
+ self.is_initialized = True
84
+ if not self.topological_sort:
85
+ raise ValueError("The agent has no templates")
86
+ self.logger.info("Agent and templates initialized")
87
+
88
+ def update_template_attribute(self, template_path: str, attribute_name: str, attribute_value: Any) -> None:
89
+ """Updates a template attribute using the template_path in the Agent
90
+ Args:
91
+ template_path (str): Path to the template in the topological sort of the Agent.
92
+ attribute_name (str): Name of the attribute to be changed.
93
+ attribute_value (Any): New value for the attribute.
94
+ """
95
+ first_template = template_path.split(".")[0]
96
+
97
+ if first_template in self.topological_sort:
98
+ template_instance = self.topological_sort[first_template].template_instance
99
+ remaining_path = ".".join(template_path.split(".")[1:])
100
+ full_attribute_path = f"{remaining_path}.{attribute_name}" if remaining_path else attribute_name
101
+ template_instance.update_attribute(full_attribute_path, attribute_value)
102
+ else:
103
+ self.logger.warning(f"Template `{template_path}` is not in the agent `{self.agent_name}`")
104
+
105
+ def _make_template_metadata(self, template_name: str) -> TemplateMetadata:
106
+ return TemplateMetadata(
107
+ template_instance_name=template_name,
108
+ agent_name=self.agent_name,
109
+ user_id=self.agent_attributes.user_id,
110
+ agent_id=self.agent_attributes.agent_id,
111
+ )
112
+
113
+ @property
114
+ def agent_name(self) -> str:
115
+ """the name of the agent"""
116
+ return self.agent_schema.agent.name
117
+
118
+ @property
119
+ def agent_attributes(self) -> AgentAttributes:
120
+ """the attributes of the agent"""
121
+ return self.agent_schema.agent.attributes
122
+
123
+ @property
124
+ def output_template_name(self) -> str:
125
+ """name of the output template"""
126
+
127
+ return self.agent_schema.sorted_templates[-1].template_name
128
+
129
+ def _make_template_entry(self, template_schema: TemplateSchema, template_instance: Template) -> AgentTemplateEntry:
130
+ _ = self
131
+ return AgentTemplateEntry(metadata=template_schema, template_instance=template_instance)
132
+
133
+ def __instantiate_templates(self) -> None:
134
+ """For each of the templates in the Agent, create an instance and add the entry to the
135
+ topological sort.
136
+ Finally, sets the order of execution if the Agent log.
137
+ """
138
+ for template_schema in self.agent_schema.sorted_templates:
139
+ if "metadata" not in template_schema.attributes or template_schema.attributes["metadata"] is None:
140
+ template_schema.attributes["metadata"] = self._make_template_metadata(template_schema.template_name)
141
+ with self.context_manager:
142
+ template_instance = instantiate_template(template_schema)
143
+ self.topological_sort[template_schema.template_name] = self._make_template_entry(
144
+ template_schema, template_instance
145
+ )
146
+ self.logger.debug(f"Initialized template: {template_schema.template_name}")
147
+ self._log_agent_execution_order()
148
+
149
+ @property
150
+ def num_templates(self) -> int:
151
+ """Number of templates in the agent"""
152
+ if self.is_initialized:
153
+ return self.__num_templates
154
+ return len(self.agent_schema.sorted_templates)
155
+
156
+ @num_templates.setter
157
+ def num_templates(self, value: int) -> None:
158
+ self.__num_templates = value
159
+
160
+ def _log_agent_execution_order(self) -> None:
161
+ """Logs the order of execution of the Agent"""
162
+ self.logger.debug("Execution Order")
163
+ for idx, template_name in enumerate(self.topological_sort.keys()):
164
+ logging_msg = f"Order: <<{idx}>>, template name: <<{template_name}>>"
165
+ self.logger.debug(logging_msg)
166
+
167
+ def reset_state(self, template_name: str | None = None) -> None:
168
+ """
169
+ Clears template state and finished status tracking for templates in the agent's topological sort.
170
+ For a specific template, it resets only that template's state and finished status, supporting nested paths
171
+ using dot notation. For all templates (when no template_name is provided), it resets state and
172
+ finished status across all templates. Templates must override this method to implement
173
+ their specific state reset logic, otherwise no state changes occur if only inherited from base template.
174
+
175
+ Args:
176
+ template_name (str | None): The name of the template to clear state for.
177
+ Can be a nested path using dot notation (e.g. "ForLoop_0.ForLoop_1").
178
+ If not specified, all templates are cleared.
179
+ """
180
+
181
+ if template_name:
182
+ first_template = template_name.split(".")[0]
183
+ if first_template in self.topological_sort:
184
+ template_idx = list(self.topological_sort.keys()).index(first_template)
185
+ self.__ancestor_finished_status[template_idx] = False
186
+ self.topological_sort[first_template].template_instance.reset_state(template_name)
187
+ else:
188
+ self.logger.warning(f"Template `{template_name}` is not in the agent `{self.agent_name}`")
189
+ else:
190
+ for template_entry in self.topological_sort.values():
191
+ template_entry.template_instance.reset_state()
192
+ self.__ancestor_finished_status = [False] * self.num_templates
193
+
194
+ def signal_block_if_needed(self, idx: int, template_name: str, template_instance: Template) -> None:
195
+ """Defines whether the blocking mode should be set to True"""
196
+ if self.num_templates == 1:
197
+ template_instance.blocking_mode = True
198
+
199
+ elif idx > 0 and all(self.__ancestor_finished_status[0:idx]) and not template_instance.blocking_mode:
200
+ log_msg = f"Signaling block mode for {template_name} no: {idx + 1}/{self.num_templates}"
201
+ self.logger.info(log_msg)
202
+ template_instance.blocking_mode = True
203
+
204
+ def get_input_container(
205
+ self, template_entry: AgentTemplateEntry, container: DataContainer
206
+ ) -> DataContainer | list[DataContainer]:
207
+ """Defines the input container for the Agent.
208
+ If the Agent only has one template, or if the template is of type InputTemplate
209
+ returns the incoming DataContainer.
210
+ If the template is of type DataJoin, return the list of DataContainers that are returned from
211
+ joining the Data.
212
+ If the template is of type DataSplit, return the processed DataContainers
213
+ Otherwise, return the processed DataContainer.
214
+ Args:
215
+ template_entry (AgentTemplateEntry): Schema for the Agent's template entry
216
+ container (DataContainer): the container to be processed first.
217
+ Returns:
218
+ DataContainer | list[DataContainer]: Container or list of Containers to be returned.
219
+ """
220
+ if template_entry.metadata.template_type == TemplateTypes.INPUT_TEMPLATE or self.num_templates == 1:
221
+ return container
222
+
223
+ if template_entry.metadata.template_type == TemplateTypes.DATA_JOIN_TEMPLATE:
224
+ input_containers = [
225
+ self.topological_sort[ancestor_name].processed_output
226
+ for ancestor_name in template_entry.metadata.template_input
227
+ ]
228
+ return input_containers
229
+
230
+ elif (
231
+ self.topological_sort[template_entry.metadata.template_input].metadata.template_type
232
+ == TemplateTypes.DATA_SPLIT_TEMPLATE
233
+ ):
234
+ if self.topological_sort[template_entry.metadata.template_input].processed_output:
235
+ return self.topological_sort[template_entry.metadata.template_input].processed_output.pop()
236
+ else:
237
+ self.logger.warning(
238
+ f"Template `{template_entry.metadata.template_input}` "
239
+ f"has no processed_output left, will return original container"
240
+ )
241
+ return container
242
+
243
+ return self.topological_sort[template_entry.metadata.template_input].processed_output
244
+
245
+ def all_templates_finished(self) -> bool:
246
+ """Checks if all the templates have finished the execution"""
247
+ return all(self.__ancestor_finished_status)
248
+
249
+ def _record_template_finished_status(self, template_idx: int, template_entry: AgentTemplateEntry) -> None:
250
+ """Determines the finished status of a given template
251
+ Args:
252
+ template_idx (int): index of the template in the topological sort.
253
+ template_entry (AgentTemplateEntry): Entry of the template to be checked.
254
+ """
255
+ has_finished = True
256
+ if template_entry.template_instance.num_elements() > 0:
257
+ has_finished = False
258
+ self.__ancestor_finished_status[template_idx] = has_finished
259
+
260
+ @property
261
+ def output_container(self) -> DataContainer | list[DataContainer]:
262
+ return self.topological_sort[self.output_template_name].processed_output
263
+
264
+ def _execute_template(self, template_entry: AgentTemplateEntry, container: DataContainer) -> None:
265
+ input_container = self.get_input_container(template_entry, container)
266
+ with self.context_manager:
267
+ template_entry.processed_output = template_entry.template_instance.execute(input_container)
268
+
269
+ def _execute_generator(self, container: DataContainer) -> Iterator[DataContainer]:
270
+ execution_running = True
271
+ while execution_running:
272
+ processed_container = self._single_execute(container)
273
+ execution_running = not self.all_templates_finished()
274
+ container = DataContainer()
275
+ yield processed_container
276
+
277
+ def _shut_down_all_templates(self) -> None:
278
+ self.logger.info("All templates returned finished, stopping execution...")
279
+ for template_entry in self.topological_sort.values():
280
+ template_entry.template_instance._shut_down()
281
+ self.agent_status = SinapsisStatusCodes.DONE
282
+
283
+ def _single_execute(self, container: DataContainer, is_production: bool = False) -> DataContainer:
284
+ if self.agent_status == SinapsisStatusCodes.CANCELLED:
285
+ raise AgentCancelledException(
286
+ f"Agent {self.agent_name} single execute was called but the agent is cancelled"
287
+ )
288
+
289
+ self.agent_status = SinapsisStatusCodes.PROCESSING
290
+
291
+ for idx, (template_name, this_template_entry) in enumerate(self.topological_sort.items()):
292
+ self.logger.info(f"Executing template: {template_name}")
293
+ if is_production:
294
+ self._execute_template(this_template_entry, container)
295
+
296
+ else:
297
+ self.signal_block_if_needed(idx, template_name, this_template_entry.template_instance)
298
+ self._execute_template(this_template_entry, container)
299
+ self._record_template_finished_status(idx, this_template_entry)
300
+ self.logger.info(f"Template {template_name} has been executed")
301
+ if self.agent_attributes.shut_down_templates and self.all_templates_finished():
302
+ self._shut_down_all_templates()
303
+
304
+ return self.output_container
305
+
306
+ def _execute(self, container: DataContainer | None = None) -> None:
307
+ """
308
+ In production mode we execute forever, always in nonblocking mode.
309
+ If we need blocking mode it should be implemented per template,
310
+ e.g. DataReader should block until a new data item is available
311
+ to be propagated (e.g. reading from a camera, from user input,
312
+ from a get request, etc.) Similarly, to obtain results, a corresponding
313
+ template should take care of publishing such results, e.g. log
314
+ to a file, post http request, etc.
315
+
316
+ Args:
317
+ container (DataContainer): Container with the outputs of the template
318
+ """
319
+ execution_running = True
320
+ while execution_running:
321
+ _ = self._single_execute(container, is_production=True)
322
+ container = DataContainer()
323
+
324
+ def _forward_execute(self, container: DataContainer) -> DataContainer:
325
+ """
326
+ Executes all the templates defined in the template list without registering
327
+ their finished status. Additionally, all templates are executed in non-blocking
328
+ mode. This execution mode is not intended to be used for producer templates
329
+ like datareaders, loaders, etc.
330
+
331
+ Args:
332
+ container (DataContainer): Container with the outputs of the template
333
+ """
334
+
335
+ container = self._single_execute(container, is_production=True)
336
+ return container
337
+
338
+ def __call__(
339
+ self, container: DataContainer | None = None, agent_mode: AgentMode | None = None
340
+ ) -> DataContainer | Iterator[DataContainer] | None:
341
+ container = container if container else DataContainer()
342
+
343
+ agent_mode = agent_mode or self.agent_attributes.agent_mode
344
+
345
+ if agent_mode == AgentMode.LOOP_FOREVER:
346
+ self._execute(container)
347
+ if agent_mode == AgentMode.GENERATOR:
348
+ return self._execute_generator(container)
349
+ if agent_mode == AgentMode.SINGLE:
350
+ return self._single_execute(container)
351
+ if agent_mode == AgentMode.FORWARD:
352
+ return self._forward_execute(container)
353
+
354
+ raise ValueError(f"Unsupported agent mode: {agent_mode}")
355
+
356
+ def cancel_agent_execution(self) -> None:
357
+ self.agent_status = SinapsisStatusCodes.CANCELLED
358
+ self.__ancestor_finished_status = [True] * self.num_templates
359
+ for template_entry in self.topological_sort.values():
360
+ template_entry.template_instance._shut_down()
361
+
362
+ raise AgentCancelledException(f"Agent {self.agent_name} cancel_agent_execution method was called but failed")
@@ -0,0 +1,112 @@
1
+ # -*- coding: utf-8 -*-
2
+ from collections import defaultdict, deque
3
+
4
+
5
+ def sort_agent_execution(template_neighbors_mapping: dict[str, list[str]]) -> list[str]:
6
+ """Applies Kahn`s algorithm for agent topological sorting
7
+
8
+ template_neighbors_mapping (dict[str, list]): Dictionary of template identifiers
9
+ with the following structure:
10
+
11
+ {
12
+ "template_1": ["template_2"]
13
+ "template_2": ["template_3"]
14
+ "template_3": []
15
+ }
16
+
17
+ Returns:
18
+ sorted_templates (list[str]): list of template names sorted in order of execution
19
+ """
20
+
21
+ # Vector to store number of inputs per each template
22
+ indegree = dict.fromkeys(template_neighbors_mapping, 0)
23
+ for from_template in template_neighbors_mapping:
24
+ for to_template in template_neighbors_mapping[from_template]:
25
+ indegree[to_template] += 1
26
+
27
+ # Queue to store vertices with indegree 0
28
+ indegree_queue: deque[str] = deque()
29
+ for template in template_neighbors_mapping:
30
+ if indegree[template] == 0:
31
+ indegree_queue.append(template)
32
+
33
+ sorted_templates = []
34
+ while indegree_queue:
35
+ node = indegree_queue.popleft()
36
+ sorted_templates.append(node)
37
+ # Decrease indegree of adjacent vertices as the current node is in topological order
38
+ for adjacent in template_neighbors_mapping[node]:
39
+ indegree[adjacent] -= 1
40
+ # If indegree becomes 0, push it to the queue
41
+ if indegree[adjacent] == 0:
42
+ indegree_queue.append(adjacent)
43
+
44
+ # Check for cycle
45
+ if len(sorted_templates) != len(template_neighbors_mapping):
46
+ raise RuntimeError("Cycle detected in agent definition!")
47
+
48
+ return sorted_templates
49
+
50
+
51
+ def valid_template_input(template_input: str | list[str], valid_templates: list[str]) -> bool:
52
+ """
53
+ Validates if a template_input is defined as Template in Agent
54
+
55
+ Args:
56
+ template_input (str | list[str]): template_input to be evaluated
57
+ valid_templates (list[str]): list of defined template names in agent
58
+
59
+ Returns:
60
+ bool: validation result
61
+ """
62
+
63
+ if isinstance(template_input, str) and template_input not in valid_templates:
64
+ return False
65
+
66
+ if isinstance(template_input, list):
67
+ for temp_input in template_input:
68
+ if not valid_template_input(temp_input, valid_templates):
69
+ return False
70
+ return True
71
+
72
+
73
+ def get_template_neighbors_mapping(template_list: list) -> dict[str, list[str]]:
74
+ """
75
+ Uses a list of Template to create a dict with the following structure
76
+ template_neighbors_mapping = {
77
+ template_0: [template_1, template_2]
78
+ template_1: [template_2]
79
+ template_2: []
80
+ template_3: []
81
+ }
82
+
83
+ Where each key-value pair represents a connection between
84
+ a template_n and a list of other templates that uses this
85
+ template_n as input.
86
+
87
+ Returns a list of template names sorted in order of execution
88
+
89
+ Args:
90
+ template_list (list[TemplateDefinition])
91
+
92
+ Returns:
93
+ dict[str, list[str]]
94
+ """
95
+
96
+ template_neighbors_mapping: dict[str, list] = defaultdict(list)
97
+ valid_template_names = [template.template_name for template in template_list]
98
+
99
+ for template in template_list:
100
+ if template.template_name not in template_neighbors_mapping:
101
+ template_neighbors_mapping[template.template_name] = []
102
+
103
+ if template.template_input and not valid_template_input(template.template_input, valid_template_names):
104
+ logging_msg = f"Invalid template_input: <<{template.template_input}>>in <<{template.template_name}>>"
105
+ raise (ValueError(logging_msg))
106
+
107
+ if isinstance(template.template_input, str):
108
+ template_neighbors_mapping[template.template_input].append(template.template_name)
109
+ elif isinstance(template.template_input, list):
110
+ for template_input in template.template_input:
111
+ template_neighbors_mapping[template_input].append(template.template_name)
112
+ return template_neighbors_mapping