aiqtoolkit 1.2.0.dev0__py3-none-any.whl → 1.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of aiqtoolkit might be problematic. Click here for more details.

Files changed (220) hide show
  1. aiq/agent/base.py +170 -8
  2. aiq/agent/dual_node.py +1 -1
  3. aiq/agent/react_agent/agent.py +146 -112
  4. aiq/agent/react_agent/prompt.py +1 -6
  5. aiq/agent/react_agent/register.py +36 -35
  6. aiq/agent/rewoo_agent/agent.py +36 -35
  7. aiq/agent/rewoo_agent/register.py +2 -2
  8. aiq/agent/tool_calling_agent/agent.py +3 -7
  9. aiq/agent/tool_calling_agent/register.py +1 -1
  10. aiq/authentication/__init__.py +14 -0
  11. aiq/authentication/api_key/__init__.py +14 -0
  12. aiq/authentication/api_key/api_key_auth_provider.py +92 -0
  13. aiq/authentication/api_key/api_key_auth_provider_config.py +124 -0
  14. aiq/authentication/api_key/register.py +26 -0
  15. aiq/authentication/exceptions/__init__.py +14 -0
  16. aiq/authentication/exceptions/api_key_exceptions.py +38 -0
  17. aiq/authentication/exceptions/auth_code_grant_exceptions.py +86 -0
  18. aiq/authentication/exceptions/call_back_exceptions.py +38 -0
  19. aiq/authentication/exceptions/request_exceptions.py +54 -0
  20. aiq/authentication/http_basic_auth/__init__.py +0 -0
  21. aiq/authentication/http_basic_auth/http_basic_auth_provider.py +81 -0
  22. aiq/authentication/http_basic_auth/register.py +30 -0
  23. aiq/authentication/interfaces.py +93 -0
  24. aiq/authentication/oauth2/__init__.py +14 -0
  25. aiq/authentication/oauth2/oauth2_auth_code_flow_provider.py +107 -0
  26. aiq/authentication/oauth2/oauth2_auth_code_flow_provider_config.py +39 -0
  27. aiq/authentication/oauth2/register.py +25 -0
  28. aiq/authentication/register.py +21 -0
  29. aiq/builder/builder.py +64 -2
  30. aiq/builder/component_utils.py +16 -3
  31. aiq/builder/context.py +37 -0
  32. aiq/builder/eval_builder.py +43 -2
  33. aiq/builder/function.py +44 -12
  34. aiq/builder/function_base.py +1 -1
  35. aiq/builder/intermediate_step_manager.py +6 -8
  36. aiq/builder/user_interaction_manager.py +3 -0
  37. aiq/builder/workflow.py +23 -18
  38. aiq/builder/workflow_builder.py +421 -61
  39. aiq/cli/commands/info/list_mcp.py +103 -16
  40. aiq/cli/commands/sizing/__init__.py +14 -0
  41. aiq/cli/commands/sizing/calc.py +294 -0
  42. aiq/cli/commands/sizing/sizing.py +27 -0
  43. aiq/cli/commands/start.py +2 -1
  44. aiq/cli/entrypoint.py +2 -0
  45. aiq/cli/register_workflow.py +80 -0
  46. aiq/cli/type_registry.py +151 -30
  47. aiq/data_models/api_server.py +124 -12
  48. aiq/data_models/authentication.py +231 -0
  49. aiq/data_models/common.py +35 -7
  50. aiq/data_models/component.py +17 -9
  51. aiq/data_models/component_ref.py +33 -0
  52. aiq/data_models/config.py +60 -3
  53. aiq/data_models/dataset_handler.py +2 -1
  54. aiq/data_models/embedder.py +1 -0
  55. aiq/data_models/evaluate.py +23 -0
  56. aiq/data_models/function_dependencies.py +8 -0
  57. aiq/data_models/interactive.py +10 -1
  58. aiq/data_models/intermediate_step.py +38 -5
  59. aiq/data_models/its_strategy.py +30 -0
  60. aiq/data_models/llm.py +1 -0
  61. aiq/data_models/memory.py +1 -0
  62. aiq/data_models/object_store.py +44 -0
  63. aiq/data_models/profiler.py +1 -0
  64. aiq/data_models/retry_mixin.py +35 -0
  65. aiq/data_models/span.py +187 -0
  66. aiq/data_models/telemetry_exporter.py +2 -2
  67. aiq/embedder/nim_embedder.py +2 -1
  68. aiq/embedder/openai_embedder.py +2 -1
  69. aiq/eval/config.py +19 -1
  70. aiq/eval/dataset_handler/dataset_handler.py +87 -2
  71. aiq/eval/evaluate.py +208 -27
  72. aiq/eval/evaluator/base_evaluator.py +73 -0
  73. aiq/eval/evaluator/evaluator_model.py +1 -0
  74. aiq/eval/intermediate_step_adapter.py +11 -5
  75. aiq/eval/rag_evaluator/evaluate.py +55 -15
  76. aiq/eval/rag_evaluator/register.py +6 -1
  77. aiq/eval/remote_workflow.py +7 -2
  78. aiq/eval/runners/__init__.py +14 -0
  79. aiq/eval/runners/config.py +39 -0
  80. aiq/eval/runners/multi_eval_runner.py +54 -0
  81. aiq/eval/trajectory_evaluator/evaluate.py +22 -65
  82. aiq/eval/tunable_rag_evaluator/evaluate.py +150 -168
  83. aiq/eval/tunable_rag_evaluator/register.py +2 -0
  84. aiq/eval/usage_stats.py +41 -0
  85. aiq/eval/utils/output_uploader.py +10 -1
  86. aiq/eval/utils/weave_eval.py +184 -0
  87. aiq/experimental/__init__.py +0 -0
  88. aiq/experimental/decorators/__init__.py +0 -0
  89. aiq/experimental/decorators/experimental_warning_decorator.py +130 -0
  90. aiq/experimental/inference_time_scaling/__init__.py +0 -0
  91. aiq/experimental/inference_time_scaling/editing/__init__.py +0 -0
  92. aiq/experimental/inference_time_scaling/editing/iterative_plan_refinement_editor.py +147 -0
  93. aiq/experimental/inference_time_scaling/editing/llm_as_a_judge_editor.py +204 -0
  94. aiq/experimental/inference_time_scaling/editing/motivation_aware_summarization.py +107 -0
  95. aiq/experimental/inference_time_scaling/functions/__init__.py +0 -0
  96. aiq/experimental/inference_time_scaling/functions/execute_score_select_function.py +105 -0
  97. aiq/experimental/inference_time_scaling/functions/its_tool_orchestration_function.py +205 -0
  98. aiq/experimental/inference_time_scaling/functions/its_tool_wrapper_function.py +146 -0
  99. aiq/experimental/inference_time_scaling/functions/plan_select_execute_function.py +224 -0
  100. aiq/experimental/inference_time_scaling/models/__init__.py +0 -0
  101. aiq/experimental/inference_time_scaling/models/editor_config.py +132 -0
  102. aiq/experimental/inference_time_scaling/models/its_item.py +48 -0
  103. aiq/experimental/inference_time_scaling/models/scoring_config.py +112 -0
  104. aiq/experimental/inference_time_scaling/models/search_config.py +120 -0
  105. aiq/experimental/inference_time_scaling/models/selection_config.py +154 -0
  106. aiq/experimental/inference_time_scaling/models/stage_enums.py +43 -0
  107. aiq/experimental/inference_time_scaling/models/strategy_base.py +66 -0
  108. aiq/experimental/inference_time_scaling/models/tool_use_config.py +41 -0
  109. aiq/experimental/inference_time_scaling/register.py +36 -0
  110. aiq/experimental/inference_time_scaling/scoring/__init__.py +0 -0
  111. aiq/experimental/inference_time_scaling/scoring/llm_based_agent_scorer.py +168 -0
  112. aiq/experimental/inference_time_scaling/scoring/llm_based_plan_scorer.py +168 -0
  113. aiq/experimental/inference_time_scaling/scoring/motivation_aware_scorer.py +111 -0
  114. aiq/experimental/inference_time_scaling/search/__init__.py +0 -0
  115. aiq/experimental/inference_time_scaling/search/multi_llm_planner.py +128 -0
  116. aiq/experimental/inference_time_scaling/search/multi_query_retrieval_search.py +122 -0
  117. aiq/experimental/inference_time_scaling/search/single_shot_multi_plan_planner.py +128 -0
  118. aiq/experimental/inference_time_scaling/selection/__init__.py +0 -0
  119. aiq/experimental/inference_time_scaling/selection/best_of_n_selector.py +63 -0
  120. aiq/experimental/inference_time_scaling/selection/llm_based_agent_output_selector.py +131 -0
  121. aiq/experimental/inference_time_scaling/selection/llm_based_output_merging_selector.py +159 -0
  122. aiq/experimental/inference_time_scaling/selection/llm_based_plan_selector.py +128 -0
  123. aiq/experimental/inference_time_scaling/selection/threshold_selector.py +58 -0
  124. aiq/front_ends/console/authentication_flow_handler.py +233 -0
  125. aiq/front_ends/console/console_front_end_plugin.py +11 -2
  126. aiq/front_ends/fastapi/auth_flow_handlers/__init__.py +0 -0
  127. aiq/front_ends/fastapi/auth_flow_handlers/http_flow_handler.py +27 -0
  128. aiq/front_ends/fastapi/auth_flow_handlers/websocket_flow_handler.py +107 -0
  129. aiq/front_ends/fastapi/fastapi_front_end_config.py +93 -9
  130. aiq/front_ends/fastapi/fastapi_front_end_controller.py +68 -0
  131. aiq/front_ends/fastapi/fastapi_front_end_plugin.py +14 -1
  132. aiq/front_ends/fastapi/fastapi_front_end_plugin_worker.py +537 -52
  133. aiq/front_ends/fastapi/html_snippets/__init__.py +14 -0
  134. aiq/front_ends/fastapi/html_snippets/auth_code_grant_success.py +35 -0
  135. aiq/front_ends/fastapi/job_store.py +47 -25
  136. aiq/front_ends/fastapi/main.py +2 -0
  137. aiq/front_ends/fastapi/message_handler.py +108 -89
  138. aiq/front_ends/fastapi/step_adaptor.py +2 -1
  139. aiq/llm/aws_bedrock_llm.py +57 -0
  140. aiq/llm/nim_llm.py +2 -1
  141. aiq/llm/openai_llm.py +3 -2
  142. aiq/llm/register.py +1 -0
  143. aiq/meta/pypi.md +12 -12
  144. aiq/object_store/__init__.py +20 -0
  145. aiq/object_store/in_memory_object_store.py +74 -0
  146. aiq/object_store/interfaces.py +84 -0
  147. aiq/object_store/models.py +36 -0
  148. aiq/object_store/register.py +20 -0
  149. aiq/observability/__init__.py +14 -0
  150. aiq/observability/exporter/__init__.py +14 -0
  151. aiq/observability/exporter/base_exporter.py +449 -0
  152. aiq/observability/exporter/exporter.py +78 -0
  153. aiq/observability/exporter/file_exporter.py +33 -0
  154. aiq/observability/exporter/processing_exporter.py +269 -0
  155. aiq/observability/exporter/raw_exporter.py +52 -0
  156. aiq/observability/exporter/span_exporter.py +264 -0
  157. aiq/observability/exporter_manager.py +335 -0
  158. aiq/observability/mixin/__init__.py +14 -0
  159. aiq/observability/mixin/batch_config_mixin.py +26 -0
  160. aiq/observability/mixin/collector_config_mixin.py +23 -0
  161. aiq/observability/mixin/file_mixin.py +288 -0
  162. aiq/observability/mixin/file_mode.py +23 -0
  163. aiq/observability/mixin/resource_conflict_mixin.py +134 -0
  164. aiq/observability/mixin/serialize_mixin.py +61 -0
  165. aiq/observability/mixin/type_introspection_mixin.py +183 -0
  166. aiq/observability/processor/__init__.py +14 -0
  167. aiq/observability/processor/batching_processor.py +316 -0
  168. aiq/observability/processor/intermediate_step_serializer.py +28 -0
  169. aiq/observability/processor/processor.py +68 -0
  170. aiq/observability/register.py +36 -39
  171. aiq/observability/utils/__init__.py +14 -0
  172. aiq/observability/utils/dict_utils.py +236 -0
  173. aiq/observability/utils/time_utils.py +31 -0
  174. aiq/profiler/calc/__init__.py +14 -0
  175. aiq/profiler/calc/calc_runner.py +623 -0
  176. aiq/profiler/calc/calculations.py +288 -0
  177. aiq/profiler/calc/data_models.py +176 -0
  178. aiq/profiler/calc/plot.py +345 -0
  179. aiq/profiler/callbacks/langchain_callback_handler.py +22 -10
  180. aiq/profiler/data_models.py +24 -0
  181. aiq/profiler/inference_metrics_model.py +3 -0
  182. aiq/profiler/inference_optimization/bottleneck_analysis/nested_stack_analysis.py +8 -0
  183. aiq/profiler/inference_optimization/data_models.py +2 -2
  184. aiq/profiler/inference_optimization/llm_metrics.py +2 -2
  185. aiq/profiler/profile_runner.py +61 -21
  186. aiq/runtime/loader.py +9 -3
  187. aiq/runtime/runner.py +23 -9
  188. aiq/runtime/session.py +25 -7
  189. aiq/runtime/user_metadata.py +2 -3
  190. aiq/tool/chat_completion.py +74 -0
  191. aiq/tool/code_execution/README.md +152 -0
  192. aiq/tool/code_execution/code_sandbox.py +151 -72
  193. aiq/tool/code_execution/local_sandbox/.gitignore +1 -0
  194. aiq/tool/code_execution/local_sandbox/local_sandbox_server.py +139 -24
  195. aiq/tool/code_execution/local_sandbox/sandbox.requirements.txt +3 -1
  196. aiq/tool/code_execution/local_sandbox/start_local_sandbox.sh +27 -2
  197. aiq/tool/code_execution/register.py +7 -3
  198. aiq/tool/code_execution/test_code_execution_sandbox.py +414 -0
  199. aiq/tool/mcp/exceptions.py +142 -0
  200. aiq/tool/mcp/mcp_client.py +41 -6
  201. aiq/tool/mcp/mcp_tool.py +3 -2
  202. aiq/tool/register.py +1 -0
  203. aiq/tool/server_tools.py +6 -3
  204. aiq/utils/exception_handlers/automatic_retries.py +289 -0
  205. aiq/utils/exception_handlers/mcp.py +211 -0
  206. aiq/utils/io/model_processing.py +28 -0
  207. aiq/utils/log_utils.py +37 -0
  208. aiq/utils/string_utils.py +38 -0
  209. aiq/utils/type_converter.py +18 -2
  210. aiq/utils/type_utils.py +87 -0
  211. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/METADATA +53 -21
  212. aiqtoolkit-1.2.0rc1.dist-info/RECORD +436 -0
  213. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/WHEEL +1 -1
  214. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/entry_points.txt +3 -0
  215. aiq/front_ends/fastapi/websocket.py +0 -148
  216. aiq/observability/async_otel_listener.py +0 -429
  217. aiqtoolkit-1.2.0.dev0.dist-info/RECORD +0 -316
  218. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/licenses/LICENSE-3rd-party.txt +0 -0
  219. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/licenses/LICENSE.md +0 -0
  220. {aiqtoolkit-1.2.0.dev0.dist-info → aiqtoolkit-1.2.0rc1.dist-info}/top_level.txt +0 -0
aiq/builder/builder.py CHANGED
@@ -20,22 +20,32 @@ from abc import abstractmethod
20
20
  from collections.abc import Sequence
21
21
  from pathlib import Path
22
22
 
23
+ from aiq.authentication.interfaces import AuthProviderBase
23
24
  from aiq.builder.context import AIQContext
24
25
  from aiq.builder.framework_enum import LLMFrameworkEnum
25
26
  from aiq.builder.function import Function
27
+ from aiq.data_models.authentication import AuthProviderBaseConfig
28
+ from aiq.data_models.component_ref import AuthenticationRef
26
29
  from aiq.data_models.component_ref import EmbedderRef
27
30
  from aiq.data_models.component_ref import FunctionRef
31
+ from aiq.data_models.component_ref import ITSStrategyRef
28
32
  from aiq.data_models.component_ref import LLMRef
29
33
  from aiq.data_models.component_ref import MemoryRef
34
+ from aiq.data_models.component_ref import ObjectStoreRef
30
35
  from aiq.data_models.component_ref import RetrieverRef
31
36
  from aiq.data_models.embedder import EmbedderBaseConfig
32
37
  from aiq.data_models.evaluator import EvaluatorBaseConfig
33
38
  from aiq.data_models.function import FunctionBaseConfig
34
39
  from aiq.data_models.function_dependencies import FunctionDependencies
40
+ from aiq.data_models.its_strategy import ITSStrategyBaseConfig
35
41
  from aiq.data_models.llm import LLMBaseConfig
36
42
  from aiq.data_models.memory import MemoryBaseConfig
43
+ from aiq.data_models.object_store import ObjectStoreBaseConfig
37
44
  from aiq.data_models.retriever import RetrieverBaseConfig
45
+ from aiq.experimental.inference_time_scaling.models.stage_enums import PipelineTypeEnum
46
+ from aiq.experimental.inference_time_scaling.models.stage_enums import StageTypeEnum
38
47
  from aiq.memory.interfaces import MemoryEditor
48
+ from aiq.object_store.interfaces import ObjectStore
39
49
  from aiq.retriever.interface import AIQRetriever
40
50
 
41
51
 
@@ -91,6 +101,10 @@ class Builder(ABC): # pylint: disable=too-many-public-methods
91
101
  async def add_llm(self, name: str | LLMRef, config: LLMBaseConfig):
92
102
  pass
93
103
 
104
+ @abstractmethod
105
+ async def get_llm(self, llm_name: str | LLMRef, wrapper_type: LLMFrameworkEnum | str) -> typing.Any:
106
+ pass
107
+
94
108
  async def get_llms(self, llm_names: Sequence[str | LLMRef],
95
109
  wrapper_type: LLMFrameworkEnum | str) -> list[typing.Any]:
96
110
 
@@ -101,11 +115,41 @@ class Builder(ABC): # pylint: disable=too-many-public-methods
101
115
  return list(llms)
102
116
 
103
117
  @abstractmethod
104
- async def get_llm(self, llm_name: str | LLMRef, wrapper_type: LLMFrameworkEnum | str) -> typing.Any:
118
+ def get_llm_config(self, llm_name: str | LLMRef) -> LLMBaseConfig:
105
119
  pass
106
120
 
107
121
  @abstractmethod
108
- def get_llm_config(self, llm_name: str | LLMRef) -> LLMBaseConfig:
122
+ async def add_auth_provider(self, name: str | AuthenticationRef, config: AuthProviderBaseConfig):
123
+ pass
124
+
125
+ @abstractmethod
126
+ async def get_auth_provider(self, auth_provider_name: str | AuthenticationRef) -> AuthProviderBase:
127
+ pass
128
+
129
+ async def get_auth_providers(self, auth_provider_names: list[str | AuthenticationRef]):
130
+
131
+ coros = [self.get_auth_provider(auth_provider_name=n) for n in auth_provider_names]
132
+
133
+ auth_providers = await asyncio.gather(*coros, return_exceptions=False)
134
+
135
+ return list(auth_providers)
136
+
137
+ @abstractmethod
138
+ async def add_object_store(self, name: str | ObjectStoreRef, config: ObjectStoreBaseConfig):
139
+ pass
140
+
141
+ async def get_object_store_clients(self, object_store_names: Sequence[str | ObjectStoreRef]) -> list[ObjectStore]:
142
+ """
143
+ Return a list of all object store clients.
144
+ """
145
+ return list(await asyncio.gather(*[self.get_object_store_client(name) for name in object_store_names]))
146
+
147
+ @abstractmethod
148
+ async def get_object_store_client(self, object_store_name: str | ObjectStoreRef) -> ObjectStore:
149
+ pass
150
+
151
+ @abstractmethod
152
+ def get_object_store_config(self, object_store_name: str | ObjectStoreRef) -> ObjectStoreBaseConfig:
109
153
  pass
110
154
 
111
155
  @abstractmethod
@@ -187,6 +231,24 @@ class Builder(ABC): # pylint: disable=too-many-public-methods
187
231
  async def get_retriever_config(self, retriever_name: str | RetrieverRef) -> RetrieverBaseConfig:
188
232
  pass
189
233
 
234
+ @abstractmethod
235
+ async def add_its_strategy(self, name: str | str, config: ITSStrategyBaseConfig):
236
+ pass
237
+
238
+ @abstractmethod
239
+ async def get_its_strategy(self,
240
+ strategy_name: str | ITSStrategyRef,
241
+ pipeline_type: PipelineTypeEnum,
242
+ stage_type: StageTypeEnum):
243
+ pass
244
+
245
+ @abstractmethod
246
+ async def get_its_strategy_config(self,
247
+ strategy_name: str | ITSStrategyRef,
248
+ pipeline_type: PipelineTypeEnum,
249
+ stage_type: StageTypeEnum) -> ITSStrategyBaseConfig:
250
+ pass
251
+
190
252
  @abstractmethod
191
253
  def get_user_manager(self) -> UserManagerHolder:
192
254
  pass
@@ -21,6 +21,7 @@ from collections.abc import Iterable
21
21
  import networkx as nx
22
22
  from pydantic import BaseModel
23
23
 
24
+ from aiq.data_models.authentication import AuthProviderBaseConfig
24
25
  from aiq.data_models.common import TypedBaseModel
25
26
  from aiq.data_models.component import ComponentGroup
26
27
  from aiq.data_models.component_ref import ComponentRef
@@ -29,8 +30,10 @@ from aiq.data_models.component_ref import generate_instance_id
29
30
  from aiq.data_models.config import AIQConfig
30
31
  from aiq.data_models.embedder import EmbedderBaseConfig
31
32
  from aiq.data_models.function import FunctionBaseConfig
33
+ from aiq.data_models.its_strategy import ITSStrategyBaseConfig
32
34
  from aiq.data_models.llm import LLMBaseConfig
33
35
  from aiq.data_models.memory import MemoryBaseConfig
36
+ from aiq.data_models.object_store import ObjectStoreBaseConfig
34
37
  from aiq.data_models.retriever import RetrieverBaseConfig
35
38
  from aiq.utils.type_utils import DecomposedType
36
39
 
@@ -38,11 +41,14 @@ logger = logging.getLogger(__name__)
38
41
 
39
42
  # Order in which we want to process the component groups
40
43
  _component_group_order = [
44
+ ComponentGroup.AUTHENTICATION,
41
45
  ComponentGroup.EMBEDDERS,
42
46
  ComponentGroup.LLMS,
43
47
  ComponentGroup.MEMORY,
48
+ ComponentGroup.OBJECT_STORES,
44
49
  ComponentGroup.RETRIEVERS,
45
- ComponentGroup.FUNCTIONS
50
+ ComponentGroup.ITS_STRATEGIES,
51
+ ComponentGroup.FUNCTIONS,
46
52
  ]
47
53
 
48
54
 
@@ -95,6 +101,8 @@ def group_from_component(component: TypedBaseModel) -> ComponentGroup | None:
95
101
  component is not a valid runtime instance, None is returned.
96
102
  """
97
103
 
104
+ if (isinstance(component, AuthProviderBaseConfig)):
105
+ return ComponentGroup.AUTHENTICATION
98
106
  if (isinstance(component, EmbedderBaseConfig)):
99
107
  return ComponentGroup.EMBEDDERS
100
108
  if (isinstance(component, FunctionBaseConfig)):
@@ -103,8 +111,12 @@ def group_from_component(component: TypedBaseModel) -> ComponentGroup | None:
103
111
  return ComponentGroup.LLMS
104
112
  if (isinstance(component, MemoryBaseConfig)):
105
113
  return ComponentGroup.MEMORY
114
+ if (isinstance(component, ObjectStoreBaseConfig)):
115
+ return ComponentGroup.OBJECT_STORES
106
116
  if (isinstance(component, RetrieverBaseConfig)):
107
117
  return ComponentGroup.RETRIEVERS
118
+ if (isinstance(component, ITSStrategyBaseConfig)):
119
+ return ComponentGroup.ITS_STRATEGIES
108
120
 
109
121
  return None
110
122
 
@@ -142,7 +154,7 @@ def recursive_componentref_discovery(cls: TypedBaseModel, value: typing.Any,
142
154
  yield from recursive_componentref_discovery(cls, field_data, field_info.annotation)
143
155
  if (decomposed_type.is_union):
144
156
  for arg in decomposed_type.args:
145
- if (isinstance(value, DecomposedType(arg).root)):
157
+ if arg is typing.Any or (isinstance(value, DecomposedType(arg).root)):
146
158
  yield from recursive_componentref_discovery(cls, value, arg)
147
159
  else:
148
160
  for arg in decomposed_type.args:
@@ -243,7 +255,8 @@ def build_dependency_sequence(config: "AIQConfig") -> list[ComponentInstanceData
243
255
  """
244
256
 
245
257
  total_node_count = len(config.embedders) + len(config.functions) + len(config.llms) + len(config.memory) + len(
246
- config.retrievers) + 1 # +1 for the workflow
258
+ config.object_stores) + len(config.retrievers) + len(config.its_strategies) + len(
259
+ config.authentication) + 1 # +1 for the workflow
247
260
 
248
261
  dependency_map: dict
249
262
  dependency_graph: nx.DiGraph
aiq/builder/context.py CHANGED
@@ -22,6 +22,9 @@ from contextvars import ContextVar
22
22
 
23
23
  from aiq.builder.intermediate_step_manager import IntermediateStepManager
24
24
  from aiq.builder.user_interaction_manager import AIQUserInteractionManager
25
+ from aiq.data_models.authentication import AuthenticatedContext
26
+ from aiq.data_models.authentication import AuthFlowType
27
+ from aiq.data_models.authentication import AuthProviderBaseConfig
25
28
  from aiq.data_models.interactive import HumanResponse
26
29
  from aiq.data_models.interactive import InteractionPrompt
27
30
  from aiq.data_models.intermediate_step import IntermediateStep
@@ -61,6 +64,7 @@ class ActiveFunctionContextManager:
61
64
  class AIQContextState(metaclass=Singleton):
62
65
 
63
66
  def __init__(self):
67
+ self.conversation_id: ContextVar[str | None] = ContextVar("conversation_id", default=None)
64
68
  self.input_message: ContextVar[typing.Any] = ContextVar("input_message", default=None)
65
69
  self.user_manager: ContextVar[typing.Any] = ContextVar("user_manager", default=None)
66
70
  self.metadata: ContextVar[RequestAttributes] = ContextVar("request_attributes", default=RequestAttributes())
@@ -75,6 +79,9 @@ class AIQContextState(metaclass=Singleton):
75
79
  | None] = ContextVar(
76
80
  "user_input_callback",
77
81
  default=AIQUserInteractionManager.default_callback_handler)
82
+ self.user_auth_callback: ContextVar[Callable[[AuthProviderBaseConfig, AuthFlowType],
83
+ Awaitable[AuthenticatedContext]]
84
+ | None] = ContextVar("user_auth_callback", default=None)
78
85
 
79
86
  @staticmethod
80
87
  def get() -> "AIQContextState":
@@ -148,6 +155,16 @@ class AIQContext:
148
155
  """
149
156
  return IntermediateStepManager(self._context_state)
150
157
 
158
+ @property
159
+ def conversation_id(self) -> str | None:
160
+ """
161
+ This property retrieves the conversation ID which is the unique identifier for the current chat conversation.
162
+
163
+ Returns:
164
+ str | None
165
+ """
166
+ return self._context_state.conversation_id.get()
167
+
151
168
  @contextmanager
152
169
  def push_active_function(self, function_name: str, input_data: typing.Any | None):
153
170
  """
@@ -213,6 +230,26 @@ class AIQContext:
213
230
  """
214
231
  return self._context_state.active_span_id_stack.get()[-1]
215
232
 
233
+ @property
234
+ def user_auth_callback(self) -> Callable[[AuthProviderBaseConfig, AuthFlowType], Awaitable[AuthenticatedContext]]:
235
+ """
236
+ Retrieves the user authentication callback function from the context state.
237
+
238
+ This property provides access to the user authentication callback function stored in the context state.
239
+ The callback function is responsible for handling user authentication based on the provided configuration.
240
+
241
+ Returns:
242
+ Callable[[AuthenticationBaseConfig], Awaitable[AuthenticatedContext]]: The user authentication
243
+ callback function.
244
+
245
+ Raises:
246
+ RuntimeError: If the user authentication callback is not set in the context.
247
+ """
248
+ callback = self._context_state.user_auth_callback.get()
249
+ if callback is None:
250
+ raise RuntimeError("User authentication callback is not set in the context.")
251
+ return callback
252
+
216
253
  @staticmethod
217
254
  def get() -> "AIQContext":
218
255
  """
@@ -102,14 +102,55 @@ class WorkflowEvalBuilder(WorkflowBuilder, EvalBuilder):
102
102
 
103
103
  return tools
104
104
 
105
+ def _log_build_failure_evaluator(self,
106
+ failing_evaluator_name: str,
107
+ completed_evaluators: list[str],
108
+ remaining_evaluators: list[str],
109
+ original_error: Exception) -> None:
110
+ """
111
+ Log comprehensive evaluator build failure information.
112
+
113
+ Args:
114
+ failing_evaluator_name (str): The name of the evaluator that failed to build
115
+ completed_evaluators (list[str]): List of evaluator names that were successfully built
116
+ remaining_evaluators (list[str]): List of evaluator names still to be built
117
+ original_error (Exception): The original exception that caused the failure
118
+ """
119
+ # Convert evaluator names to (name, type) tuples for consistent logging
120
+ completed_components = [(name, "evaluator") for name in completed_evaluators]
121
+ remaining_components = [(name, "evaluator") for name in remaining_evaluators]
122
+
123
+ # Use the inherited common logging method from WorkflowBuilder
124
+ self._log_build_failure(failing_evaluator_name,
125
+ "evaluator",
126
+ completed_components,
127
+ remaining_components,
128
+ original_error)
129
+
105
130
  async def populate_builder(self, config: AIQConfig):
106
131
  # Skip setting workflow if workflow config is EmptyFunctionConfig
107
132
  skip_workflow = isinstance(config.workflow, EmptyFunctionConfig)
108
133
 
109
134
  await super().populate_builder(config, skip_workflow)
110
- # Instantiate the evaluators
135
+
136
+ # Initialize progress tracking for evaluators
137
+ completed_evaluators = []
138
+ remaining_evaluators = list(config.eval.evaluators.keys())
139
+
140
+ # Instantiate the evaluators with enhanced error logging
111
141
  for name, evaluator_config in config.eval.evaluators.items():
112
- await self.add_evaluator(name, evaluator_config)
142
+ try:
143
+ # Remove from remaining as we start building
144
+ remaining_evaluators.remove(name)
145
+
146
+ await self.add_evaluator(name, evaluator_config)
147
+
148
+ # Add to completed after successful build
149
+ completed_evaluators.append(name)
150
+
151
+ except Exception as e:
152
+ self._log_build_failure_evaluator(name, completed_evaluators, remaining_evaluators, e)
153
+ raise
113
154
 
114
155
  @classmethod
115
156
  @asynccontextmanager
aiq/builder/function.py CHANGED
@@ -48,7 +48,8 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
48
48
  input_schema: type[BaseModel] | None = None,
49
49
  streaming_output_schema: type[BaseModel] | type[None] | None = None,
50
50
  single_output_schema: type[BaseModel] | type[None] | None = None,
51
- converters: list[Callable[[typing.Any], typing.Any]] | None = None):
51
+ converters: list[Callable[[typing.Any], typing.Any]] | None = None,
52
+ instance_name: str | None = None):
52
53
 
53
54
  super().__init__(input_schema=input_schema,
54
55
  streaming_output_schema=streaming_output_schema,
@@ -57,6 +58,7 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
57
58
 
58
59
  self.config = config
59
60
  self.description = description
61
+ self.instance_name = instance_name or config.type
60
62
  self._context = AIQContext.get()
61
63
 
62
64
  def convert(self, value: typing.Any, to_type: type[_T]) -> _T:
@@ -78,6 +80,25 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
78
80
 
79
81
  return self._converter.convert(value, to_type=to_type)
80
82
 
83
+ def try_convert(self, value: typing.Any, to_type: type[_T]) -> _T:
84
+ """
85
+ Converts the given value to the specified type using graceful error handling.
86
+ If conversion fails, returns the original value and continues processing.
87
+
88
+ Parameters
89
+ ----------
90
+ value : typing.Any
91
+ The value to convert.
92
+ to_type : type
93
+ The type to convert the value to.
94
+
95
+ Returns
96
+ -------
97
+ _T
98
+ The converted value, or original value if conversion fails.
99
+ """
100
+ return self._converter.try_convert(value, to_type=to_type)
101
+
81
102
  @abstractmethod
82
103
  async def _ainvoke(self, value: InputT) -> SingleOutputT:
83
104
  pass
@@ -110,7 +131,7 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
110
131
  The output of the function optionally converted to the specified type.
111
132
  """
112
133
 
113
- with self._context.push_active_function(self.config.type,
134
+ with self._context.push_active_function(self.instance_name,
114
135
  input_data=value) as manager: # Set the current invocation context
115
136
  try:
116
137
  converted_input: InputT = self._convert_input(value) # type: ignore
@@ -118,7 +139,7 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
118
139
  result = await self._ainvoke(converted_input)
119
140
 
120
141
  if to_type is not None and not isinstance(result, to_type):
121
- result = self._converter.convert(result, to_type=to_type)
142
+ result = self._converter.try_convert(result, to_type=to_type)
122
143
 
123
144
  manager.set_output(result)
124
145
 
@@ -196,16 +217,25 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
196
217
  The output of the function optionally converted to the specified type.
197
218
  """
198
219
 
199
- with self._context.push_active_function(self.config.type, input_data=value):
220
+ with self._context.push_active_function(self.instance_name, input_data=value) as manager:
200
221
  try:
201
222
  converted_input: InputT = self._convert_input(value) # type: ignore
202
223
 
203
- async for data in self._astream(converted_input):
224
+ # Collect streaming outputs to capture the final result
225
+ final_output: list[typing.Any] = []
204
226
 
227
+ async for data in self._astream(converted_input):
205
228
  if to_type is not None and not isinstance(data, to_type):
206
- yield self._converter.convert(data, to_type=to_type)
229
+ converted_data = self._converter.try_convert(data, to_type=to_type)
230
+ final_output.append(converted_data)
231
+ yield converted_data
207
232
  else:
233
+ final_output.append(data)
208
234
  yield data
235
+
236
+ # Set the final output for intermediate step tracking
237
+ manager.set_output(final_output)
238
+
209
239
  except Exception as e:
210
240
  logger.error("Error with astream in function with input: %s.", value, exc_info=True)
211
241
  raise e
@@ -254,17 +284,17 @@ class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
254
284
 
255
285
  class LambdaFunction(Function[InputT, StreamingOutputT, SingleOutputT]):
256
286
 
257
- def __init__(self, *, config: FunctionBaseConfig, info: FunctionInfo):
287
+ def __init__(self, *, config: FunctionBaseConfig, info: FunctionInfo, instance_name: str | None = None):
258
288
 
259
289
  super().__init__(config=config,
260
290
  description=info.description,
261
291
  input_schema=info.input_schema,
262
292
  streaming_output_schema=info.stream_output_schema,
263
293
  single_output_schema=info.single_output_schema,
264
- converters=info.converters)
294
+ converters=info.converters,
295
+ instance_name=instance_name)
265
296
 
266
297
  self._info = info
267
-
268
298
  self._ainvoke_fn: _InvokeFnT = info.single_fn
269
299
  self._astream_fn: _StreamFnT = info.stream_fn
270
300
 
@@ -284,8 +314,10 @@ class LambdaFunction(Function[InputT, StreamingOutputT, SingleOutputT]):
284
314
  yield x
285
315
 
286
316
  @staticmethod
287
- def from_info(*, config: FunctionBaseConfig,
288
- info: FunctionInfo) -> 'LambdaFunction[InputT, StreamingOutputT, SingleOutputT]':
317
+ def from_info(*,
318
+ config: FunctionBaseConfig,
319
+ info: FunctionInfo,
320
+ instance_name: str | None = None) -> 'LambdaFunction[InputT, StreamingOutputT, SingleOutputT]':
289
321
 
290
322
  input_type: type = info.input_type
291
323
  streaming_output_type = info.stream_output_type
@@ -294,4 +326,4 @@ class LambdaFunction(Function[InputT, StreamingOutputT, SingleOutputT]):
294
326
  class FunctionImpl(LambdaFunction[input_type, streaming_output_type, single_output_type]):
295
327
  pass
296
328
 
297
- return FunctionImpl(config=config, info=info)
329
+ return FunctionImpl(config=config, info=info, instance_name=instance_name)
@@ -373,4 +373,4 @@ class FunctionBase(typing.Generic[InputT, StreamingOutputT, SingleOutputT], ABC)
373
373
  return value
374
374
 
375
375
  # Fallback to the converter
376
- return self._converter.convert(value, to_type=self.input_class)
376
+ return self._converter.try_convert(value, to_type=self.input_class)
@@ -20,7 +20,6 @@ import typing
20
20
  from aiq.data_models.intermediate_step import IntermediateStep
21
21
  from aiq.data_models.intermediate_step import IntermediateStepPayload
22
22
  from aiq.data_models.intermediate_step import IntermediateStepState
23
- from aiq.data_models.invocation_node import InvocationNode
24
23
  from aiq.utils.reactive.observable import OnComplete
25
24
  from aiq.utils.reactive.observable import OnError
26
25
  from aiq.utils.reactive.observable import OnNext
@@ -37,7 +36,7 @@ class OpenStep:
37
36
  step_id: str
38
37
  step_name: str
39
38
  step_type: str
40
- step_parent_id: str | None
39
+ step_parent_id: str
41
40
  prev_stack: list[str]
42
41
  active_stack: list[str]
43
42
 
@@ -153,15 +152,14 @@ class IntermediateStepManager:
153
152
  return
154
153
 
155
154
  parent_step_id = open_step.step_parent_id
155
+ else:
156
+ assert False, "Invalid event state"
156
157
 
157
158
  active_function = self._context_state.active_function.get()
158
159
 
159
- function_ancestry = InvocationNode(function_name=active_function.function_name,
160
- function_id=active_function.function_id,
161
- parent_id=parent_step_id,
162
- parent_name=active_function.parent_name)
163
-
164
- intermediate_step = IntermediateStep(function_ancestry=function_ancestry, payload=payload)
160
+ intermediate_step = IntermediateStep(parent_id=parent_step_id,
161
+ function_ancestry=active_function,
162
+ payload=payload)
165
163
 
166
164
  self._context_state.event_stream.get().on_next(intermediate_step)
167
165
 
@@ -13,6 +13,7 @@
13
13
  # See the License for the specific language governing permissions and
14
14
  # limitations under the License.
15
15
 
16
+ import logging
16
17
  import time
17
18
  import uuid
18
19
 
@@ -22,6 +23,8 @@ from aiq.data_models.interactive import InteractionPrompt
22
23
  from aiq.data_models.interactive import InteractionResponse
23
24
  from aiq.data_models.interactive import InteractionStatus
24
25
 
26
+ logger = logging.getLogger(__name__)
27
+
25
28
 
26
29
  class AIQUserInteractionManager:
27
30
  """
aiq/builder/workflow.py CHANGED
@@ -27,19 +27,12 @@ from aiq.builder.function_base import StreamingOutputT
27
27
  from aiq.builder.llm import LLMProviderInfo
28
28
  from aiq.builder.retriever import RetrieverProviderInfo
29
29
  from aiq.data_models.config import AIQConfig
30
+ from aiq.experimental.inference_time_scaling.models.strategy_base import StrategyBase
30
31
  from aiq.memory.interfaces import MemoryEditor
32
+ from aiq.object_store.interfaces import ObjectStore
33
+ from aiq.observability.exporter.base_exporter import BaseExporter
34
+ from aiq.observability.exporter_manager import ExporterManager
31
35
  from aiq.runtime.runner import AIQRunner
32
- from aiq.utils.optional_imports import TelemetryOptionalImportError
33
- from aiq.utils.optional_imports import try_import_opentelemetry
34
-
35
- # Try to import OpenTelemetry modules
36
- # If the dependencies are not installed, use a dummy span exporter here
37
- try:
38
- opentelemetry = try_import_opentelemetry()
39
- from opentelemetry.sdk.trace.export import SpanExporter
40
- except TelemetryOptionalImportError:
41
- from aiq.utils.optional_imports import DummySpanExporter # pylint: disable=ungrouped-imports
42
- SpanExporter = DummySpanExporter
43
36
 
44
37
  callback_handler_var: ContextVar[Any | None] = ContextVar("callback_handler_var", default=None)
45
38
 
@@ -54,8 +47,10 @@ class Workflow(FunctionBase[InputT, StreamingOutputT, SingleOutputT]):
54
47
  llms: dict[str, LLMProviderInfo] | None = None,
55
48
  embeddings: dict[str, EmbedderProviderInfo] | None = None,
56
49
  memory: dict[str, MemoryEditor] | None = None,
57
- exporters: dict[str, SpanExporter] | None = None,
50
+ object_stores: dict[str, ObjectStore] | None = None,
51
+ telemetry_exporters: dict[str, BaseExporter] | None = None,
58
52
  retrievers: dict[str | None, RetrieverProviderInfo] | None = None,
53
+ its_strategies: dict[str, StrategyBase] | None = None,
59
54
  context_state: AIQContextState):
60
55
 
61
56
  super().__init__(input_schema=entry_fn.input_schema,
@@ -67,14 +62,17 @@ class Workflow(FunctionBase[InputT, StreamingOutputT, SingleOutputT]):
67
62
  self.llms = llms or {}
68
63
  self.embeddings = embeddings or {}
69
64
  self.memory = memory or {}
65
+ self.telemetry_exporters = telemetry_exporters or {}
66
+ self.object_stores = object_stores or {}
70
67
  self.retrievers = retrievers or {}
71
68
 
69
+ self._exporter_manager = ExporterManager.from_exporters(self.telemetry_exporters)
70
+ self.its_strategies = its_strategies or {}
71
+
72
72
  self._entry_fn = entry_fn
73
73
 
74
74
  self._context_state = context_state
75
75
 
76
- self._exporters = exporters or {}
77
-
78
76
  @property
79
77
  def has_streaming_output(self) -> bool:
80
78
 
@@ -91,8 +89,11 @@ class Workflow(FunctionBase[InputT, StreamingOutputT, SingleOutputT]):
91
89
  Called each time we start a new workflow run. We'll create
92
90
  a new top-level workflow span here.
93
91
  """
94
- async with AIQRunner(input_message=message, entry_fn=self._entry_fn,
95
- context_state=self._context_state) as runner:
92
+
93
+ async with AIQRunner(input_message=message,
94
+ entry_fn=self._entry_fn,
95
+ context_state=self._context_state,
96
+ exporter_manager=self._exporter_manager.get()) as runner:
96
97
 
97
98
  # The caller can `yield runner` so they can do `runner.result()` or `runner.result_stream()`
98
99
  yield runner
@@ -121,8 +122,10 @@ class Workflow(FunctionBase[InputT, StreamingOutputT, SingleOutputT]):
121
122
  llms: dict[str, LLMProviderInfo] | None = None,
122
123
  embeddings: dict[str, EmbedderProviderInfo] | None = None,
123
124
  memory: dict[str, MemoryEditor] | None = None,
124
- exporters: dict[str, SpanExporter] | None = None,
125
+ object_stores: dict[str, ObjectStore] | None = None,
126
+ telemetry_exporters: dict[str, BaseExporter] | None = None,
125
127
  retrievers: dict[str | None, RetrieverProviderInfo] | None = None,
128
+ its_strategies: dict[str, StrategyBase] | None = None,
126
129
  context_state: AIQContextState) -> 'Workflow[InputT, StreamingOutputT, SingleOutputT]':
127
130
 
128
131
  input_type: type = entry_fn.input_type
@@ -138,6 +141,8 @@ class Workflow(FunctionBase[InputT, StreamingOutputT, SingleOutputT]):
138
141
  llms=llms,
139
142
  embeddings=embeddings,
140
143
  memory=memory,
141
- exporters=exporters,
144
+ object_stores=object_stores,
145
+ telemetry_exporters=telemetry_exporters,
142
146
  retrievers=retrievers,
147
+ its_strategies=its_strategies,
143
148
  context_state=context_state)