aiqtoolkit 1.2.0rc3__py3-none-any.whl → 1.2.0rc5__py3-none-any.whl

aiq/builder/eval_builder.py

@@ -1,161 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import dataclasses
-import logging
-from contextlib import asynccontextmanager
-from pathlib import Path
-
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.builder.framework_enum import LLMFrameworkEnum
-from aiq.builder.workflow_builder import WorkflowBuilder
-from aiq.cli.type_registry import TypeRegistry
-from aiq.data_models.config import AIQConfig
-from aiq.data_models.config import GeneralConfig
-from aiq.data_models.evaluate import EvalGeneralConfig
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-from aiq.data_models.function import EmptyFunctionConfig
-from aiq.utils.type_utils import override
-
-logger = logging.getLogger(__name__)
-
-
-@dataclasses.dataclass
-class ConfiguredEvaluator:
-    config: EvaluatorBaseConfig
-    instance: EvaluatorInfo
-
-
-class WorkflowEvalBuilder(WorkflowBuilder, EvalBuilder):
-
-    def __init__(self,
-                 general_config: GeneralConfig | None = None,
-                 eval_general_config: EvalGeneralConfig | None = None,
-                 registry: TypeRegistry | None = None):
-        super().__init__(general_config=general_config, registry=registry)
-        self.eval_general_config = eval_general_config
-        self._evaluators: dict[str, ConfiguredEvaluator] = {}
-
-    @override
-    async def add_evaluator(self, name: str, config: EvaluatorBaseConfig):
-        if name in self._evaluators:
-            raise ValueError(f"Evaluator `{name}` already exists in the list of evaluators")
-
-        try:
-            evaluator_info = self._registry.get_evaluator(type(config))
-            info_obj = await self._get_exit_stack().enter_async_context(evaluator_info.build_fn(config, self))
-
-            # Store the evaluator
-            self._evaluators[name] = ConfiguredEvaluator(config=config, instance=info_obj)
-        except Exception as e:
-            logger.error("Error %s adding evaluator `%s` with config `%s`", e, name, config, exc_info=True)
-            raise
-
-    @override
-    def get_evaluator(self, evaluator_name: str) -> EvaluatorInfo:
-
-        if (evaluator_name not in self._evaluators):
-            raise ValueError(f"Evaluator `{evaluator_name}` not found")
-
-        return self._evaluators[evaluator_name].instance
-
-    @override
-    def get_evaluator_config(self, evaluator_name: str) -> EvaluatorBaseConfig:
-
-        if evaluator_name not in self._evaluators:
-            raise ValueError(f"Evaluator `{evaluator_name}` not found")
-
-        # Return the tool configuration object
-        return self._evaluators[evaluator_name].config
-
-    @override
-    def get_max_concurrency(self) -> int:
-        return self.eval_general_config.max_concurrency
-
-    @override
-    def get_output_dir(self) -> Path:
-        return self.eval_general_config.output_dir
-
-    @override
-    def get_all_tools(self, wrapper_type: LLMFrameworkEnum | str):
-        tools = []
-        tool_wrapper_reg = self._registry.get_tool_wrapper(llm_framework=wrapper_type)
-        for fn_name in self._functions:
-            fn = self.get_function(fn_name)
-            try:
-                tools.append(tool_wrapper_reg.build_fn(fn_name, fn, self))
-            except Exception:
-                logger.exception("Error fetching tool `%s`", fn_name, exc_info=True)
-
-        return tools
-
-    def _log_build_failure_evaluator(self,
-                                     failing_evaluator_name: str,
-                                     completed_evaluators: list[str],
-                                     remaining_evaluators: list[str],
-                                     original_error: Exception) -> None:
-        """
-        Log comprehensive evaluator build failure information.
-
-        Args:
-            failing_evaluator_name (str): The name of the evaluator that failed to build
-            completed_evaluators (list[str]): List of evaluator names that were successfully built
-            remaining_evaluators (list[str]): List of evaluator names still to be built
-            original_error (Exception): The original exception that caused the failure
-        """
-        # Convert evaluator names to (name, type) tuples for consistent logging
-        completed_components = [(name, "evaluator") for name in completed_evaluators]
-        remaining_components = [(name, "evaluator") for name in remaining_evaluators]
-
-        # Use the inherited common logging method from WorkflowBuilder
-        self._log_build_failure(failing_evaluator_name,
-                                "evaluator",
-                                completed_components,
-                                remaining_components,
-                                original_error)
-
-    async def populate_builder(self, config: AIQConfig):
-        # Skip setting workflow if workflow config is EmptyFunctionConfig
-        skip_workflow = isinstance(config.workflow, EmptyFunctionConfig)
-
-        await super().populate_builder(config, skip_workflow)
-
-        # Initialize progress tracking for evaluators
-        completed_evaluators = []
-        remaining_evaluators = list(config.eval.evaluators.keys())
-
-        # Instantiate the evaluators with enhanced error logging
-        for name, evaluator_config in config.eval.evaluators.items():
-            try:
-                # Remove from remaining as we start building
-                remaining_evaluators.remove(name)
-
-                await self.add_evaluator(name, evaluator_config)
-
-                # Add to completed after successful build
-                completed_evaluators.append(name)
-
-            except Exception as e:
-                self._log_build_failure_evaluator(name, completed_evaluators, remaining_evaluators, e)
-                raise
-
-    @classmethod
-    @asynccontextmanager
-    async def from_config(cls, config: AIQConfig):
-
-        async with cls(config.general, config.eval.general, registry=None) as builder:
-            await builder.populate_builder(config)
-            yield builder

aiq/builder/evaluator.py

@@ -1,29 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from collections.abc import Callable
-
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-
-
-class EvaluatorInfo:
-
-    def __init__(self, *, config: EvaluatorBaseConfig, evaluate_fn: Callable[[EvalInput], EvalOutput],
-                 description: str):
-        self.config = config
-        self.evaluate_fn = evaluate_fn
-        self.description = description

aiq/builder/framework_enum.py

@@ -1,24 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from enum import Enum
-
-
-class LLMFrameworkEnum(str, Enum):
-    LANGCHAIN = "langchain"
-    LLAMA_INDEX = "llama_index"
-    CREWAI = "crewai"
-    SEMANTIC_KERNEL = "semantic_kernel"
-    AGNO = "agno"

aiq/builder/front_end.py

@@ -1,73 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import typing
-from abc import ABC
-from abc import abstractmethod
-
-from aiq.data_models.front_end import FrontEndConfigT
-
-if (typing.TYPE_CHECKING):
-    from aiq.data_models.config import AIQConfig
-
-
-class FrontEndBase(typing.Generic[FrontEndConfigT], ABC):
-
-    def __init__(self, full_config: "AIQConfig"):
-        """
-        Initializes the FrontEndBase object with the specified AIQ Toolkit configuration.
-
-        Parameters
-        ----------
-        full_config : AIQConfig
-            The configuration object to use for the front end.
-        """
-
-        super().__init__()
-
-        self._full_config: "AIQConfig" = full_config
-        self._front_end_config: FrontEndConfigT = typing.cast(FrontEndConfigT, full_config.general.front_end)
-
-    @property
-    def front_end_config(self) -> FrontEndConfigT:
-        """
-        Returns the front end configuration object extracted from the AIQ Toolkit configuration.
-
-        Returns
-        -------
-        FrontEndConfigT
-            The front end configuration object.
-        """
-        return self._front_end_config
-
-    @property
-    def full_config(self) -> "AIQConfig":
-        """
-        Returns the full AIQ Toolkit configuration object.
-
-        Returns
-        -------
-        AIQConfig
-            The full AIQ Toolkit configuration object.
-        """
-
-        return self._full_config
-
-    @abstractmethod
-    async def run(self):
-        """
-        Runs the specified configuration file, launching the workflow until the front end is complete.
-        """
-        pass

aiq/builder/function.py

@@ -1,344 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-import typing
-from abc import ABC
-from abc import abstractmethod
-from collections.abc import AsyncGenerator
-from collections.abc import Awaitable
-from collections.abc import Callable
-
-from pydantic import BaseModel
-
-from aiq.builder.context import AIQContext
-from aiq.builder.function_base import FunctionBase
-from aiq.builder.function_base import InputT
-from aiq.builder.function_base import SingleOutputT
-from aiq.builder.function_base import StreamingOutputT
-from aiq.builder.function_info import FunctionInfo
-from aiq.data_models.function import FunctionBaseConfig
-
-_InvokeFnT = Callable[[InputT], Awaitable[SingleOutputT]]
-_StreamFnT = Callable[[InputT], AsyncGenerator[StreamingOutputT]]
-
-_T = typing.TypeVar("_T")
-
-logger = logging.getLogger(__name__)
-
-
-class Function(FunctionBase[InputT, StreamingOutputT, SingleOutputT], ABC):
-
-    def __init__(self,
-                 *,
-                 config: FunctionBaseConfig,
-                 description: str | None,
-                 input_schema: type[BaseModel] | None = None,
-                 streaming_output_schema: type[BaseModel] | type[None] | None = None,
-                 single_output_schema: type[BaseModel] | type[None] | None = None,
-                 converters: list[Callable[[typing.Any], typing.Any]] | None = None,
-                 instance_name: str | None = None):
-
-        super().__init__(input_schema=input_schema,
-                         streaming_output_schema=streaming_output_schema,
-                         single_output_schema=single_output_schema,
-                         converters=converters)
-
-        self.config = config
-        self.description = description
-        self.instance_name = instance_name or config.type
-        self._context = AIQContext.get()
-
-    def convert(self, value: typing.Any, to_type: type[_T]) -> _T:
-        """
-        Converts the given value to the specified type using the function's converter.
-
-        Parameters
-        ----------
-        value : typing.Any
-            The value to convert.
-        to_type : type
-            The type to convert the value to.
-
-        Returns
-        -------
-        _T
-            The converted value.
-
-        Raises
-        ------
-        ValueError
-            If the value cannot be converted to the specified type (when `to_type` is specified).
-        """
-
-        return self._converter.convert(value, to_type=to_type)
-
-    def try_convert(self, value: typing.Any, to_type: type[_T]) -> _T | typing.Any:
-        """
-        Converts the given value to the specified type using graceful error handling.
-        If conversion fails, returns the original value and continues processing.
-
-        Parameters
-        ----------
-        value : typing.Any
-            The value to convert.
-        to_type : type
-            The type to convert the value to.
-
-        Returns
-        -------
-        _T | typing.Any
-            The converted value, or original value if conversion fails.
-        """
-        return self._converter.try_convert(value, to_type=to_type)
-
-    @abstractmethod
-    async def _ainvoke(self, value: InputT) -> SingleOutputT:
-        pass
-
-    @typing.overload
-    async def ainvoke(self, value: InputT | typing.Any) -> SingleOutputT:
-        ...
-
-    @typing.overload
-    async def ainvoke(self, value: InputT | typing.Any, to_type: type[_T]) -> _T:
-        ...
-
-    @typing.final
-    async def ainvoke(self, value: InputT | typing.Any, to_type: type | None = None):
-        """
-        Runs the function with the given input and returns a single output from the function. This is the
-        main entry point for running a function.
-
-        Parameters
-        ----------
-        value : InputT | typing.Any
-            The input to the function.
-        to_type : type | None, optional
-            The type to convert the output to using the function's converter. When not specified, the
-            output will match `single_output_type`.
-
-        Returns
-        -------
-        typing.Any
-            The output of the function optionally converted to the specified type.
-
-        Raises
-        ------
-        ValueError
-            If the output of the function cannot be converted to the specified type.
-        """
-
-        with self._context.push_active_function(self.instance_name,
-                                                input_data=value) as manager:  # Set the current invocation context
-            try:
-                converted_input: InputT = self._convert_input(value)
-
-                result = await self._ainvoke(converted_input)
-
-                if to_type is not None and not isinstance(result, to_type):
-                    result = self.convert(result, to_type)
-
-                manager.set_output(result)
-
-                return result
-            except Exception as e:
-                logger.error("Error with ainvoke in function with input: %s.", value, exc_info=True)
-                raise e
-
-    @typing.final
-    async def acall_invoke(self, *args, **kwargs):
-        """
-        A wrapper around `ainvoke` that allows for calling the function with arbitrary arguments and keyword arguments.
-        This is useful in scenarios where the function might be called by an LLM or other system which gives varying
-        inputs to the function. The function will attempt to convert the args and kwargs to the input schema of the
-        function.
-
-        Returns
-        -------
-        SingleOutputT
-            The output of the function.
-        """
-
-        if (len(args) == 1 and not kwargs):
-            # If only one argument is passed, assume it is the input just like ainvoke
-            return await self.ainvoke(value=args[0])
-
-        if (not args and kwargs):
-            # If only kwargs are passed, assume we are calling a function with named arguments in a dict
-            # This will rely on the processing in ainvoke to convert from dict to the correct input type
-            return await self.ainvoke(value=kwargs)
-
-        # Possibly have both args and kwargs, final attempt is to use the input schema object constructor.
-        try:
-            input_obj = self.input_schema(*args, **kwargs)
-
-            return await self.ainvoke(value=input_obj)
-        except Exception as e:
-            logger.error(
-                "Error in acall_invoke() converting input to function schema. Both args and kwargs were "
-                "supplied which could not be converted to the input schema. args: %s\nkwargs: %s\nschema: %s",
-                args,
-                kwargs,
-                self.input_schema)
-            raise e
-
-    @abstractmethod
-    async def _astream(self, value: InputT) -> AsyncGenerator[StreamingOutputT]:
-        yield  # type: ignore
-
-    @typing.overload
-    async def astream(self, value: InputT | typing.Any) -> AsyncGenerator[SingleOutputT]:
-        ...
-
-    @typing.overload
-    async def astream(self, value: InputT | typing.Any, to_type: type[_T]) -> AsyncGenerator[_T]:
-        ...
-
-    @typing.final
-    async def astream(self, value: InputT | typing.Any, to_type: type | None = None):
-        """
-        Runs the function with the given input and returns a stream of outputs from the function. This is the main entry
-        point for running a function with streaming output.
-
-        Parameters
-        ----------
-        value : InputT | typing.Any
-            The input to the function.
-        to_type : type | None, optional
-            The type to convert the output to using the function's converter. When not specified, the
-            output will match `streaming_output_type`.
-
-        Yields
-        ------
-        typing.Any
-            The output of the function optionally converted to the specified type.
-
-        Raises
-        ------
-        ValueError
-            If the output of the function cannot be converted to the specified type (when `to_type` is specified).
-        """
-
-        with self._context.push_active_function(self.instance_name, input_data=value) as manager:
-            try:
-                converted_input: InputT = self._convert_input(value)
-
-                # Collect streaming outputs to capture the final result
-                final_output: list[typing.Any] = []
-
-                async for data in self._astream(converted_input):
-                    if to_type is not None and not isinstance(data, to_type):
-                        converted_data = self.convert(data, to_type=to_type)
-                        final_output.append(converted_data)
-                        yield converted_data
-                    else:
-                        final_output.append(data)
-                        yield data
-
-                # Set the final output for intermediate step tracking
-                manager.set_output(final_output)
-
-            except Exception as e:
-                logger.error("Error with astream in function with input: %s.", value, exc_info=True)
-                raise e
-
-    @typing.final
-    async def acall_stream(self, *args, **kwargs):
-        """
-        A wrapper around `astream` that allows for calling the function with arbitrary arguments and keyword arguments.
-        This is useful in scenarios where the function might be called by an LLM or other system which gives varying
-        inputs to the function. The function will attempt to convert the args and kwargs to the input schema of the
-        function.
-
-        Yields
-        ------
-        StreamingOutputT
-            The output of the function.
-        """
-
-        if (len(args) == 1 and not kwargs):
-            # If only one argument is passed, assume it is the input just like ainvoke
-            async for x in self.astream(value=args[0]):
-                yield x
-
-        elif (not args and kwargs):
-            # If only kwargs are passed, assume we are calling a function with named arguments in a dict
-            # This will rely on the processing in ainvoke to convert from dict to the correct input type
-            async for x in self.astream(value=kwargs):
-                yield x
-
-        # Possibly have both args and kwargs, final attempt is to use the input schema object constructor.
-        else:
-            try:
-                input_obj = self.input_schema(*args, **kwargs)
-
-                async for x in self.astream(value=input_obj):
-                    yield x
-            except Exception as e:
-                logger.error(
-                    "Error in acall_stream() converting input to function schema. Both args and kwargs were "
-                    "supplied which could not be converted to the input schema. args: %s\nkwargs: %s\nschema: %s",
-                    args,
-                    kwargs,
-                    self.input_schema)
-                raise e
-
-
-class LambdaFunction(Function[InputT, StreamingOutputT, SingleOutputT]):
-
-    def __init__(self, *, config: FunctionBaseConfig, info: FunctionInfo, instance_name: str | None = None):
-
-        super().__init__(config=config,
-                         description=info.description,
-                         input_schema=info.input_schema,
-                         streaming_output_schema=info.stream_output_schema,
-                         single_output_schema=info.single_output_schema,
-                         converters=info.converters,
-                         instance_name=instance_name)
-
-        self._info = info
-        self._ainvoke_fn: _InvokeFnT = info.single_fn
-        self._astream_fn: _StreamFnT = info.stream_fn
-
-    @property
-    def has_streaming_output(self) -> bool:
-        return self._astream_fn is not None
-
-    @property
-    def has_single_output(self) -> bool:
-        return self._ainvoke_fn is not None
-
-    async def _ainvoke(self, value: InputT) -> SingleOutputT:
-        return await self._ainvoke_fn(value)
-
-    async def _astream(self, value: InputT) -> AsyncGenerator[StreamingOutputT]:
-        async for x in self._astream_fn(value):
-            yield x
-
-    @staticmethod
-    def from_info(*,
-                  config: FunctionBaseConfig,
-                  info: FunctionInfo,
-                  instance_name: str | None = None) -> 'LambdaFunction[InputT, StreamingOutputT, SingleOutputT]':
-
-        input_type: type = info.input_type
-        streaming_output_type = info.stream_output_type
-        single_output_type = info.single_output_type
-
-        class FunctionImpl(LambdaFunction[input_type, streaming_output_type, single_output_type]):
-            pass
-
-        return FunctionImpl(config=config, info=info, instance_name=instance_name)