aiqtoolkit 1.1.0a20250516__py3-none-any.whl → 1.1.0a20251020__py3-none-any.whl

aiq/eval/rag_evaluator/evaluate.py

@@ -1,138 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from collections.abc import Sequence
-
-from ragas import EvaluationDataset
-from ragas import SingleTurnSample
-from ragas.dataset_schema import EvaluationResult
-from ragas.llms import LangchainLLMWrapper
-from ragas.metrics import Metric
-from tqdm import tqdm
-
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
-
-logger = logging.getLogger(__name__)
-
-
-class RAGEvaluator:
-
-    def __init__(self, evaluator_llm: LangchainLLMWrapper, metrics: Sequence[Metric]):
-        self.evaluator_llm = evaluator_llm
-        self.metrics = metrics
-
-    @staticmethod
-    def eval_input_to_ragas(eval_input: EvalInput) -> EvaluationDataset:
-        """Converts EvalInput into a Ragas-compatible EvaluationDataset."""
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
-
-        samples = []
-
-        intermediate_step_adapter = IntermediateStepAdapter()
-        for item in eval_input.eval_input_items:
-            # Extract required fields from EvalInputItem
-            user_input = item.input_obj  # Assumes input_obj is a string (modify if needed)
-            reference = item.expected_output_obj  # Reference correct answer
-            response = item.output_obj  # Model's generated response
-
-            # Handle context extraction from trajectory if available
-            reference_contexts = [""]  # Default to empty context
-            # implement context extraction from expected_trajectory
-
-            retrieved_contexts = intermediate_step_adapter.get_context(item.trajectory)
-            # implement context extraction from expected_trajectory
-
-            # Create a SingleTurnSample
-            sample = SingleTurnSample(
-                user_input=user_input,
-                reference=reference,
-                response=response,
-                reference_contexts=reference_contexts,
-                retrieved_contexts=retrieved_contexts,
-            )
-            samples.append(sample)
-
-        return EvaluationDataset(samples=samples)
-
-    def ragas_to_eval_output(self, eval_input: EvalInput, results_dataset: EvaluationResult | None) -> EvalOutput:
-        """Converts the ragas EvaluationResult to aiq EvalOutput"""
-
-        if not results_dataset:
-            logger.error("Ragas evaluation failed with no results")
-            return EvalOutput(average_score=0.0, eval_output_items=[])
-
-        scores: list[dict[str, float]] = results_dataset.scores
-        if not scores:
-            logger.error("Ragas returned empty score list")
-            return EvalOutput(average_score=0.0, eval_output_items=[])
-
-        # Convert from list of dicts to dict of lists
-        scores_dict = {metric: [score[metric] for score in scores] for metric in scores[0]}
-
-        # Compute the average of each metric
-        average_scores = {metric: sum(values) / len(values) for metric, values in scores_dict.items()}
-
-        # Extract the first (and only) metric's average score
-        first_avg_score = next(iter(average_scores.values()))
-        first_metric_name = list(scores_dict.keys())[0]
-
-        df = results_dataset.to_pandas()
-        # Get id from eval_input if df size matches number of eval_input_items
-        if len(eval_input.eval_input_items) >= len(df):
-            ids = [item.id for item in eval_input.eval_input_items]  # Extract IDs
-        else:
-            ids = df["user_input"].tolist()  # Use "user_input" as ID fallback
-
-        # Construct EvalOutputItem list
-        eval_output_items = [
-            EvalOutputItem(
-                id=ids[i],
-                score=getattr(row, first_metric_name, 0.0),
-                reasoning={
-                    key:
-                        getattr(row, key, None)  # Use getattr to safely access attributes
-                    for key in ["user_input", "reference", "response", "retrieved_contexts"]
-                }) for i, row in enumerate(df.itertuples(index=False))
-        ]
-        # Return EvalOutput
-        return EvalOutput(average_score=first_avg_score, eval_output_items=eval_output_items)
-
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        """Run Ragas metrics evaluation on the provided EvalInput"""
-        from ragas import evaluate as ragas_evaluate
-
-        ragas_dataset = self.eval_input_to_ragas(eval_input)
-        tqdm_position = TqdmPositionRegistry.claim()
-        first_metric_name = self.metrics[0].name
-        pbar = tqdm(total=len(ragas_dataset), desc=f"Evaluating Ragas {first_metric_name}", position=tqdm_position)
-        try:
-            results_dataset = ragas_evaluate(dataset=ragas_dataset,
-                                             metrics=self.metrics,
-                                             show_progress=True,
-                                             llm=self.evaluator_llm,
-                                             _pbar=pbar)
-        except Exception as e:
-            # On exception we still continue with other evaluators. Log and return an avg_score of 0.0
-            logger.exception("Error evaluating ragas metric, Error: %s", e, exc_info=True)
-            results_dataset = None
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        return self.ragas_to_eval_output(eval_input, results_dataset)

aiq/eval/rag_evaluator/register.py

@@ -1,138 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from pydantic import BaseModel
-from pydantic import Field
-from pydantic import model_validator
-
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.builder.framework_enum import LLMFrameworkEnum
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-
-logger = logging.getLogger(__name__)
-
-
-class RagasMetricConfig(BaseModel):
-    ''' RAGAS metrics configuration
-    skip: Allows the metric config to be present but not used
-    kwargs: Additional arguments to pass to the metric's callable
-    '''
-    skip: bool = False
-    # kwargs specific to the metric's callable
-    kwargs: dict | None = None
-
-
-class RagasEvaluatorConfig(EvaluatorBaseConfig, name="ragas"):
-    """Evaluation using RAGAS metrics."""
-
-    llm_name: str = Field(description="LLM as a judge.")
-    # Ragas metric
-    metric: str | dict[str, RagasMetricConfig] = Field(default="AnswerAccuracy",
-                                                       description="RAGAS metric callable with optional 'kwargs:'")
-
-    @model_validator(mode="before")
-    @classmethod
-    def validate_metric(cls, values):
-        """Ensures metric is either a string or a single-item dictionary."""
-        metric = values.get("metric")
-
-        if isinstance(metric, dict):
-            if len(metric) != 1:
-                raise ValueError("Only one metric is allowed in the configuration.")
-            _, value = next(iter(metric.items()))
-            if not isinstance(value, dict):
-                raise ValueError("Metric value must be a RagasMetricConfig object.")
-        elif not isinstance(metric, str):
-            raise ValueError("Metric must be either a string or a single-item dictionary.")
-
-        return values
-
-    @property
-    def metric_name(self) -> str:
-        """Returns the single metric name."""
-        if isinstance(self.metric, str):
-            return self.metric
-        if isinstance(self.metric, dict) and self.metric:
-            return next(iter(self.metric.keys()))  # pylint: disable=no-member
-        return ""
-
-    @property
-    def metric_config(self) -> RagasMetricConfig:
-        """Returns the metric configuration (or a default if only a string is provided)."""
-        if isinstance(self.metric, str):
-            return RagasMetricConfig()  # Default config when only a metric name is given
-        if isinstance(self.metric, dict) and self.metric:
-            return next(iter(self.metric.values()))  # pylint: disable=no-member
-        return RagasMetricConfig()  # Default config when an invalid type is provided
-
-
-@register_evaluator(config_type=RagasEvaluatorConfig)
-async def register_ragas_evaluator(config: RagasEvaluatorConfig, builder: EvalBuilder):
-    from ragas.metrics import Metric
-
-    def get_ragas_metric(metric_name: str) -> Metric | None:
-        """
-        Fetch callable for RAGAS metrics
-        """
-        try:
-            import ragas.metrics as ragas_metrics
-
-            return getattr(ragas_metrics, metric_name)
-        except ImportError as e:
-            message = f"Ragas metrics not found {e}."
-            logger.error(message)
-            raise ValueError(message) from e
-        except AttributeError as e:
-            message = f"Ragas metric {metric_name} not found {e}."
-            logger.error(message)
-            return None
-
-    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
-        '''Run the RAGAS evaluation and return the average scores and evaluation results dataframe'''
-        if not _evaluator:
-            logger.warning("No evaluator found for RAGAS metrics.")
-            # return empty results if no evaluator is found
-            return EvalOutput(average_score=0.0, eval_output_items=[])
-
-        return await _evaluator.evaluate(eval_input)
-
-    from .evaluate import RAGEvaluator
-
-    # Get LLM
-    llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
-
-    # Get RAGAS metric callable from the metric config and create a list of metric-callables
-    metrics = []
-    # currently only one metric is supported
-    metric_name = config.metric_name  # Extracts the metric name
-    metric_config = config.metric_config  # Extracts the config (handles str/dict cases)
-
-    # Skip if `skip` is True
-    if not metric_config.skip:
-        metric_callable = get_ragas_metric(metric_name)
-        if metric_callable:
-            kwargs = metric_config.kwargs or {}
-            metrics.append(metric_callable(**kwargs))
-
-    # Create the RAG evaluator
-    _evaluator = RAGEvaluator(evaluator_llm=llm, metrics=metrics) if metrics else None
-
-    yield EvaluatorInfo(config=config, evaluate_fn=evaluate_fn, description="Evaluator for RAGAS metrics")

aiq/eval/register.py

@@ -1,23 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# flake8: noqa
-# pylint: disable=unused-import
-
-# Import evaluators which need to be automatically registered here
-from .rag_evaluator.register import register_ragas_evaluator
-from .swe_bench_evaluator.register import register_swe_bench_evaluator
-from .trajectory_evaluator.register import register_trajectory_evaluator
-from .tunable_rag_evaluator.register import register_tunable_rag_evaluator

aiq/eval/remote_workflow.py

@@ -1,128 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import json
-import logging
-
-import aiohttp
-from pydantic import ValidationError
-from tqdm import tqdm
-
-from aiq.data_models.api_server import AIQResponseIntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStepPayload
-from aiq.eval.config import EvaluationRunConfig
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-
-logger = logging.getLogger(__name__)
-
-# Constants for streaming response prefixes
-DATA_PREFIX = "data: "
-INTERMEDIATE_DATA_PREFIX = "intermediate_data: "
-
-
-class EvaluationRemoteWorkflowHandler:
-
-    def __init__(self, config: EvaluationRunConfig, max_concurrency: int):
-        self.config = config
-        # Run metadata
-        self.semaphore = asyncio.Semaphore(max_concurrency)
-
-    async def run_workflow_remote_single(self, session: aiohttp.ClientSession, item: EvalInputItem):
-        """
-        Sends a single input to the endpoint hosting the workflow and retrieves the response.
-        """
-        question = item.input_obj
-        # generate request format
-        payload = {"input_message": question}
-
-        try:
-            # Use the streaming endpoint
-            endpoint = f"{self.config.endpoint}/generate/full"
-            async with session.post(endpoint, json=payload) as response:
-                response.raise_for_status()  # Raise an exception for HTTP errors
-
-                # Initialize variables to store the response
-                final_response = None
-                intermediate_steps = []
-
-                # Process the streaming response
-                async for line in response.content:
-                    line = line.decode('utf-8').strip()
-                    if not line:
-                        continue
-
-                    if line.startswith(DATA_PREFIX):
-                        # This is a generate response chunk
-                        try:
-                            chunk_data = json.loads(line[len(DATA_PREFIX):])
-                            if chunk_data.get("value"):
-                                final_response = chunk_data.get("value")
-                        except json.JSONDecodeError as e:
-                            logger.error("Failed to parse generate response chunk: %s", e)
-                            continue
-                    elif line.startswith(INTERMEDIATE_DATA_PREFIX):
-                        # This is an intermediate step
-                        try:
-                            step_data = json.loads(line[len(INTERMEDIATE_DATA_PREFIX):])
-                            response_intermediate = AIQResponseIntermediateStep.model_validate(step_data)
-                            # The payload is expected to be IntermediateStepPayload
-                            intermediate_step = IntermediateStep(
-                                payload=IntermediateStepPayload.model_validate_json(response_intermediate.payload))
-                            intermediate_steps.append(intermediate_step)
-                        except (json.JSONDecodeError, ValidationError) as e:
-                            logger.error("Failed to parse intermediate step: %s", e)
-                            continue
-
-        except aiohttp.ClientError as e:
-            # Handle connection or HTTP-related errors
-            logger.error("Request failed for question %s: %s", question, e)
-            item.output_obj = None
-            item.trajectory = []
-            return
-
-        # Extract and fill the item with the response and intermediate steps
-        item.output_obj = final_response
-        item.trajectory = intermediate_steps
-        return
-
-    async def run_workflow_remote_with_limits(self, session: aiohttp.ClientSession, item: EvalInputItem, pbar: tqdm):
-        """
-        Sends limited number of concurrent requests to a remote workflow and retrieves responses.
-        """
-        async with self.semaphore:
-            await self.run_workflow_remote_single(session=session, item=item)
-            pbar.update(1)
-
-    async def run_workflow_remote(self, eval_input: EvalInput) -> EvalInput:
-        """
-        Sends inputs to a workflow hosted on a remote endpoint.
-        """
-        timeout = aiohttp.ClientTimeout(total=self.config.endpoint_timeout)
-        try:
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Running workflow", unit="item")
-            async with aiohttp.ClientSession(timeout=timeout) as session:
-                # get the questions from the eval_input
-                tasks = [
-                    self.run_workflow_remote_with_limits(session, item, pbar) for item in eval_input.eval_input_items
-                ]
-                await asyncio.gather(*tasks)
-
-        finally:
-            pbar.close()
-
-        return eval_input

aiq/eval/runtime_event_subscriber.py

@@ -1,52 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import logging
-
-from aiq.builder.context import AIQContext
-from aiq.data_models.intermediate_step import IntermediateStep
-
-logger = logging.getLogger(__name__)
-
-
-def pull_intermediate() -> asyncio.Future[list[dict]]:
-    """
-    Subscribes to the runner's event stream using callbacks.
-    Intermediate steps are collected and, when complete, the future is set
-    with the list of dumped intermediate steps.
-    """
-    future = asyncio.Future()
-    intermediate_steps = []  # We'll store the dumped steps here.
-    context = AIQContext.get()
-
-    def on_next_cb(item: IntermediateStep):
-        # Append each new intermediate step (dumped to dict) to the list.
-        intermediate_steps.append(item.model_dump())
-
-    def on_error_cb(exc: Exception):
-        logger.error("Hit on_error: %s", exc)
-        if not future.done():
-            future.set_exception(exc)
-
-    def on_complete_cb():
-        logger.debug("Completed reading intermediate steps")
-        if not future.done():
-            future.set_result(intermediate_steps)
-
-    # Subscribe with our callbacks.
-    context.intermediate_step_manager.subscribe(on_next=on_next_cb, on_error=on_error_cb, on_complete=on_complete_cb)
-
-    return future