aiqtoolkit 1.2.0.dev0__py3-none-any.whl → 1.2.0rc2__py3-none-any.whl

aiq/eval/remote_workflow.py

@@ -24,6 +24,7 @@ from tqdm import tqdm
 from aiq.data_models.api_server import AIQResponseIntermediateStep
 from aiq.data_models.intermediate_step import IntermediateStep
 from aiq.data_models.intermediate_step import IntermediateStepPayload
+from aiq.data_models.invocation_node import InvocationNode
 from aiq.eval.config import EvaluationRunConfig
 from aiq.eval.evaluator.evaluator_model import EvalInput
 from aiq.eval.evaluator.evaluator_model import EvalInputItem
@@ -81,8 +82,12 @@ class EvaluationRemoteWorkflowHandler:
                             step_data = json.loads(line[len(INTERMEDIATE_DATA_PREFIX):])
                             response_intermediate = AIQResponseIntermediateStep.model_validate(step_data)
                             # The payload is expected to be IntermediateStepPayload
-                            intermediate_step = IntermediateStep(
-                                payload=IntermediateStepPayload.model_validate_json(response_intermediate.payload))
+                            payload = IntermediateStepPayload.model_validate_json(response_intermediate.payload)
+                            intermediate_step = IntermediateStep(parent_id="remote",
+                                                                 function_ancestry=InvocationNode(
+                                                                     function_name=payload.name or "remote_function",
+                                                                     function_id=payload.UUID or "remote_function_id"),
+                                                                 payload=payload)
                             intermediate_steps.append(intermediate_step)
                         except (json.JSONDecodeError, ValidationError) as e:
                             logger.error("Failed to parse intermediate step: %s", e)

aiq/eval/runners/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

aiq/eval/runners/config.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import BaseModel
+
+from aiq.eval.config import EvaluationRunConfig
+from aiq.eval.config import EvaluationRunOutput
+
+
+class MultiEvaluationRunConfig(BaseModel):
+    """
+    Parameters used for a multi-evaluation run.
+    This includes a dict of configs. The key is an id of any type.
+    Each pass loads the config, applies the overrides and runs to completion
+    before the next pass starts.
+    """
+    configs: dict[typing.Any, EvaluationRunConfig]
+
+
+class MultiEvaluationRunOutput(BaseModel):
+    """
+    Output of a multi-evaluation run.
+    The results per-pass are accumulated in the evaluation_run_outputs dict.
+    """
+    evaluation_run_outputs: dict[typing.Any, EvaluationRunOutput]

aiq/eval/runners/multi_eval_runner.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import typing
+
+from aiq.eval.config import EvaluationRunConfig
+from aiq.eval.config import EvaluationRunOutput
+from aiq.eval.evaluate import EvaluationRun
+from aiq.eval.runners.config import MultiEvaluationRunConfig
+
+
+class MultiEvaluationRunner:
+    """
+    Run a multi-evaluation run.
+    """
+
+    def __init__(self, config: MultiEvaluationRunConfig):
+        """
+        Initialize a multi-evaluation run.
+        """
+        self.config = config
+        self.evaluation_run_outputs: dict[typing.Any, EvaluationRunOutput] = {}
+
+    async def run_all(self):
+        """
+        Run all evaluations defined by the overrides.
+        """
+        for id, config in self.config.configs.items():
+            output = await self.run_single_evaluation(id, config)
+            self.evaluation_run_outputs[id] = output
+
+        return self.evaluation_run_outputs
+
+    async def run_single_evaluation(self, id: typing.Any, config: EvaluationRunConfig) -> EvaluationRunOutput:
+        """
+        Run a single evaluation and return the output.
+        """
+        # copy the config in case the caller is using the same config for multiple evaluations
+        config_copy = copy.deepcopy(config)
+        evaluation_run = EvaluationRun(config_copy)
+        return await evaluation_run.run_and_evaluate()

aiq/eval/trajectory_evaluator/evaluate.py

@@ -13,24 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import asyncio
 import logging
 
 from langchain.evaluation import TrajectoryEvalChain
 from langchain_core.language_models import BaseChatModel
 from langchain_core.tools import BaseTool
-from tqdm import tqdm
 
-from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.base_evaluator import BaseEvaluator
 from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
 from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 logger = logging.getLogger(__name__)
 
 
-class TrajectoryEvaluator:
+class TrajectoryEvaluator(BaseEvaluator):
 
     def __init__(
         self,
@@ -38,11 +34,9 @@ class TrajectoryEvaluator:
         tools: list[BaseTool] | None = None,
         max_concurrency: int = 8,
     ):
-
+        super().__init__(max_concurrency=max_concurrency, tqdm_desc="Evaluating Trajectory")
         self.llm = llm
         self.tools = tools
-        self.max_concurrency = max_concurrency
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
         # Initialize trajectory evaluation chain
         self.traj_eval_chain = TrajectoryEvalChain.from_llm(llm=self.llm,
                                                             tools=self.tools,
@@ -50,69 +44,32 @@ class TrajectoryEvaluator:
                                                             requires_reference=True)
         logger.debug("Trajectory evaluation chain initialized.")
 
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+    async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
         """
-        Evaluates the agent trajectories using trajectory evaluation chain.
+        Evaluate a single EvalInputItem and return an EvalOutputItem.
         """
-
-        num_records = len(eval_input.eval_input_items)
-        logger.info("Running trajectory evaluation with %d records", num_records)
         from aiq.data_models.intermediate_step import IntermediateStepType
         from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         intermediate_step_adapter = IntermediateStepAdapter()
         event_filter = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]
 
-        async def process_item(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Evaluate a single EvalInputItem asynchronously and return a tuple of-
-            1. score
-            2. reasoning for the score
-            """
-            question = item.input_obj
-            generated_answer = item.output_obj
-            agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
-            try:
-                eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
-                    input=question,
-                    agent_trajectory=agent_trajectory,
-                    prediction=generated_answer,
-                )
-            except Exception as e:
-                logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
-                return 0.0, f"Error evaluating trajectory: {e}"
-
-            reasoning = {
-                "reasoning": eval_result["reasoning"],
-                "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
-            }
-            return eval_result["score"], reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            async with self.semaphore:
-                result = await process_item(item)
-                pbar.update(1)
-                return result
+        question = item.input_obj
+        generated_answer = item.output_obj
+        agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
 
-        # Execute all evaluations asynchronously
         try:
-            tqdm_position = TqdmPositionRegistry.claim()
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Trajectory", position=tqdm_position)
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
-
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
+            eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
+                input=question,
+                agent_trajectory=agent_trajectory,
+                prediction=generated_answer,
+            )
+        except Exception as e:
+            logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
+            return EvalOutputItem(id=item.id, score=0.0, reasoning=f"Error evaluating trajectory: {e}")
+
+        reasoning = {
+            "reasoning": eval_result["reasoning"],
+            "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
+        }
+        return EvalOutputItem(id=item.id, score=eval_result["score"], reasoning=reasoning)

aiq/eval/tunable_rag_evaluator/evaluate.py

@@ -15,19 +15,19 @@
 
 import asyncio
 import logging
+from typing import Callable
 
 from langchain.output_parsers import ResponseSchema
 from langchain.output_parsers import StructuredOutputParser
 from langchain.schema import HumanMessage
 from langchain.schema import SystemMessage
 from langchain_core.language_models import BaseChatModel
+from langchain_core.runnables import RunnableLambda
 from tqdm import tqdm
 
-from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.base_evaluator import BaseEvaluator
 from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
 from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 logger = logging.getLogger(__name__)
 
@@ -69,195 +69,177 @@ def evaluation_prompt(judge_llm_prompt: str,
     return EVAL_PROMPT if not default_scoring else DEFAULT_EVAL_PROMPT
 
 
-class TunableRagEvaluator:
+def runnable_with_retries(original_fn: Callable, llm_retry_control_params: dict | None = None):
+    runnable = RunnableLambda(original_fn)
+
+    if llm_retry_control_params is None:
+        llm_retry_control_params = {
+            "stop_after_attempt": 3, "initial_backoff_delay_seconds": 1, "has_exponential_jitter": True
+        }
+
+    if llm_retry_control_params["has_exponential_jitter"] is None:
+        llm_retry_control_params["has_exponential_jitter"] = True
+    if llm_retry_control_params["stop_after_attempt"] is None:
+        llm_retry_control_params["stop_after_attempt"] = 3
+    if llm_retry_control_params["initial_backoff_delay_seconds"] is None:
+        llm_retry_control_params["initial_backoff_delay_seconds"] = 1
+
+    # Add retry logic with exponential backoff and jitter
+    return runnable.with_retry(
+        retry_if_exception_type=(Exception, ),  # Retry on any error
+        wait_exponential_jitter=llm_retry_control_params["has_exponential_jitter"],  # Add jitter to exponential backoff
+        stop_after_attempt=llm_retry_control_params["stop_after_attempt"],
+        exponential_jitter_params={"initial": llm_retry_control_params["initial_backoff_delay_seconds"]
+                                   }  # Optional: set initial backoff (seconds)
+    )
+
+
+class TunableRagEvaluator(BaseEvaluator):
     '''Tunable RAG evaluator class with customizable LLM prompt for scoring.'''
 
     def __init__(self,
                  llm: BaseChatModel,
                  judge_llm_prompt: str,
+                 llm_retry_control_params: dict | None,
                  max_concurrency: int,
                  default_scoring: bool,
                  default_score_weights: dict):
+        super().__init__(max_concurrency=max_concurrency, tqdm_desc="Evaluating RAG")
         self.llm = llm
-        self.max_concurrency = max_concurrency
         self.judge_llm_prompt = judge_llm_prompt
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
+        self.llm_retry_control_params = llm_retry_control_params
         self.default_scoring = default_scoring
         # Use user-provided weights if available; otherwise, set equal weights for each score
         self.default_score_weights = default_score_weights if default_score_weights else {
             "coverage": 1 / 3, "correctness": 1 / 3, "relevance": 1 / 3
         }
 
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        '''Evaluate function'''
-
-        async def process_item(item):
-            """Compute RAG evaluation for an individual item"""
-            question = item.input_obj
-            answer_description = item.expected_output_obj
-            generated_answer = item.output_obj
-
-            # Call judge LLM to generate score
-            score = 0.0
-
-            default_evaluation_schema = [
-                ResponseSchema(
-                    name="coverage_score",
-                    description=
-                    "Score for the coverage of all critical aspects mentioned in the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(
-                    name="correctness_score",
-                    description=
-                    "Score for the accuracy of the generated answer compared to the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(name="relevance_score",
-                               description="Score for the relevance of the generated answer to the question. Ex. 0.5",
-                               type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 summarized sentences of reasoning for the scores. Ex. 'The generated answer covers all critical aspects mentioned in the expected answer, is correct, and is relevant to the question.'",
-                    type="string"),
-            ]
-
-            custom_evaluation_schema = [
-                ResponseSchema(name="score", description="Score for the generated answer. Ex. 0.5", type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 sentence reasoning for the score. Ex. 'The generated answer is exactly the same as the description of the expected answer.'",
-                    type="string"),
-            ]
+    async def evaluate_item(self, item: EvalInputItem) -> EvalOutputItem:
+        """Compute RAG evaluation for an individual item and return EvalOutputItem"""
+        question = item.input_obj
+        answer_description = item.expected_output_obj
+        generated_answer = item.output_obj
+
+        # Call judge LLM to generate score
+        score = 0.0
+
+        default_evaluation_schema = [
+            ResponseSchema(
+                name="coverage_score",
+                description="Score for the coverage of all critical aspects mentioned in the expected answer. Ex. 0.5",
+                type="float"),
+            ResponseSchema(
+                name="correctness_score",
+                description="Score for the accuracy of the generated answer compared to the expected answer. Ex. 0.5",
+                type="float"),
+            ResponseSchema(name="relevance_score",
+                           description="Score for the relevance of the generated answer to the question. Ex. 0.5",
+                           type="float"),
+            ResponseSchema(
+                name="reasoning",
+                description=
+                "1-2 summarized sentences of reasoning for the scores. Ex. 'The generated answer covers all critical aspects mentioned in the expected answer, is correct, and is relevant to the question.'",
+                type="string"),
+        ]
 
-            if self.default_scoring:
-                evaluation_schema = default_evaluation_schema
-            else:
-                evaluation_schema = custom_evaluation_schema
+        custom_evaluation_schema = [
+            ResponseSchema(name="score", description="Score for the generated answer. Ex. 0.5", type="float"),
+            ResponseSchema(
+                name="reasoning",
+                description=
+                "1-2 sentence reasoning for the score. Ex. 'The generated answer is exactly the same as the description of the expected answer.'",
+                type="string"),
+        ]
 
-            llm_input_response_parser = StructuredOutputParser.from_response_schemas(evaluation_schema)
-            format_instructions = llm_input_response_parser.get_format_instructions()
+        if self.default_scoring:
+            evaluation_schema = default_evaluation_schema
+        else:
+            evaluation_schema = custom_evaluation_schema
 
-            eval_prompt = evaluation_prompt(judge_llm_prompt=self.judge_llm_prompt,
-                                            question=question,
-                                            answer_description=answer_description,
-                                            generated_answer=generated_answer,
-                                            format_instructions=format_instructions,
-                                            default_scoring=self.default_scoring)
+        llm_input_response_parser = StructuredOutputParser.from_response_schemas(evaluation_schema)
+        format_instructions = llm_input_response_parser.get_format_instructions()
 
-            messages = [
-                SystemMessage(content="You must respond only in JSON format."), HumanMessage(content=eval_prompt)
-            ]
+        eval_prompt = evaluation_prompt(judge_llm_prompt=self.judge_llm_prompt,
+                                        question=question,
+                                        answer_description=answer_description,
+                                        generated_answer=generated_answer,
+                                        format_instructions=format_instructions,
+                                        default_scoring=self.default_scoring)
 
-            response = await self.llm.ainvoke(messages)
+        messages = [SystemMessage(content="You must respond only in JSON format."), HumanMessage(content=eval_prompt)]
 
-            # Initialize default values to handle service errors
-            coverage_score = 0.0
-            correctness_score = 0.0
-            relevance_score = 0.0
-            reasoning = "Error in evaluator from parsing judge LLM response."
+        response = await runnable_with_retries(self.llm.ainvoke, self.llm_retry_control_params).ainvoke(messages)
 
-            try:
-                parsed_response = llm_input_response_parser.parse(response.content)
-                if self.default_scoring:
-                    try:
-                        coverage_score = parsed_response["coverage_score"]
-                        correctness_score = parsed_response["correctness_score"]
-                        relevance_score = parsed_response["relevance_score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in default scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-
-                    coverage_weight = self.default_score_weights.get("coverage", 1 / 3)
-                    correctness_weight = self.default_score_weights.get("correctness", 1 / 3)
-                    relevance_weight = self.default_score_weights.get("relevance", 1 / 3)
-
-                    # Calculate score
-                    total_weight = coverage_weight + correctness_weight + relevance_weight
-                    coverage_weight = coverage_weight / total_weight
-                    correctness_weight = correctness_weight / total_weight
-                    relevance_weight = relevance_weight / total_weight
-
-                    if round(coverage_weight + correctness_weight + relevance_weight, 2) != 1:
-                        logger.warning("The sum of the default score weights is not 1. The weights will be normalized.")
-                        coverage_weight = coverage_weight / (coverage_weight + correctness_weight + relevance_weight)
-                        correctness_weight = correctness_weight / (coverage_weight + correctness_weight +
-                                                                   relevance_weight)
-                        relevance_weight = relevance_weight / (coverage_weight + correctness_weight + relevance_weight)
-
-                    score = (coverage_weight * coverage_score + correctness_weight * correctness_score +
-                             relevance_weight * relevance_score)
-
-                else:
-                    try:
-                        score = parsed_response["score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in custom scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-                        raise
-            except (KeyError, ValueError) as e:
-                logger.error("Error parsing judge LLM response: %s", e)
-                score = 0.0
-                reasoning = "Error in evaluator from parsing judge LLM response."
+        # Initialize default values to handle service errors
+        coverage_score = 0.0
+        correctness_score = 0.0
+        relevance_score = 0.0
+        reasoning = "Error in evaluator from parsing judge LLM response."
 
+        try:
+            parsed_response = llm_input_response_parser.parse(response.content)
             if self.default_scoring:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "score_breakdown": {
-                        "coverage_score": coverage_score,
-                        "correctness_score": correctness_score,
-                        "relevance_score": relevance_score,
-                    },
-                    "reasoning": reasoning,
-                }
-            else:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "reasoning": reasoning
-                }
-
-            return score, reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Process an item asynchronously and update the progress bar.
-            Use the semaphore to limit the number of concurrent items.
-            """
-            async with self.semaphore:
-                result = await process_item(item)
-                # Update the progress bar
-                pbar.update(1)
-                return result
+                try:
+                    coverage_score = parsed_response["coverage_score"]
+                    correctness_score = parsed_response["correctness_score"]
+                    relevance_score = parsed_response["relevance_score"]
+                    reasoning = parsed_response["reasoning"]
+                except KeyError as e:
+                    logger.error("Missing required keys in default scoring response: %s",
+                                 ", ".join(str(arg) for arg in e.args))
+                    reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
+
+                coverage_weight = self.default_score_weights.get("coverage", 1 / 3)
+                correctness_weight = self.default_score_weights.get("correctness", 1 / 3)
+                relevance_weight = self.default_score_weights.get("relevance", 1 / 3)
+
+                # Calculate score
+                total_weight = coverage_weight + correctness_weight + relevance_weight
+                coverage_weight = coverage_weight / total_weight
+                correctness_weight = correctness_weight / total_weight
+                relevance_weight = relevance_weight / total_weight
+
+                if round(coverage_weight + correctness_weight + relevance_weight, 2) != 1:
+                    logger.warning("The sum of the default score weights is not 1. The weights will be normalized.")
+                    coverage_weight = coverage_weight / (coverage_weight + correctness_weight + relevance_weight)
+                    correctness_weight = correctness_weight / (coverage_weight + correctness_weight + relevance_weight)
+                    relevance_weight = relevance_weight / (coverage_weight + correctness_weight + relevance_weight)
+
+                score = (coverage_weight * coverage_score + correctness_weight * correctness_score +
+                         relevance_weight * relevance_score)
 
-        try:
-            # Claim a tqdm position to display the progress bar
-            tqdm_position = TqdmPositionRegistry.claim()
-            # Create a progress bar
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating RAG", position=tqdm_position)
-            # Process items concurrently with a limit on concurrency
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
+            else:
+                try:
+                    score = parsed_response["score"]
+                    reasoning = parsed_response["reasoning"]
+                except KeyError as e:
+                    logger.error("Missing required keys in custom scoring response: %s",
+                                 ", ".join(str(arg) for arg in e.args))
+                    reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
+                    raise
+        except (KeyError, ValueError) as e:
+            logger.error("Error parsing judge LLM response: %s", e)
+            score = 0.0
+            reasoning = "Error in evaluator from parsing judge LLM response."
 
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)
+        if self.default_scoring:
+            reasoning = {
+                "question": question,
+                "answer_description": answer_description,
+                "generated_answer": generated_answer,
+                "score_breakdown": {
+                    "coverage_score": coverage_score,
+                    "correctness_score": correctness_score,
+                    "relevance_score": relevance_score,
+                },
+                "reasoning": reasoning,
+            }
+        else:
+            reasoning = {
+                "question": question,
+                "answer_description": answer_description,
+                "generated_answer": generated_answer,
+                "reasoning": reasoning
+            }
+
+        return EvalOutputItem(id=item.id, score=score, reasoning=reasoning)

aiq/eval/tunable_rag_evaluator/register.py

@@ -26,6 +26,7 @@ from aiq.data_models.evaluator import EvaluatorBaseConfig
 class TunableRagEvaluatorConfig(EvaluatorBaseConfig, name="tunable_rag_evaluator"):
     '''Configuration for tunable RAG evaluator'''
     llm_name: LLMRef = Field(description="Name of the judge LLM")
+    llm_retry_control_params: dict | None = Field(description="Parameters to control LLM retry behavior", default=None)
     judge_llm_prompt: str = Field(description="LLM prompt for the judge LLM")
     default_scoring: bool = Field(description="Whether to use default scoring", default=False)
     default_score_weights: dict = Field(
@@ -43,6 +44,7 @@ async def register_tunable_rag_evaluator(config: TunableRagEvaluatorConfig, buil
     llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
     evaluator = TunableRagEvaluator(llm,
                                     config.judge_llm_prompt,
+                                    config.llm_retry_control_params,
                                     builder.get_max_concurrency(),
                                     config.default_scoring,
                                     config.default_score_weights)