aiqtoolkit 1.1.0a20250515__py3-none-any.whl → 1.1.0a20251020__py3-none-any.whl

aiq/eval/swe_bench_evaluator/evaluate.py

@@ -1,215 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import logging
-import os
-import shutil
-from pathlib import Path
-
-from aiq.data_models.swe_bench_model import SWEBenchInput
-from aiq.data_models.swe_bench_model import SWEBenchOutput
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-
-try:
-    import swebench.harness.run_evaluation as swebench_eval
-    from swebench.harness.constants import MAP_REPO_VERSION_TO_SPECS
-except ImportError as exc:
-    raise ImportError("Please install swebench to use this evaluator") from exc
-
-logger = logging.getLogger(__name__)
-
-
-class SweBenchEvaluator:
-
-    def __init__(self, run_id: str, max_workers: int, output_dir: Path):
-
-        self.run_id = run_id
-        self.max_workers = max_workers
-        self.output_dir = output_dir
-
-        # metadata
-        self._unsupported_repos = []
-        self._swe_bench_inputs = []
-        self._swe_bench_outputs = []
-        self._model_name_or_path = "no_llm"
-
-    def get_model_name_from_output(self, workflow_output: list[dict]) -> str | None:
-        """Fetch the `model_name_or_path` from the first entry in the list."""
-        return workflow_output[0].get("model_name_or_path") if workflow_output else None
-
-    @staticmethod
-    def empty_report_dir(report_dir: Path):
-        """Remove the current contents of the report directory."""
-        os.makedirs(report_dir, exist_ok=True)
-
-        # Iterate through all files in the directory and remove them
-        for item in report_dir.iterdir():
-            if item.is_file():  # Remove files only
-                item.unlink()
-            elif item.is_dir():  # Remove subdirectories and their contents
-                shutil.rmtree(item)
-
-    @staticmethod
-    def move_report_and_logs(swe_bench_report_file: str, logs_dir: str, report_dir: Path):
-        """ Temorary function to move the report and logs to the output directory"""
-        try:
-            shutil.move(swe_bench_report_file, report_dir)
-        except Exception as e:
-            logger.exception("Error moving report file: %s", e, exc_info=True)
-
-        try:
-            dest_logs_dir = os.path.join(report_dir, 'logs')
-            shutil.move(logs_dir, dest_logs_dir)
-        except Exception as e:
-            logger.exception("Error moving logs directory: %s", e, exc_info=True)
-
-    def is_repo_supported(self, repo: str, version: str) -> bool:
-        """Check if the repo is supported by swebench"""
-
-        try:
-            _ = MAP_REPO_VERSION_TO_SPECS[repo][str(version)]
-        except KeyError:
-            self._unsupported_repos.append({repo, version})
-            return False
-        return True
-
-    def process_eval_input(self, eval_input: EvalInput) -> tuple[Path, Path]:
-        """Converts EvalInput into lists of SWEBenchInput and SWEBenchOutput models and applies filtering."""
-        # Convert input_obj and output_obj JSON strings to SWEBenchInput and SWEBenchOutput models
-        swebench_inputs = []
-        swebench_outputs = []
-
-        for item in eval_input.eval_input_items:
-            try:
-                swebench_input = SWEBenchInput.model_validate_json(item.input_obj)  # Convert input JSON to model
-                swebench_input.version = str(swebench_input.version)  # Convert version to string
-                swebench_inputs.append(swebench_input)
-
-                if item.output_obj:  # Convert output JSON to model if available
-                    swebench_output = SWEBenchOutput.model_validate_json(item.output_obj)
-                    swebench_outputs.append(swebench_output)
-                    # this is bit of a hack to match the swe_bench harness
-                    self._model_name_or_path = swebench_output.model_name_or_path
-
-            except Exception as e:
-                logger.exception("Failed to parse EvalInputItem %s: %s", item.id, e, exc_info=True)
-
-        # Filter out repos/version not supported by SWEBench
-        supported_inputs = [
-            swebench for swebench in swebench_inputs if self.is_repo_supported(swebench.repo, swebench.version)
-        ]
-
-        if not supported_inputs:
-            logger.error("No supported instances; nothing to evaluate")
-            return None, None
-
-        if len(supported_inputs) < len(swebench_inputs):
-            logger.warning("The following repos are not supported by SWEBench and were skipped:\n %s",
-                           {s.repo
-                            for s in swebench_inputs if s not in supported_inputs})
-
-        # Write SWEBenchInput to file
-        workflow_input_file = self.output_dir / "aiq_workflow_input.json"
-        workflow_input_file.parent.mkdir(parents=True, exist_ok=True)
-        Path(workflow_input_file).write_text(json.dumps([swebench.model_dump() for swebench in supported_inputs],
-                                                        indent=2),
-                                             encoding="utf-8")
-        logger.info("Workflow input written to %s", workflow_input_file)
-
-        # Filter SWEBenchOutput to include only instance_ids present in SWEBenchInput
-        valid_instance_ids = {swebench.instance_id for swebench in supported_inputs}
-        filtered_outputs = [output for output in swebench_outputs if output.instance_id in valid_instance_ids]
-
-        if not filtered_outputs:
-            logger.error("No supported outputs; nothing to evaluate")
-            return None, None
-
-        # Write SWEBenchOutput to file
-        workflow_output_file = self.output_dir / "aiq_workflow_output.json"
-        Path(workflow_output_file).write_text(json.dumps([output.model_dump() for output in filtered_outputs],
-                                                         indent=2),
-                                              encoding="utf-8")
-        logger.info("Workflow output written to %s", workflow_output_file)
-
-        self._swe_bench_inputs = supported_inputs
-        self._swe_bench_outputs = filtered_outputs
-        return workflow_input_file, workflow_output_file
-
-    def build_eval_output(self):
-        """Builds the EvalOutput object from the SWEBenchOutput models and the average score."""
-        # WIP: Build a score based on eval run logs
-        for swebench_output in self._swe_bench_outputs:
-            yield {"id": swebench_output.instance_id, "score": "-", "reasoning": "-"}
-
-    @staticmethod
-    def compute_score(success_cnt: int, total_cnt: int) -> float:
-        if total_cnt == 0:
-            return 0.0
-        score = success_cnt / total_cnt
-        return min(max(score, 0.0), 1.0)
-
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        '''Run the swebench evaluation and store the report in the output directory'''
-
-        # Process the EvalInput
-        workflow_input_file, workflow_output_file = self.process_eval_input(eval_input)
-        if not workflow_input_file or not workflow_output_file:
-            # nothing to evaluate
-            return EvalOutput(average_score=0.0, eval_output_items=[])
-
-        report_dir = self.output_dir / "swe_bench_reports"
-        self.empty_report_dir(report_dir)
-
-        logger.info("Starting swe_bench run %s", self.run_id)
-        swebench_eval.main(dataset_name=str(workflow_input_file),
-                           split="dev",
-                           instance_ids=[],
-                           predictions_path=str(workflow_output_file),
-                           max_workers=self.max_workers,
-                           force_rebuild=False,
-                           cache_level="env",
-                           clean=False,
-                           open_file_limit=4096,
-                           run_id=self.run_id,
-                           timeout=1800,
-                           namespace=None,
-                           rewrite_reports=False,
-                           modal=False,
-                           instance_image_tag='latest',
-                           report_dir=str(report_dir))
-        logger.info("Completed swe_bench run %s", self.run_id)
-
-        swe_bench_report_file = f"{self._model_name_or_path}.{self.run_id}.json"
-
-        # There is a bug in swebench because of which report_dir is being ignored. Copy the report to the output dir
-        self.move_report_and_logs(swe_bench_report_file=swe_bench_report_file, logs_dir="logs", report_dir=report_dir)
-        logger.info("SWE_bench report and logs written to %s directory", report_dir)
-
-        # read the swe_bench report file
-        report_file = report_dir / swe_bench_report_file
-        # if report file is not present, return empty EvalOutput
-        avg_score = 0.0
-        if report_file.exists():
-            with open(report_file, "r", encoding="utf-8") as f:
-                report = json.load(f)
-                resolved_instances = report.get("resolved_instances", 0)
-                total_instances = report.get("total_instances", 0)
-                avg_score = self.compute_score(resolved_instances, total_instances)
-
-        # Build the EvalOutput from self._swe_bench_outputs and avg_score
-        eval_output_items = list(self.build_eval_output())
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)

aiq/eval/swe_bench_evaluator/register.py

@@ -1,36 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from pydantic import Field
-
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-
-
-class SweBenchEvaluatorConfig(EvaluatorBaseConfig, name="swe_bench"):
-    """Code patch evaluation for SWE Bench problems."""
-
-    run_id: str = Field(description="swe-bench test harness run identifier.")
-
-
-@register_evaluator(config_type=SweBenchEvaluatorConfig)
-async def register_swe_bench_evaluator(config: SweBenchEvaluatorConfig, builder: EvalBuilder):
-
-    from .evaluate import SweBenchEvaluator
-    _evaluator = SweBenchEvaluator(config.run_id, builder.get_max_concurrency(), builder.get_output_dir())
-
-    yield EvaluatorInfo(config=config, evaluate_fn=_evaluator.evaluate, description="SWE Bench Evaluator")

aiq/eval/trajectory_evaluator/evaluate.py

@@ -1,118 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import logging
-
-from langchain.evaluation import TrajectoryEvalChain
-from langchain_core.language_models import BaseChatModel
-from langchain_core.tools import BaseTool
-from tqdm import tqdm
-
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
-
-logger = logging.getLogger(__name__)
-
-
-class TrajectoryEvaluator:
-
-    def __init__(
-        self,
-        llm: BaseChatModel,
-        tools: list[BaseTool] | None = None,
-        max_concurrency: int = 8,
-    ):
-
-        self.llm = llm
-        self.tools = tools
-        self.max_concurrency = max_concurrency
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
-        # Initialize trajectory evaluation chain
-        self.traj_eval_chain = TrajectoryEvalChain.from_llm(llm=self.llm,
-                                                            tools=self.tools,
-                                                            return_reasoning=True,
-                                                            requires_reference=True)
-        logger.debug("Trajectory evaluation chain initialized.")
-
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        """
-        Evaluates the agent trajectories using trajectory evaluation chain.
-        """
-
-        num_records = len(eval_input.eval_input_items)
-        logger.info("Running trajectory evaluation with %d records", num_records)
-        from aiq.data_models.intermediate_step import IntermediateStepType
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
-
-        intermediate_step_adapter = IntermediateStepAdapter()
-        event_filter = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]
-
-        async def process_item(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Evaluate a single EvalInputItem asynchronously and return a tuple of-
-            1. score
-            2. reasoning for the score
-            """
-            question = item.input_obj
-            generated_answer = item.output_obj
-            agent_trajectory = intermediate_step_adapter.get_agent_actions(item.trajectory, event_filter)
-            try:
-                eval_result = await self.traj_eval_chain.aevaluate_agent_trajectory(
-                    input=question,
-                    agent_trajectory=agent_trajectory,
-                    prediction=generated_answer,
-                )
-            except Exception as e:
-                logger.exception("Error evaluating trajectory for question: %s, Error: %s", question, e, exc_info=True)
-                return 0.0, f"Error evaluating trajectory: {e}"
-
-            reasoning = {
-                "reasoning": eval_result["reasoning"],
-                "trajectory": [(action.model_dump(), output) for (action, output) in agent_trajectory]
-            }
-            return eval_result["score"], reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            async with self.semaphore:
-                result = await process_item(item)
-                pbar.update(1)
-                return result
-
-        # Execute all evaluations asynchronously
-        try:
-            tqdm_position = TqdmPositionRegistry.claim()
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating Trajectory", position=tqdm_position)
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
-
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)

aiq/eval/trajectory_evaluator/register.py

@@ -1,40 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from pydantic import Field
-
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-
-
-class TrajectoryEvaluatorConfig(EvaluatorBaseConfig, name="trajectory"):
-    """Agent Trajectory Evaluation."""
-
-    llm_name: str = Field(description="LLM as a judge.")
-
-
-@register_evaluator(config_type=TrajectoryEvaluatorConfig)
-async def register_trajectory_evaluator(config: TrajectoryEvaluatorConfig, builder: EvalBuilder):
-    from aiq.builder.framework_enum import LLMFrameworkEnum
-
-    from .evaluate import TrajectoryEvaluator
-    llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
-    tools = builder.get_all_tools(wrapper_type=LLMFrameworkEnum.LANGCHAIN)
-
-    _evaluator = TrajectoryEvaluator(llm, tools, builder.get_max_concurrency())
-
-    yield EvaluatorInfo(config=config, evaluate_fn=_evaluator.evaluate, description="Trajectory Evaluator")

aiq/eval/tunable_rag_evaluator/evaluate.py

@@ -1,263 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import asyncio
-import logging
-
-from langchain.output_parsers import ResponseSchema
-from langchain.output_parsers import StructuredOutputParser
-from langchain.schema import HumanMessage
-from langchain.schema import SystemMessage
-from langchain_core.language_models import BaseChatModel
-from tqdm import tqdm
-
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
-
-logger = logging.getLogger(__name__)
-
-# pylint: disable=line-too-long
-# flake8: noqa: E501
-
-
-def evaluation_prompt(judge_llm_prompt: str,
-                      question: str,
-                      answer_description: str,
-                      generated_answer: str,
-                      format_instructions: str,
-                      default_scoring: bool):
-    """
-    This function generates a prompt for the judge LLM to evaluate the generated answer.
-    """
-
-    DEFAULT_SCORING_INSTRUCTIONS = """
-    The coverage score is a measure of how well the generated answer covers the critical aspects mentioned in the expected answer. A low coverage score indicates that the generated answer misses critical aspects of the expected answer. A middle coverage score indicates that the generated answer covers some of the must-haves of the expected answer but lacks other details. A high coverage score indicates that all of the expected aspects are present in the generated answer.
-    The correctness score is a measure of how well the generated answer matches the expected answer. A low correctness score indicates that the generated answer is incorrect or does not match the expected answer. A middle correctness score indicates that the generated answer is correct but lacks some details. A high correctness score indicates that the generated answer is exactly the same as the expected answer.
-    The relevance score is a measure of how well the generated answer is relevant to the question. A low relevance score indicates that the generated answer is not relevant to the question. A middle relevance score indicates that the generated answer is somewhat relevant to the question. A high relevance score indicates that the generated answer is exactly relevant to the question.
-    The reasoning is a 1-2 sentence explanation for the scoring.
-    """
-
-    DEFAULT_EVAL_PROMPT = (f"You are an intelligent assistant that responds strictly in JSON format."
-                           f"Judge based on the following scoring rubric: {DEFAULT_SCORING_INSTRUCTIONS}"
-                           f"{judge_llm_prompt}\n"
-                           f"{format_instructions}\n"
-                           f"Here is the user's query: {question}"
-                           f"Here is the description of the expected answer: {answer_description}"
-                           f"Here is the generated answer: {generated_answer}")
-
-    EVAL_PROMPT = (f"You are an intelligent assistant that responds strictly in JSON format. {judge_llm_prompt}\n"
-                   f"{format_instructions}\n"
-                   f"Here is the user's query: {question}"
-                   f"Here is the description of the expected answer: {answer_description}"
-                   f"Here is the generated answer: {generated_answer}")
-
-    return EVAL_PROMPT if not default_scoring else DEFAULT_EVAL_PROMPT
-
-
-class TunableRagEvaluator:
-    '''Tunable RAG evaluator class with customizable LLM prompt for scoring.'''
-
-    def __init__(self,
-                 llm: BaseChatModel,
-                 judge_llm_prompt: str,
-                 max_concurrency: int,
-                 default_scoring: bool,
-                 default_score_weights: dict):
-        self.llm = llm
-        self.max_concurrency = max_concurrency
-        self.judge_llm_prompt = judge_llm_prompt
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
-        self.default_scoring = default_scoring
-        # Use user-provided weights if available; otherwise, set equal weights for each score
-        self.default_score_weights = default_score_weights if default_score_weights else {
-            "coverage": 1 / 3, "correctness": 1 / 3, "relevance": 1 / 3
-        }
-
-    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
-        '''Evaluate function'''
-
-        async def process_item(item):
-            """Compute RAG evaluation for an individual item"""
-            question = item.input_obj
-            answer_description = item.expected_output_obj
-            generated_answer = item.output_obj
-
-            # Call judge LLM to generate score
-            score = 0.0
-
-            default_evaluation_schema = [
-                ResponseSchema(
-                    name="coverage_score",
-                    description=
-                    "Score for the coverage of all critical aspects mentioned in the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(
-                    name="correctness_score",
-                    description=
-                    "Score for the accuracy of the generated answer compared to the expected answer. Ex. 0.5",
-                    type="float"),
-                ResponseSchema(name="relevance_score",
-                               description="Score for the relevance of the generated answer to the question. Ex. 0.5",
-                               type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 summarized sentences of reasoning for the scores. Ex. 'The generated answer covers all critical aspects mentioned in the expected answer, is correct, and is relevant to the question.'",
-                    type="string"),
-            ]
-
-            custom_evaluation_schema = [
-                ResponseSchema(name="score", description="Score for the generated answer. Ex. 0.5", type="float"),
-                ResponseSchema(
-                    name="reasoning",
-                    description=
-                    "1-2 sentence reasoning for the score. Ex. 'The generated answer is exactly the same as the description of the expected answer.'",
-                    type="string"),
-            ]
-
-            if self.default_scoring:
-                evaluation_schema = default_evaluation_schema
-            else:
-                evaluation_schema = custom_evaluation_schema
-
-            llm_input_response_parser = StructuredOutputParser.from_response_schemas(evaluation_schema)
-            format_instructions = llm_input_response_parser.get_format_instructions()
-
-            eval_prompt = evaluation_prompt(judge_llm_prompt=self.judge_llm_prompt,
-                                            question=question,
-                                            answer_description=answer_description,
-                                            generated_answer=generated_answer,
-                                            format_instructions=format_instructions,
-                                            default_scoring=self.default_scoring)
-
-            messages = [
-                SystemMessage(content="You must respond only in JSON format."), HumanMessage(content=eval_prompt)
-            ]
-
-            response = await self.llm.ainvoke(messages)
-
-            # Initialize default values to handle service errors
-            coverage_score = 0.0
-            correctness_score = 0.0
-            relevance_score = 0.0
-            reasoning = "Error in evaluator from parsing judge LLM response."
-
-            try:
-                parsed_response = llm_input_response_parser.parse(response.content)
-                if self.default_scoring:
-                    try:
-                        coverage_score = parsed_response["coverage_score"]
-                        correctness_score = parsed_response["correctness_score"]
-                        relevance_score = parsed_response["relevance_score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in default scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-
-                    coverage_weight = self.default_score_weights.get("coverage", 1 / 3)
-                    correctness_weight = self.default_score_weights.get("correctness", 1 / 3)
-                    relevance_weight = self.default_score_weights.get("relevance", 1 / 3)
-
-                    # Calculate score
-                    total_weight = coverage_weight + correctness_weight + relevance_weight
-                    coverage_weight = coverage_weight / total_weight
-                    correctness_weight = correctness_weight / total_weight
-                    relevance_weight = relevance_weight / total_weight
-
-                    if round(coverage_weight + correctness_weight + relevance_weight, 2) != 1:
-                        logger.warning("The sum of the default score weights is not 1. The weights will be normalized.")
-                        coverage_weight = coverage_weight / (coverage_weight + correctness_weight + relevance_weight)
-                        correctness_weight = correctness_weight / (coverage_weight + correctness_weight +
-                                                                   relevance_weight)
-                        relevance_weight = relevance_weight / (coverage_weight + correctness_weight + relevance_weight)
-
-                    score = (coverage_weight * coverage_score + correctness_weight * correctness_score +
-                             relevance_weight * relevance_score)
-
-                else:
-                    try:
-                        score = parsed_response["score"]
-                        reasoning = parsed_response["reasoning"]
-                    except KeyError as e:
-                        logger.error("Missing required keys in custom scoring response: %s",
-                                     ", ".join(str(arg) for arg in e.args))
-                        reasoning = f"Error in evaluator from parsing judge LLM response. Missing required key(s): {', '.join(str(arg) for arg in e.args)}"
-                        raise
-            except (KeyError, ValueError) as e:
-                logger.error("Error parsing judge LLM response: %s", e)
-                score = 0.0
-                reasoning = "Error in evaluator from parsing judge LLM response."
-
-            if self.default_scoring:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "score_breakdown": {
-                        "coverage_score": coverage_score,
-                        "correctness_score": correctness_score,
-                        "relevance_score": relevance_score,
-                    },
-                    "reasoning": reasoning,
-                }
-            else:
-                reasoning = {
-                    "question": question,
-                    "answer_description": answer_description,
-                    "generated_answer": generated_answer,
-                    "reasoning": reasoning
-                }
-
-            return score, reasoning
-
-        async def wrapped_process(item: EvalInputItem) -> tuple[float, dict]:
-            """
-            Process an item asynchronously and update the progress bar.
-            Use the semaphore to limit the number of concurrent items.
-            """
-            async with self.semaphore:
-                result = await process_item(item)
-                # Update the progress bar
-                pbar.update(1)
-                return result
-
-        try:
-            # Claim a tqdm position to display the progress bar
-            tqdm_position = TqdmPositionRegistry.claim()
-            # Create a progress bar
-            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Evaluating RAG", position=tqdm_position)
-            # Process items concurrently with a limit on concurrency
-            results = await asyncio.gather(*[wrapped_process(item) for item in eval_input.eval_input_items])
-        finally:
-            pbar.close()
-            TqdmPositionRegistry.release(tqdm_position)
-
-        # Extract scores and reasonings
-        sample_scores, sample_reasonings = zip(*results) if results else ([], [])
-
-        # Compute average score
-        avg_score = round(sum(sample_scores) / len(sample_scores), 2) if sample_scores else 0.0
-
-        # Construct EvalOutputItems
-        eval_output_items = [
-            EvalOutputItem(id=item.id, score=score, reasoning=reasoning)
-            for item, score, reasoning in zip(eval_input.eval_input_items, sample_scores, sample_reasonings)
-        ]
-
-        return EvalOutput(average_score=avg_score, eval_output_items=eval_output_items)