evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -1,6 +1,14 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import LLMJudge
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 TEMPLATE_0SHOT = """Please read the following text and answer the question below.
 
@@ -13,73 +21,123 @@ TEMPLATE_0SHOT = """Please read the following text and answer the question below
 Format your response as follows: "Therefore, the answer is (insert answer here)"."""
 
 
-@Benchmark.register(
-    name='docmath',
-    pretty_name='DocMath',
-    tags=['Reasoning', 'Mathematics', 'Long Context'],
-    description=
-    'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
-    dataset_id='yale-nlp/DocMath-Eval',
-    metric_list=['AverageAccuracy'],
-    subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=TEMPLATE_0SHOT,
+@register_benchmark(
+    BenchmarkMeta(
+        name='docmath',
+        pretty_name='DocMath',
+        tags=[Tags.REASONING, Tags.MATH, Tags.LONG_CONTEXT],
+        description=
+        'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
+        dataset_id='yale-nlp/DocMath-Eval',
+        metric_list=['acc'],
+        subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
 )
-class DocMathAdapter(DataAdapter):
+class DocMathAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for DocMath
+        self.split_as_subset = True  # Use split as subset for DocMath
 
-    def load(self, **kwargs):
-        # default load mini test
-        kwargs['split_as_subset'] = True
-        data_dict = super().load(**kwargs)
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        context = context = '\n'.join(input_d['paragraphs'])
-        question = input_d['question']
-        prompt = self.prompt_template.format(context=context, question=question)
-        return self.gen_prompt_data(prompt)
+        Convert a data record to a Sample object.
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d['ground_truth']
+        Args:
+            record (Dict[str, Any]): Input data record.
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        ground_truth = record['ground_truth']
+
+        return Sample(
+            input=record['question'],
+            target=str(ground_truth),
+            metadata={
+                'question_id': record.get('question_id', ''),
+                'paragraphs': record['paragraphs'],
+                'answer_type': type(ground_truth).__name__
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = '\n'.join(sample.metadata['paragraphs'])
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
         from .utils import extract_answer
 
-        extracted_answer = extract_answer(result)
+        extracted_answer = extract_answer(prediction)
         return extracted_answer
 
-    def match(self, gold: str, pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference.
         """
         from .utils import get_acc
 
-        return get_acc(prediction=pred, gt=gold)
-
-    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        answer_type = task_state.metadata.get('answer_type', 'unknown')
+        accuracy = get_acc(prediction=filtered_prediction, gt=reference, answer_type=answer_type)
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
         from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
 
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['question']
-        # get grading response
-        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
-        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
-        # parse grading response
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.metadata.get('question', '')
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
         if 'YES' in orm_response:
-            return 1.0
+            accuracy = 1.0
         else:
-            return 0.0
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/docmath/utils.py

@@ -193,23 +193,22 @@ def compare_two_numbers(p, gt):
     return within_eps(pred=p, gt=gt)
 
 
-def get_acc(prediction, gt, cot=True):
+def get_acc(prediction, gt, answer_type, cot=True):
     try:
         if cot:
             prediction = normalize(prediction)
         else:
             prediction = float(prediction)
 
-        answer_type = type(gt).__name__
         assert answer_type in ['int', 'float', 'float64', 'bool'], answer_type
         if isinstance(prediction, (str, int, float, bool)) or isinstance(prediction, list):
             # Comparing prediction against the reference
             if answer_type in ['bool']:
-                acc = int(prediction == gt)
+                acc = int(prediction == bool(gt))
             elif answer_type == 'int':
-                acc = int(compare_two_numbers(prediction, gt))
+                acc = int(compare_two_numbers(prediction, int(gt)))
             elif answer_type == 'float' or answer_type == 'float64':
-                acc = int(compare_two_numbers(prediction, gt))
+                acc = int(compare_two_numbers(prediction, float(gt)))
             else:
                 acc = 0
         else:

evalscope/benchmarks/drop/drop_adapter.py

@@ -1,8 +1,13 @@
+import ast
 import re
-from typing import List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -28,54 +33,82 @@ Answer:  43
 '''  # noqa: E501
 
 
-@Benchmark.register(
-    name='drop',
-    pretty_name='DROP',
-    tags=['Reasoning'],
-    description=
-    'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
-    dataset_id='AI-ModelScope/DROP',
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='validation',
-    prompt_template=
-    'You will be asked to read a passage and answer a question.{drop_examples}# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='drop',
+        pretty_name='DROP',
+        tags=[Tags.REASONING],
+        description=
+        'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
+        dataset_id='AI-ModelScope/DROP',
+        metric_list=['acc'],
+        few_shot_num=3,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=
+        'You will be asked to read a passage and answer a question. {drop_examples}\n# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+    )
 )
-class DROPAdapter(DataAdapter):
+class DROPAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        few_shot_num = kwargs.get('few_shot_num', 0)
-        if few_shot_num != 0:
+        if self.few_shot_num != 0:
             self.few_shot_num = 3
             logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
         else:
             self.few_shot_num = 0
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        Generate model prompt from input data.
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
         """
-        drop_examples = '' if self.few_shot_num == 0 else DROP_EXAMPLES
-        query = f"Passage: {input_d['passage']}\nQuestion: {input_d['question']}"
-        prompt = self.prompt_template.format(
+
+        # Parse gold answers
+        gold_answers = self._get_gold_answers(record)
+
+        return Sample(
+            input=record['question'],
+            target=str(gold_answers),
+            metadata={
+                'passage': record['passage'],
+                'answer': record['answer'],
+                'validated_answers': record['validated_answers']
+            }
+        )
+
+    def format_prompt_template(self, sample: Sample) -> str:
+        drop_examples = ''
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
             drop_examples=drop_examples,
             query=query,
         )
-        return self.gen_prompt_data(prompt)
 
-    def get_gold_answer(self, input_d: dict) -> List[str]:
+    def format_fewshot_template(self, fewshot, sample):
+        drop_examples = DROP_EXAMPLES
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+
+    def _get_gold_answers(self, input_d: dict) -> List[str]:
         """
         Parse the raw input labels (gold).
         """
 
         def _flatten_validated_answers(validated_answers):
-            """Flattens a dict of lists of validated answers.
-            {"number": ['1', '8'], ...}
-            -> [{"number": ['1'], ...}, {"number": ['8'], ...}]
-            """
+            """Flattens a dict of lists of validated answers."""
             valid_answers = []
             for i in range(len(validated_answers['number'])):
                 valid_answers.append({
@@ -96,24 +129,36 @@ class DROPAdapter(DataAdapter):
             answers.append(answer)
         return answers
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
-        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', result)
-        extracted_answer = match.group(1) if match else result
+        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', prediction)
+        extracted_answer = match.group(1) if match else prediction
         return extracted_answer
 
-    def match(self, gold: List[str], pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference answers.
         """
         from .utils import _answer_to_bags
 
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
         max_em = 0
-        for gold_answer in gold:
+        reference = ast.literal_eval(reference) if isinstance(reference, str) else reference
+        for gold_answer in reference:
             # Convert the answers to bags of answers
-            predicted_bags = _answer_to_bags(pred)
+            predicted_bags = _answer_to_bags(filtered_prediction)
             gold_bags = _answer_to_bags(gold_answer)
 
             if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
@@ -124,7 +169,10 @@ class DROPAdapter(DataAdapter):
             if gold_answer[0].strip():
                 max_em = max(max_em, exact_match)
 
-        return max_em
+        score.value = {'acc': max_em}
+        score.main_score_name = 'acc'
+
+        return score
 
     @staticmethod
     def parse_answer(answer):

evalscope/benchmarks/frames/frames_adapter.py

@@ -1,6 +1,15 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import LLMJudge, exact_match
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 TEMPLATE_0SHOT = """Please read the following text and answer the question below.
 
@@ -13,52 +22,82 @@ TEMPLATE_0SHOT = """Please read the following text and answer the question below
 Format your response as follows: "Therefore, the answer is (insert answer here)"."""
 
 
-@Benchmark.register(
-    name='frames',
-    pretty_name='FRAMES',
-    tags=['Reasoning', 'Long Context'],
-    description=
-    'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
-    dataset_id='iic/frames',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.GENERATION],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template=TEMPLATE_0SHOT,
+@register_benchmark(
+    BenchmarkMeta(
+        name='frames',
+        pretty_name='FRAMES',
+        tags=[Tags.REASONING, Tags.LONG_CONTEXT],
+        description=
+        'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
+        dataset_id='iic/frames',
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
 )
-class FramesAdapter(DataAdapter):
+class FramesAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for FRAMES
+
+    def load(self):
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
 
-    def load(self, **kwargs):
-        # default load with snapshot
-        kwargs['file_structure'] = {'default': ['test.jsonl']}
-        data_dict = super().load_with_snapshot(**kwargs)
-        return data_dict
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='test.jsonl')
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        """
-        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in input_d['wiki_items']])
-        question = input_d['Prompt']
-        prompt = self.prompt_template.format(context=context, question=question)
-        return self.gen_prompt_data(prompt)
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset='test',
+            limit=self.limit,
+            repeats=self.repeats
+        ).load()
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
+        test_dataset = DatasetDict({'test': dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        return input_d['Answer']
+        Convert a data record to a Sample object.
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in record['wiki_items']])
+        question = record['Prompt']
+
+        return Sample(
+            input=question, target=record['Answer'], metadata={
+                'context': context,
+                'wiki_items': record['wiki_items']
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = sample.metadata['context']
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
         """
-        Parse the predicted result and extract proper answer.
+        Extract the answer from the model prediction.
         """
-        response = result.replace('*', '')
+        response = prediction.replace('*', '')
 
         if 'the answer is' in response:
             ans = response.rsplit('the answer is', 1)[-1].strip().strip('.').strip()
@@ -67,25 +106,69 @@ class FramesAdapter(DataAdapter):
 
         return ans
 
-    def match(self, gold: str, pred: str) -> float:
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
         """
-        Match the gold answer and the predicted answer.
+        Calculate accuracy score by matching prediction with reference.
         """
+        from evalscope.metrics import exact_match
         from .utils import normalize_answer
-        gold = normalize_answer(gold)
-        pred = normalize_answer(pred)
-        return exact_match(gold=gold, pred=pred)
 
-    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        gold = normalize_answer(reference)
+        pred = normalize_answer(filtered_prediction)
+        accuracy = exact_match(gold=gold, pred=pred)
+
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
         from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
 
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['Prompt']
-        # get grading response
-        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
-        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
-        # parse grading response
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
         if 'YES' in orm_response:
-            return 1.0
+            accuracy = 1.0
         else:
-            return 0.0
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score