evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/process_bench/process_bench_adapter.py

@@ -1,100 +1,168 @@
-import os
+# flake8: noqa: E501
 import re
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys, EvalType
-from evalscope.metrics import Metric, mean, metric_registry, simple_f1_score
-
-cur_path = os.path.dirname(os.path.abspath(__file__))
-
-
-@Benchmark.register(
-    name='process_bench',
-    pretty_name='ProcessBench',
-    tags=['Mathematical', 'Reasoning'],
-    description=
-    'ProcessBench is a benchmark for evaluating AI models on mathematical reasoning tasks. It includes various subsets such as GSM8K, Math, OlympiadBench, and OmniMath, each with its own set of problems that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
-    dataset_id='Qwen/ProcessBench',
-    subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
-    metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class ProcessBenchAdapter(DataAdapter):
+from typing import Any, Dict, List
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.metric.scorer import AggScore, SampleScore
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
 
-        self.prompt_template = open(os.path.join(cur_path, 'critique_template.txt'), encoding='utf-8').read()
+logger = get_logger()
 
-        # register metrics
-        metric_registry.register(Metric(name='error_acc', object=mean))
-        metric_registry.register(Metric(name='correct_acc', object=mean))
-        metric_registry.register(Metric(name='simple_f1_score', object=simple_f1_score))
+CRITIQUE_TEMPLATE = """CThe following is a math problem and a solution (split into paragraphs, enclosed with tags and indexed from 0):
 
-    def load(self, **kwargs):
-        # default load all levels
-        kwargs['split_as_subset'] = True
-        data_dict = super().load(**kwargs)
-        return data_dict
+[Math Problem]
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+{problem}
 
-        problem = input_d['problem']
-        steps = input_d['steps']
-        tagged_response = ''
-        for sdx, step in enumerate(steps):
-            tagged_response += f'<paragraph_{sdx}>\n{step}\n</paragraph_{sdx}>\n\n'
-        tagged_response = tagged_response.strip()
+[Solution]
 
-        full_prompt = self.prompt_template.format(problem=problem, tagged_response=tagged_response)
+{tagged_response}
 
-        return self.gen_prompt_data(full_prompt)
+Your task is to review and critique the solution paragraph by paragraph. Once you identify an error in a paragraph, return the index of the paragraph where the earliest error occurs. Otherwise, return the index of -1 (which typically denotes "not found").
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return int(input_d['label'])
+Please put your final answer (i.e., the index) in \boxed{{}}.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='process_bench',
+        pretty_name='ProcessBench',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'ProcessBench is a benchmark for evaluating AI models on mathematical reasoning tasks. It includes various subsets such as GSM8K, Math, OlympiadBench, and OmniMath, each with its own set of problems that require step-by-step reasoning to arrive at the correct answer.',  # noqa: E501
+        dataset_id='Qwen/ProcessBench',
+        subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
+        metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
+        eval_split='test',
+        prompt_template=CRITIQUE_TEMPLATE
+    )
+)
+class ProcessBenchAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.split_as_subset = True  # Use split as subset
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """
-        Parse the predicted result and extract proper answer.
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
         """
-        pred = ProcessBenchAdapter.extract_answer(result)
+        problem = record['problem']
+        steps = record['steps']
+        tagged_response = ''
+        for sdx, step in enumerate(steps):
+            tagged_response += f'<paragraph_{sdx}>\n{step}\n</paragraph_{sdx}>\n\n'
+        tagged_response = tagged_response.strip()
+
+        return Sample(
+            input=problem,
+            target=str(record['label']),
+            metadata={
+                'steps': steps,
+                'tagged_response': tagged_response,
+                'final_answer_correct': record['final_answer_correct']
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        """Format the prompt template with problem and tagged response."""
+        problem = sample.input
+        tagged_response = sample.metadata['tagged_response']
+        return self.prompt_template.format(problem=problem, tagged_response=tagged_response)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        """Extract the answer from the model prediction."""
+        pred = self._extract_answer_from_text(prediction)
         try:
             pred = int(pred)
         except Exception:
             pred = None
         return pred
 
-    def match(self, gold: int, pred: int) -> float:
-        """
-        Match the gold answer and the predicted answer.
-        """
-        return gold == pred
-
-    def compute_metric(self, review_res_list: list, **kwargs) -> List[dict]:
-        reviews_list = kwargs['reviews_list']
-        error_data = []
-        correct_data = []
-        for res, raw in zip(review_res_list, reviews_list):
-            if raw[AnswerKeys.RAW_INPUT]['label'] == -1:
-                correct_data.append(res)
-            else:
-                error_data.append(res)
-        data = {}
-        if len(correct_data) != 0:
-            data.update({'correct_acc': correct_data})
-        if len(error_data) != 0:
-            data.update({'error_acc': error_data})
-        data.update({'simple_f1_score': (correct_data, error_data)})
-        return super().compute_metric(data)
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        """Calculate evaluation scores by comparing prediction with reference."""
+        score = Score(
+            extracted_prediction=str(filtered_prediction) if filtered_prediction is not None else None,
+            prediction=original_prediction,
+        )
+
+        # Convert filtered_prediction to int if possible
+        try:
+            pred_int = int(filtered_prediction) if filtered_prediction is not None else None
+        except (ValueError, TypeError):
+            pred_int = None
+
+        # Calculate accuracy
+        reference = int(reference) if reference is not None else None
+        accuracy = 1.0 if reference == pred_int else 0.0
+
+        # Determine metric name based on label
+        if reference == -1:
+            metric_name = 'correct_acc'
+        else:
+            metric_name = 'error_acc'
+
+        score.value = {metric_name: accuracy}
+        score.main_score_name = metric_name
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """Aggregate scores to compute final metrics including F1 score."""
+        correct_scores = []
+        error_scores = []
+
+        for sample_score in sample_scores:
+            score = sample_score.score
+            if 'correct_acc' in score.value:
+                correct_scores.append(score.value['correct_acc'])
+            elif 'error_acc' in score.value:
+                error_scores.append(score.value['error_acc'])
+
+        agg_list = []
+
+        if correct_scores:
+            agg_list.append(
+                AggScore(
+                    metric_name='correct_acc', score=sum(correct_scores) / len(correct_scores), num=len(correct_scores)
+                )
+            )
+
+        if error_scores:
+            agg_list.append(
+                AggScore(metric_name='error_acc', score=sum(error_scores) / len(error_scores), num=len(error_scores))
+            )
+
+        # Calculate simple F1 score
+        if correct_scores and error_scores:
+            from evalscope.metrics import simple_f1_score
+            agg_list.append(
+                AggScore(
+                    metric_name='simple_f1_score',
+                    score=simple_f1_score((correct_scores, error_scores)),
+                    num=len(correct_scores) + len(error_scores)
+                )
+            )
+
+        return agg_list
 
     @staticmethod
-    def extract_answer(solution_text: str):
+    def _extract_answer_from_text(solution_text: str):
+        """Extract answer from solution text using boxed pattern."""
         boxed_pattern = r'\\boxed\{([^}]*)\}'
         matches = re.findall(boxed_pattern, solution_text)
         if matches:

evalscope/benchmarks/race/race_adapter.py

@@ -1,135 +1,49 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='race',
-    pretty_name='RACE',
-    tags=['Reasoning', 'MCQ'],
-    description=
-    'RACE is a benchmark for testing reading comprehension and reasoning abilities of neural models. It is constructed from Chinese middle and high school examinations.',  # noqa: E501
-    dataset_id='modelscope/race',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['high', 'middle'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=3,
-    train_split='train',
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='race',
+        pretty_name='RACE',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=
+        'RACE is a benchmark for testing reading comprehension and reasoning abilities of neural models. It is constructed from Chinese middle and high school examinations.',  # noqa: E501
+        dataset_id='evalscope/race',
+        metric_list=['acc'],
+        subset_list=['high', 'middle'],
+        few_shot_num=3,
+        train_split='train',
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
 )
-class RACEAdapter(DataAdapter):
+class RACEAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
-        few_shot_num = kwargs.get('few_shot_num', 3)
-        if few_shot_num > 3:
-            logger.warning(f'few_shot_num <= 3 for RACE, but got {few_shot_num}. Use 3-shot by default.')
-            kwargs['few_shot_num'] = 3
-
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, subset_name, f'{split}.jsonl')
-                else:
-                    file_path = os.path.join(work_dir, dataset_name_or_path, subset_name, f'{split}.jsonl')
-                if os.path.exists(file_path):
-                    data_dict[subset_name][split] = jsonl_to_list(file_path)
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for RACE benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the RACE:
-
-            {'example_id': 'high3680.txt',
-            'article': 'Astronauts on shorter shuttle missions often work very long days. Tasks are scheduled so tightly that break times are often used to finish the day's work. This type of schedule is far too demanding for long missions on the International Space Station(ISS). ISS crewmembers usually live in space for at least a quarter of a year. They work five days on and two days off to _ the normal way they do things on Earth as much as possible. Weekends give the crew valuable time to rest and do a few hours of housework. They can communicate with family and friends by email , internet phone and through private video conferences. While astronauts cannot go to a baseball game or a movie in orbit, there are many familiar activities that they can still enjoy . Before a mission, the family and friends of each ISS crewmember put together a collection of family photos, messages, videos and reading material for the astronauts to look at when they will be floating 370 kilometers above the Earth. During their mission, the crew also receives care packages with CDs, books, magazines, photos and letters . And as from early 2010, the internet became available on the ISS , giving astronauts the chance to do some "web surfing "in their personal time. Besides relaxing with these more common entertainments, astronauts can simply enjoy the experience of living in space. Many astronauts say that one of the most relaxing things to do in space is to look out the window and stare at the universe and the Earth's vast land mass and oceans.',
-            'answer': 'C',
-            'question': 'The passage mainly discusses how astronauts _ .',
-            'options': [
-                "work for longer missions in space",
-                "connect with people on the Earth",
-                "spend their free time in space",
-                "observe the Earth from space"]}
-
-        Returns:
-            {'data': [(context, continuation), ...]}
-
-        """
-        prompt = 'The following are multiple choice reading comprehension questions (with answers).\n\n'.format(
-            self._format_subject(subset_name))
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        context = prompt + context
-
-        full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        return self.gen_prompt_data(full_prompt)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: The evaluation type. e.g. 'checkpoint' or 'service' or 'custom'.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    def _generate_prompt(self, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = input_d['options']
-
-        example: str = 'Article:\n{}\nQuestion:\n{}'.format(input_d['article'], input_d['question'])
-        for j in range(len(self.choices)):
-            example += '\n{}. {}'.format(self.choices[j], input_choices[j])
-
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['answer'])
+        if self.few_shot_num > 3:
+            logger.warning(f'few_shot_num <= 3 for RACE, but got {self.few_shot_num}. Use 3-shot by default.')
+            self.few_shot_num = 3
 
-        return example
+    def record_to_sample(self, record) -> Sample:
+        # Format the article and question as context
+        context = f"Article:\n{record['article']}\nQuestion:\n{record['question']}"
 
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+        return Sample(
+            input=context,
+            choices=record['options'],
+            target=record['answer'],
+            metadata={'example_id': record.get('example_id', 'unknown')},
+        )

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -1,13 +1,15 @@
+import ast
 import re
-from collections import defaultdict
-from typing import Any, List
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 GRADER_TEMPLATE = """
@@ -76,7 +78,7 @@ Also note the following things:
     - For example, if the gold target is "Hyung Won Chung", you can consider the following predicted answers as correct: "Hyoong Won Choong", "Hyungwon Chung", or "Hyun Won Chung".
 
 
-Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT ATTEMPTED. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer.
+Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT_ATTEMPTED. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer.
 ```
 Question: {question}
 Gold target: {target}
@@ -92,76 +94,76 @@ Just return the letters "A", "B", or "C", with no text around it.
 """.strip()  # noqa: E501
 
 
-@Benchmark.register(
-    name='simple_qa',
-    pretty_name='SimpleQA',
-    tags=['Knowledge', 'QA'],
-    description=
-    'SimpleQA is a benchmark designed to evaluate the performance of language models on simple question-answering tasks. It includes a set of straightforward questions that require basic reasoning and understanding capabilities.',  # noqa: E501
-    dataset_id='AI-ModelScope/SimpleQA',
-    metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test')
-class SimpleQAAdapter(DataAdapter):
+@register_benchmark(
+    BenchmarkMeta(
+        name='simple_qa',
+        pretty_name='SimpleQA',
+        tags=[Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'SimpleQA is a benchmark designed to evaluate the performance of language models on simple question-answering tasks. It includes a set of straightforward questions that require basic reasoning and understanding capabilities.',  # noqa: E501
+        dataset_id='AI-ModelScope/SimpleQA',
+        metric_list=['is_correct', 'is_incorrect', 'is_not_attempted'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='Answer the question:\n\n{question}'
+    )
+)
+class SimpleQAAdapter(DefaultDataAdapter):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-        # register metrics
-        metric_registry.register(Metric(name='is_correct', object=mean))
-        metric_registry.register(Metric(name='is_incorrect', object=mean))
-        metric_registry.register(Metric(name='is_not_attempted', object=mean))
-
-        # whether to use LLM as a judge
-        self.llm_as_a_judge = True
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        question = input_d['problem']
-        return self.gen_prompt_data(question)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, **kwargs) -> str:
-        return result.strip()
-
-    def match(self, gold: str, pred: str) -> float:
-        # simple match
-        logger.warning(f'Please use LLMJudge to match the result for {self.name}')
-        is_correct = 1 if gold.lower().strip() == pred.lower().strip() else 0
-        is_incorrect = not is_correct
-        is_not_attempted = 0
-        return {
-            'is_correct': is_correct,
-            'is_incorrect': is_incorrect,
-            'is_not_attempted': is_not_attempted,
-        }
+        self._use_llm_judge = True  # Use LLM as a judge by default
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
 
-    def llm_match(self, gold: Any, pred: Any, judge: LLMJudge, **kwargs) -> dict:
-        raw_input = kwargs.get('raw_input', None)
-        question = raw_input['problem']
-        # get grading response
-        prompt = GRADER_TEMPLATE.format(question=question, target=gold, predicted_answer=pred)
-        grading_response = judge(prompt)
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['problem']
+        answer = record['answer']
+        metadata = record.get('metadata')
+
+        return Sample(input=question, target=answer, metadata=ast.literal_eval(metadata))
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = GRADER_TEMPLATE.format(question=question, target=reference, predicted_answer=filtered_prediction)
+        judge_response = self.llm_judge.judge(prompt)
         # parse grading response
-        match = re.search(r'(A|B|C)', grading_response)
+        match = re.search(r'(A|B|C)', judge_response)
         res = match.group(0) if match else 'C'
-        return {
+
+        # Set score based on the match result
+        score.value = {
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
-            'judge_response': grading_response,
         }
-
-    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
-        """
-        # zip dict answers
-        res_dict = super().compute_dict_metric(review_res_list, **kwargs)
-
-        return super().compute_metric(res_dict, **kwargs)
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'is_correct'
+        return score