evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -1,35 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import csv
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser, normalize_score
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
 
 # flake8: noqa
 
 logger = get_logger()
 
-SUBSET_LIST = [
-    'agronomy', 'anatomy', 'ancient_chinese', 'arts', 'astronomy', 'business_ethics', 'chinese_civil_service_exam',
-    'chinese_driving_rule', 'chinese_food_culture', 'chinese_foreign_policy', 'chinese_history', 'chinese_literature',
-    'chinese_teacher_qualification', 'college_actuarial_science', 'college_education', 'college_engineering_hydrology',
-    'college_law', 'college_mathematics', 'college_medical_statistics', 'clinical_knowledge', 'college_medicine',
-    'computer_science', 'computer_security', 'conceptual_physics', 'construction_project_management', 'economics',
-    'education', 'elementary_chinese', 'elementary_commonsense', 'elementary_information_and_technology',
-    'electrical_engineering', 'elementary_mathematics', 'ethnology', 'food_science', 'genetics', 'global_facts',
-    'high_school_biology', 'high_school_chemistry', 'high_school_geography', 'high_school_mathematics',
-    'high_school_physics', 'high_school_politics', 'human_sexuality', 'international_law', 'journalism',
-    'jurisprudence', 'legal_and_moral_basis', 'logical', 'machine_learning', 'management', 'marketing',
-    'marxist_theory', 'modern_chinese', 'nutrition', 'philosophy', 'professional_accounting', 'professional_law',
-    'professional_medicine', 'professional_psychology', 'public_relations', 'security_study', 'sociology',
-    'sports_science', 'traditional_chinese_medicine', 'virology', 'world_history', 'world_religions'
-]
-
 SUBJECT_MAPPING = {
     'agronomy': ['other', 'Other'],
     'anatomy': ['biology', 'STEM'],
@@ -101,128 +82,41 @@ SUBJECT_MAPPING = {
 }
 
 
-@Benchmark.register(
-    name='cmmlu',
-    dataset_id='modelscope/cmmlu',
-    model_adapter=MultiChoiceModelAdapter,
-    subset_list=SUBSET_LIST,
-    metric_list=[AverageAccuracy],
-    few_shot_num=5,
-    train_split='dev',
-    eval_split='test',
+@register_benchmark(
+    BenchmarkMeta(
+        name='cmmlu',
+        pretty_name='C-MMLU',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'C-MMLU is a benchmark designed to evaluate the performance of AI models on Chinese language tasks, including reading comprehension, text classification, and more.',
+        dataset_id='evalscope/cmmlu',
+        metric_list=['acc'],
+        subset_list=list(SUBJECT_MAPPING.keys()),
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT,
+    )
 )
-class CMMLUAdapter(DataAdapter):
-
-    choices = ['A', 'B', 'C', 'D']
+class CMMLUAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
+        self.reformat_subset = True
         self.category_map = {k: v[-1] for k, v in SUBJECT_MAPPING.items()}
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            for split_name in [self.train_split, self.eval_split]:
-                file_path = os.path.join(work_dir, dataset_name_or_path, split_name, f'{subset_name}.csv')
-                if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        for row in reader:
-                            if len(row) != 7:
-                                logger.error(f'Mismatch len of row: {row}, len of row should be 6. Skip this row.')
-                                continue
-                            rows.append({
-                                'Question': row[1],
-                                'A': row[2],
-                                'B': row[3],
-                                'C': row[4],
-                                'D': row[5],
-                                'Answer': row[6],
-                            })
-
-                        data_dict[subset_name].update({split_name: rows})
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw input, unify the prompt format for CMMLU benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the CMMLU:
-
-            {'Question': '下列关于重力的说法正确的是',
-            'A': '在地球周围的物体都要受到重力作用，与其运动状态无关',
-            'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变',
-            'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下',
-            'D': '在地球表面各处的重力方向都是相同的',
-            'Answer': 'A'}
-
-        Returns:
-            {'data': [(context, continuation), ...]}
-
-        """
-        prompt = '以下是关于{}的单项选择题。\n\n'.format(self._format_subject(subset_name))
-        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
-
-        context: str = '\n'.join(few_shot_prompts) + '\n'
-        context += self._generate_prompt(input_d=input_d, include_answer=False)
-        context = prompt + context
-
-        full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
-
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': prompt}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        return input_d.get('Answer', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: The evaluation type. 'checkpoint', 'service', 'custom'.
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        elif eval_type == EvalType.CUSTOM:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)  # TODO: to be checked !
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
-
-        input_choices: list = [input_d['A'], input_d['B'], input_d['C'], input_d['D']]
-
-        example: str = input_d['Question']
-        for j in range(len(cls.choices)):
-            example += '\n{}. {}'.format(cls.choices[j], input_choices[j])
-
-        example += '\nAnswer:'
-        if include_answer:
-            example += ' {}\n\n'.format(input_d['Answer'])
+    def record_to_sample(self, record) -> Sample:
 
-        return example
+        # choices: ["(A) 农业生产工具","(B) 土地","(C) 劳动力","(D) 资金"]
+        # remove the leading (A), (B), (C), (D)
+        raw_choices = record['choices']
+        choice_list = [choice[3:].strip() for choice in raw_choices]
 
-    @classmethod
-    def _format_subject(cls, subject):
-        l = subject.split('_')
-        s = ''
-        for entry in l:
-            s += ' ' + entry
-        return s
+        return Sample(
+            input=record['question'],
+            choices=choice_list,
+            target=record['answer'][1],  # answer is like "A"
+            subset_key=record['category'],
+            metadata={'subject': record['category']},
+        )

evalscope/benchmarks/coin_flip/coin_flip_adapter.py

@@ -0,0 +1,128 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = (
+    "CoinFlip is a symbolic reasoning dataset that tests an LLM's ability "
+    'to track binary state changes through a sequence of actions. '
+    'Each example describes whether a coin is flipped or not by different person, '
+    'requiring logical inference to determine the final state (heads or tails).'
+)  # noqa: E501
+
+PROMPT_TEMPLATE = """
+Solve the following coin flip problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer YES or NO to the problem.
+
+Reasoning:
+"""  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
+
+""".lstrip() + PROMPT_TEMPLATE  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='coin_flip',
+        pretty_name='CoinFlip',
+        tags=[Tags.REASONING, Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/coin-flip',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        train_split='validation',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CoinFlipAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        answer = record['answer']
+        input_text = self.prompt_template.format(question=question)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        answer = str(answer).upper()  # 'YES' or 'NO'
+        return Sample(input=[ChatMessageUser(content=content_list)], target=answer, metadata={
+            'answer': answer,
+        })
+
+    def extract_answer(self, prediction, task_state):
+        import re
+
+        match = re.search(r'ANSWER:\s*(.*)', prediction)
+        return match.group(1) if match else prediction
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check for an exact match against the extracted answer.
+        result = 1 if reference in filtered_prediction else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        tp = fp = tn = fn = 0
+        yes_count = 0
+        total_count = len(sample_scores)
+
+        for ss in sample_scores:
+            gt = ss.sample_metadata['answer'].strip().upper()
+            pred = ss.score.extracted_prediction.strip().upper()
+
+            if pred == 'YES':
+                yes_count += 1
+            if pred == 'YES' and gt == 'YES':
+                tp += 1
+            elif pred == 'YES' and gt == 'NO':
+                fp += 1
+            elif pred == 'NO' and gt == 'NO':
+                tn += 1
+            elif pred == 'NO' and gt == 'YES':
+                fn += 1
+
+        accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+        yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+        overall_metrics = {
+            'accuracy': accuracy,
+            'precision': precision,
+            'recall': recall,
+            'f1_score': f1_score,
+            'yes_ratio': yes_ratio
+        }
+
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/commonsense_qa/commonsense_qa_adapter.py

@@ -0,0 +1,32 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = 'CommonsenseQA requires different types of commonsense knowledge to predict the correct answers.'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='commonsense_qa',
+        pretty_name='CommonsenseQA',
+        tags=[Tags.REASONING, Tags.COMMONSENSE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/commonsense-qa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class CommonsenseQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -1,126 +1,78 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI, Inc. and its affiliates.
-import glob
-import json
-import os
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import AverageAccuracy
-from evalscope.metrics.math_accuracy import is_equiv, last_boxed_only_string, remove_boxed
-from evalscope.models import ChatGenerationModelAdapter
+
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-
-@Benchmark.register(
-    name='competition_math',
-    dataset_id='modelscope/competition_math',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
-    metric_list=[AverageAccuracy],
-    few_shot_num=4,
-    train_split='train',
-    eval_split='test',
-    prompt_template='Put the final answer in \\boxed{}.',
+PROMPT_TEMPLATE = """
+Problem:
+{question}
+
+Please reason step by step, and put your final answer within \\boxed{{}}.
+""".lstrip()
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='competition_math',
+        pretty_name='MATH',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'The MATH (Mathematics) benchmark is designed to evaluate the mathematical reasoning abilities of AI models through a variety of problem types, including arithmetic, algebra, geometry, and more.',
+        dataset_id='evalscope/competition_math',
+        subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        few_shot_num=4,
+        train_split='train',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class CompetitionMathAdapter(DataAdapter):
-    """ To be tested for all models. """
+class CompetitionMathAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
-        few_shot_num = kwargs.get('few_shot_num', 4)
-        if few_shot_num != 4 and few_shot_num != 0:
-            logger.error(f'The MATH benchmark ONLY supports 4-shot by system or 0-shot settings, '
-                         f'but got {few_shot_num}. Use 4-shot by default.')
-            kwargs['few_shot_num'] = 4
+        self.reformat_subset = True
 
-        super().__init__(**kwargs)
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        from evalscope.metrics.math_parser import extract_answer
+
+        return Sample(
+            input=record['problem'],
+            target=extract_answer(record['solution']),
+            subset_key=record['level'],
+            metadata={
+                'reasoning': record.get('solution', ''),
+                'type': record.get('type', ''),
+            },
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        return f'Problem:\n{sample.input}\nSolution:\n{sample.target}'
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict: dict = {}
-        for subset_name in subset_list:
-            for split_name in [self.train_split, self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    split_dir = os.path.join(dataset_name_or_path, split_name)
-                else:
-                    split_dir = os.path.join(work_dir, dataset_name_or_path, split_name)
-                split_files = glob.glob(os.path.join(split_dir, '**', '*.json'))
-                split_data = []
-                for file_path in split_files:
-                    if os.path.exists(file_path):
-                        with open(file_path, 'r') as f:
-                            split_data.append(json.load(f))
-                if subset_name in data_dict:
-                    data_dict[subset_name].update({split_name: split_data})
-                else:
-                    data_dict[subset_name] = {split_name: split_data}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-
-        Args:
-            input_d: raw input dict.
-                {"problem": "How many vertical asymptotes does the graph of $y=\\frac{2}{x^2+x-6}$ have?", "level": "Level 3", "type": "Algebra", "solution": "The denominator of the rational function factors into $x^2+x-6=(x-2)(x+3)$. Since the numerator is always nonzero, there is a vertical asymptote whenever the denominator is $0$, which occurs for $x = 2$ and $x = -3$.  Therefore, the graph has $\\boxed{2}$ vertical asymptotes."}
-
-            few_shot_list:  few shot list. Each item is a raw input dict.
-            **kwargs:
-
-        Returns:
-            {'data': [prompt]}
-        """
-        use_fewshot = self.few_shot_num > 0
-        full_prompt = self._generate_prompt(input_d, use_fewshot=use_fewshot)
-
-        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return remove_boxed(last_boxed_only_string(input_d['solution']))
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: Use same extraction method for both of checkpoint/service/custom
-        try:
-            result = remove_boxed(last_boxed_only_string(result))
-        except Exception:
-            return None
-        return result
-
-    def match(self, gold: str, pred: str) -> float:
-        res = 0
-        if is_equiv(pred, gold):
-            res = 1
-
-        return res
-
-    @classmethod
-    def _generate_prompt(cls, input_d: dict, use_fewshot: bool = True) -> str:
-        problem: str = input_d['problem']
-
-        if use_fewshot:
-            # Use 4-shot examples by system
-            context = (
-                'Problem:\nFind the domain of the expression $\\frac{{\sqrt{{x-2}}}}{{\sqrt{{5-x}}}}$.}}\nSolution:\nThe expressions inside each square root must be non-negative. Therefore, $x-2 \ge 0$, so $x\ge2$, and $5 - x \ge 0$, so $x \le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{{[2,5)}}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.\n'
-                'Problem:\nIf $\det \mathbf{{A}} = 2$ and $\det \mathbf{{B}} = 12,$ then find $\det (\mathbf{{A}} \mathbf{{B}}).$\nSolution:\nWe have that $\det (\mathbf{{A}} \mathbf{{B}}) = (\det \mathbf{{A}})(\det \mathbf{{B}}) = (2)(12) = \\boxed{{24}}.$\nFinal Answer: The final answer is $24$. I hope it is correct.\n'
-                'Problem:\nTerrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?\nSolution:\nIf Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\cdot 12\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\cdot15\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$: \\begin{{align*}} 30n&=480\\\\ \Rightarrow\qquad n&=480/30=\\boxed{{16}} \end{{align*}}\nFinal Answer: The final answer is $16$. I hope it is correct.\n'
-                'Problem:\nIf the system of equations: \\begin{{align*}} 6x-4y&=a,\\\\ 6y-9x &=b. \end{{align*}}has a solution $(x, y)$ where $x$ and $y$ are both nonzero, find $\\frac{{a}}{{b}},$ assuming $b$ is nonzero.\nSolution:\nIf we multiply the first equation by $-\\frac{{3}}{{2}}$, we obtain $$6y-9x=-\\frac{{3}}{{2}}a.$$Since we also know that $6y-9x=b$, we have $$-\\frac{{3}}{{2}}a=b\Rightarrow\\frac{{a}}{{b}}=\\boxed{{-\\frac{{2}}{{3}}}}.$$\nFinal Answer: The final answer is $-\\frac{{2}}{{3}}$. I hope it is correct.\n'
-                f'Problem:\n{problem}\nSolution:\n')
-        else:
-            context = 'Problem:\n' + problem + '\nSolution:\n'
-        return context
+        return extract_answer(prediction)