evalscope 0.5.0__py3-none-any.whl

evalscope/benchmarks/arc/ai2_arc.py

@@ -0,0 +1,148 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright (c) Allen Institute, and its affiliates.
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+
+"""AI2 ARC (Abstraction and Reasoning Corpus) for General Artificial Intelligence Benchmark."""
+
+"""AUTO GENERATED, DO NOT EDIT"""
+
+import json
+import os
+import datasets
+
+# flake8: noqa
+
+
+_CITATION = """\
+@article{allenai:arc,
+      author    = {Peter Clark  and Isaac Cowhey and Oren Etzioni and Tushar Khot and
+                    Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
+      title     = {Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
+      journal   = {arXiv:1803.05457v1},
+      year      = {2018},
+}
+"""
+
+_DESCRIPTION = """\
+A new dataset of 7,787 genuine grade-school level, multiple-choice science questions, assembled to encourage research in
+ advanced question-answering. The dataset is partitioned into a Challenge Set and an Easy Set, where the former contains
+ only questions answered incorrectly by both a retrieval-based algorithm and a word co-occurrence algorithm. We are also
+ including a corpus of over 14 million science sentences relevant to the task,
+ and an implementation of three neural baseline models for this dataset. We pose ARC as a challenge to the community.
+
+ARC-Easy:
+    train: 2251
+    test: 2376
+    validation: 570
+
+ARC-Challenge:
+    train: 1119
+    test: 1172
+    validation: 299
+"""
+
+_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/arc/ARC-V1-Feb2018.zip'
+
+# tasks: ['ARC-Easy', 'ARC-Challenge']
+
+
+class Ai2ArcConfig(datasets.BuilderConfig):
+    """BuilderConfig for Ai2ARC."""
+
+    def __init__(self, **kwargs):
+        """BuilderConfig for Ai2Arc.
+
+        Args:
+          **kwargs: keyword arguments forwarded to super.
+        """
+        super(Ai2ArcConfig, self).__init__(version=datasets.Version('1.0.0', ''), **kwargs)
+
+
+class Ai2Arc(datasets.GeneratorBasedBuilder):
+    """
+    The AI2 Reasoning Challenge (ARC) dataset.
+    Subset: ARC-Easy, ARC-Challenge.
+    """
+
+    VERSION = datasets.Version('1.0.0')
+    BUILDER_CONFIGS = [
+        Ai2ArcConfig(
+            name='ARC-Challenge',
+            description="""\
+          Challenge Set of 2590 “hard” questions (those that both a retrieval and a co-occurrence method fail to answer correctly)
+          """,
+        ),
+        Ai2ArcConfig(
+            name='ARC-Easy',
+            description="""\
+          Easy Set of 5197 questions
+          """,
+        ),
+    ]
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            # This is the description that will appear on the datasets page.
+            description=_DESCRIPTION,
+            # datasets.features.FeatureConnectors
+            features=datasets.Features(
+                {
+                    'id': datasets.Value('string'),
+                    'question': datasets.Value('string'),
+                    'choices': datasets.features.Sequence(
+                        {'text': datasets.Value('string'), 'label': datasets.Value('string')}
+                    ),
+                    'answerKey': datasets.Value('string')
+                    # These are the features of your dataset like images, labels ...
+                }
+            ),
+            # If there's a common (input, target) tuple from the features,
+            # specify them here. They'll be used if as_supervised=True in
+            # builder.as_dataset.
+            supervised_keys=None,
+            # Homepage of the dataset for documentation
+            homepage='https://allenai.org/data/arc',
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):
+        """Returns SplitGenerators."""
+        # dl_manager is a datasets.download.DownloadManager that can be used to
+        # download and extract URLs
+        dl_dir = dl_manager.download_and_extract(_URL)
+        data_dir = os.path.join(dl_dir, 'ARC-V1-Feb2018-2')
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Train.jsonl')},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Test.jsonl')},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                # These kwargs will be passed to _generate_examples
+                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Dev.jsonl')},
+            ),
+        ]
+
+    def _generate_examples(self, filepath):
+        """Yields examples."""
+        with open(filepath, encoding='utf-8') as f:
+            for row in f:
+                data = json.loads(row)
+                answerkey = data['answerKey']
+                id_ = data['id']
+                question = data['question']['stem']
+                choices = data['question']['choices']
+                text_choices = [choice['text'] for choice in choices]
+                label_choices = [choice['label'] for choice in choices]
+                yield id_, {
+                    'id': id_,
+                    'answerKey': answerkey,
+                    'question': question,
+                    'choices': {'text': text_choices, 'label': label_choices},
+                }

evalscope/benchmarks/arc/arc_adapter.py

@@ -0,0 +1,231 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+import json
+from evalscope.benchmarks.data_adapter import DataAdapter
+from evalscope.metrics.metrics import exact_match, weighted_mean
+from evalscope.utils import normalize_score, ResponseParser
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+DATASET_ID = 'modelscope/ai2_arc'
+
+# task_list = ['ARC-Easy', 'ARC-Challenge']
+SUBSET_LIST = ['ARC-Challenge']
+
+
+class ARCAdapter(DataAdapter):
+
+    choices = ['A', 'B', 'C', 'D']
+
+    def __init__(self,
+                 subset_list: list = None,
+                 metric_list: list = None,
+                 few_shot_num: int = None,
+                 train_split: str = 'train',
+                 eval_split: str = 'test',
+                 prompt_template: str = '',
+                 **kwargs):
+
+        if subset_list is None:
+            subset_list = SUBSET_LIST
+
+        if metric_list is None:
+            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
+
+        if few_shot_num is None:
+            # Use 0-shot by default
+            logger.info(f'Set 0-shot examples by system for ARC.')
+            few_shot_num = 0
+
+        if few_shot_num != 0:
+            logger.warning(f'few_shot_num is recommended to set 0 for ARC, got {few_shot_num}.')
+
+        super().__init__(subset_list=subset_list,
+                         metric_list=metric_list,
+                         few_shot_num=few_shot_num,
+                         train_split=train_split,
+                         eval_split=eval_split,
+                         prompt_template=prompt_template,
+                         **kwargs)
+
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
+        """
+        Load the dataset from local disk.
+
+        dataset_name_or_path: str, the dataset id or path. e.g. 'arc'
+        subset_list: list, the subset list to load. e.g. ['ARC-Easy', 'ARC-Challenge']
+        work_dir: str, the local root data directory. e.g. '/path/to/data'
+        kwargs: dict, other arguments.
+        """
+        data_dict = {}
+        for subset_name in subset_list:
+            if os.path.exists(dataset_name_or_path):
+                subset_path = os.path.join(dataset_name_or_path, subset_name)
+            else:
+                subset_path = os.path.join(work_dir, dataset_name_or_path, subset_name)
+            for split_name in ['Train', 'Test']:
+                split_path = os.path.join(subset_path, f'{subset_name}-{split_name}.jsonl')
+                if os.path.exists(split_path):
+                    with open(split_path, 'r', errors='ignore') as in_f:
+                        rows = []
+                        for line in in_f:
+                            item = json.loads(line.strip())
+                            raw_choices = item['question']['choices']
+                            rows.append({
+                                'id': item['id'],
+                                'question': item['question']['stem'],
+                                'choices': {'text': [d['text'] for d in raw_choices],
+                                            'label': [d['label'] for d in raw_choices]},
+                                'answerKey': item['answerKey'],
+                            })
+
+                        if subset_name in data_dict:
+                            data_dict[subset_name].update({split_name.lower(): rows})
+                        else:
+                            data_dict[subset_name] = {split_name.lower(): rows}
+
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from raw data, unify the prompt format for ARC benchmark.
+
+        Args:
+            input_d (dict): The raw input. A single data format of the ARC:
+
+            {
+                'id': 'Mercury_7220990',
+                'question': 'Which factor will most likely cause a person to develop a fever?',
+                'choices':
+                    {
+                        'text':['a leg muscle relaxing after exercise',
+                                'a bacterial population in the bloodstream',
+                                'several viral particles on the skin',
+                                'carbohydrates being digested in the stomach'],
+                        'label': ['A', 'B', 'C', 'D']
+                    },
+                'answerKey': 'B'
+            }
+
+        Returns:
+            {'data': ['xxx'], 'multi_choices': ['A', 'B', 'C', 'D']}
+        """
+        few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
+        context: str = '\n'.join(few_shot_prompts)
+
+        context = f'{self.prompt_template}\n{context}' if self.prompt_template else context
+
+        # context = f'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:\n {context}'
+        full_prompt: str = context + self._generate_prompt(input_d=input_d, include_answer=False)
+
+        return {'data': [full_prompt], 'multi_choices': self.choices}
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('answerKey', '')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d (dict): The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        if eval_type == 'checkpoint':
+            return result
+        elif eval_type == 'service':
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)  # TODO: to be checked !
+        elif eval_type == 'custom':
+            return ResponseParser.parse_first_option_with_choices(text=result, options=self.choices)  # TODO: to be checked !
+        else:
+            raise ValueError(f'Invalid eval_type: {eval_type}')
+
+    def match(self, gold: str, pred: str) -> float:
+        return exact_match(gold=gold, pred=pred)
+
+    def compute_metric(self, review_res_list: list) -> float:
+        """
+        Compute evaluation result by specific metric.
+
+        Args:
+            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
+
+        Returns:
+            The metric score.
+        """
+        items = [(score, 1.0) for score in review_res_list]
+        return weighted_mean(items)
+
+    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
+        """
+        Generate the report for the model output.
+
+        Args:
+            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
+            report_name: The user-defined report name.
+
+        Returns: A dict of metric calculation results. The format is like:
+        {
+            "name":"ARC",
+            "metric":"WeightedAverageAccuracy",
+            "score":0.3389,
+            "category":[
+                {
+                    "name":"DEFAULT",
+                    "score":0.4128,
+                    "subset":[
+                        {
+                            "name":"ARC-Easy",
+                            "score":0.5632
+                        },
+                        {
+                            "name":"ARC-Challenge",
+                            "score":0.3157
+                        }
+                    ]
+                }
+            ],
+            "total_num":7800
+        }
+        """
+        total_num: int = sum([num for _, num in subset_score_map.values()])
+        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
+        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
+        cate_avg_list = [{'name': subset_name, 'score': normalize_score(score=score)} for subset_name, (score, _) in subset_score_map.items()]
+
+        category_d = dict(name='DEFAULT',
+                          score=weighted_avg_acc,
+                          subset=cate_avg_list)
+
+        res_map = dict(name=report_name or 'arc',
+                       metric=self.metric_list[0]['name'],
+                       score=weighted_avg_acc,
+                       category=[category_d],
+                       total_num=total_num)
+
+        return res_map
+
+    @classmethod
+    def _generate_prompt(cls, input_d: dict, include_answer=True) -> str:
+
+        example: str = input_d['question']
+
+        choices_texts: list = input_d['choices']['text']
+        choices_labels: list = input_d['choices']['label']
+        choices_prompts: str = '\n'.join([label + '. ' + text for text, label in zip(choices_texts, choices_labels)])
+        example += '\n' + choices_prompts
+
+        example += '\nAnswer:'
+        if include_answer:
+            example += ' {}\n\n'.format(input_d['answerKey'])
+
+        return example

evalscope/benchmarks/bbh/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+
+from evalscope.benchmarks.bbh.bbh_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.bbh.bbh_adapter import BBHAdapter as DataAdapterClass
+from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass    # noqa

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -0,0 +1,308 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+import re
+import random
+import json
+
+from evalscope.benchmarks.data_adapter import DataAdapter
+from evalscope.constants import AnswerKeys
+from evalscope.metrics.metrics import exact_match, weighted_mean
+from evalscope.utils import normalize_score, ResponseParser
+from evalscope.utils.logger import get_logger
+# flake8: noqa
+
+logger = get_logger()
+
+DATASET_ID = 'modelscope/bbh'
+
+
+# BBH multiple choice subset list
+MULTIPLE_CHOICE = 'multiple_choice'
+MULTIPLE_CHOICE_LIST = [
+    'temporal_sequences',
+    'disambiguation_qa',
+    'date_understanding',
+    'tracking_shuffled_objects_three_objects',
+    'penguins_in_a_table',
+    'geometric_shapes',
+    'snarks',
+    'ruin_names',
+    'tracking_shuffled_objects_seven_objects',
+    'tracking_shuffled_objects_five_objects',
+    'logical_deduction_three_objects',
+    'hyperbaton',
+    'logical_deduction_five_objects',
+    'logical_deduction_seven_objects',
+    'movie_recommendation',
+    'salient_translation_error_detection',
+    'reasoning_about_colored_objects',
+]
+
+# The free form subset list of BBH dataset
+FREE_FORM = 'free_form'
+FREE_FORM_LIST = [
+    'multistep_arithmetic_two',
+    'navigate',
+    'dyck_languages',
+    'word_sorting',
+    'sports_understanding',
+    'boolean_expressions',
+    'object_counting',
+    'formal_fallacies',
+    'causal_judgement',
+    'web_of_lies',
+]
+
+# BBH sub-task type
+TASK_TYPE = 'task_type'
+SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
+
+
+class BBHAdapter(DataAdapter):
+    """
+    Adapter for BBH free-form and multiple-choices sub-tasks.
+    """
+
+    def __init__(self,
+                 subset_list: list = None,
+                 metric_list: list = None,
+                 few_shot_num: int = None,
+                 train_split: str = None,
+                 eval_split: str = 'test',
+                 **kwargs):
+
+        if subset_list is None:
+            subset_list = SUBSET_LIST
+
+        if metric_list is None:
+            metric_list = [{'name': 'WeightedAverageAccuracy', 'object': weighted_mean}]
+
+        if few_shot_num is None:
+            logger.info(f'Set 3-shot examples by system for BBH.')
+            few_shot_num = 3
+
+        if few_shot_num != 3 and few_shot_num != 0:
+            logger.error(f'BBH uses 3-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
+                         f'Use 3-shot by default.')
+            few_shot_num = 3
+
+        super().__init__(subset_list=subset_list,
+                         metric_list=metric_list,
+                         few_shot_num=few_shot_num,
+                         train_split=train_split,
+                         eval_split=eval_split,
+                         **kwargs)
+
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
+        data_dict = {}
+        for subset_name in subset_list:
+            for split_name in [self.eval_split]:
+                if os.path.exists(dataset_name_or_path):
+                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}.json')
+                else:
+                    file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
+                if os.path.exists(file_path):
+                    with open(file_path, 'r') as f:
+                        examples = json.load(f)['examples']
+                        if subset_name in data_dict:
+                            data_dict[subset_name].update({split_name: examples})
+                        else:
+                            data_dict[subset_name] = {split_name: examples}
+
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from raw data, unify the prompt format for bbh(multiple choice) benchmark.
+
+        Args:
+            input_d (dict): The raw input. A single data format of the BBH:
+
+            {
+                'input': '((-1 + 2 + 9 * 5) - (-2 + -4 + -4 * -7)) =',
+                'target': '24',
+            }
+
+        Returns:
+            {'data': ['xxx']}
+        """
+        # few_shot_list: should be ['xxxx']
+        cot_prompts: str = few_shot_list[0] if len(few_shot_list) > 0 else ''
+        full_prompt: str = f"Follow the given examples and answer the question.\n{cot_prompts}\n\nQ: {input_d['input']}\nA: Let's think step by step."
+
+        return {'data': [full_prompt]}
+
+    def gen_prompts(self, data_dict: dict) -> dict:
+        """
+        Generate dataset prompts from raw input, unify the prompt format for different datasets.
+
+        Args:
+            data_dict:  Refer to the output of load method: evalscope.benchmarks.benchmark.Benchmark.load
+
+        Returns:
+            {'subset_name': [prompt_d_1, prompt_d_2, ...]}
+            prompt_d_i (dict): refer to the output of gen_prompt method.
+
+        e.g. train -- few-shot data, test -- target dataset to evaluate.
+        """
+        res_dict: dict = {}
+
+        if self.few_shot_num < 0:
+            raise ValueError(f'Invalid shot_num: {self.few_shot_num} for few-shot evaluation.')
+
+        logger.info(f'\n** Use default settings: \n'
+                    f'>few_shot_num: {self.few_shot_num}, '
+                    f'>few_shot_split: {self.train_split}, '
+                    f'>target_eval_split: {self.eval_split}')
+
+        for sub_name, sub_data_dict in data_dict.items():
+            few_shot_data = []
+            if self.few_shot_num > 0:
+                with open(os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r') as f:
+                    cot_prompt_str = f.read()
+                few_shot_data = [cot_prompt_str]
+
+            res_dict[sub_name] = []
+            for sample_d in sub_data_dict[self.eval_split]:
+                prompt_d = self.gen_prompt(input_d=sample_d, few_shot_list=few_shot_data)
+                sample_d_new = sample_d.copy()
+                if sub_name in MULTIPLE_CHOICE_LIST:
+                    sample_d_new[TASK_TYPE] = MULTIPLE_CHOICE
+                elif sub_name in FREE_FORM_LIST:
+                    sample_d_new[TASK_TYPE] = FREE_FORM
+                else:
+                    raise ValueError(f'Invalid subset name: {sub_name}')
+
+                prompt_d[AnswerKeys.RAW_INPUT] = sample_d_new
+                res_dict[sub_name].append(prompt_d)
+
+        rnd = random.Random()
+        rnd.seed(42)
+        for k, v in res_dict.items():
+            rnd.shuffle(v)
+
+        return res_dict
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        gold = input_d.get('target')
+        if gold is None:
+            logger.error(f'BBHAdapter: gold is None.')
+        return gold
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d (dict): The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        # Note: to use same extraction method for both of checkpoint/service/custom.
+        task_type: str = raw_input_d.get(TASK_TYPE)
+
+        if task_type == MULTIPLE_CHOICE:
+            return self._extract_mc_answer(result)
+        elif task_type == FREE_FORM:
+            return self._extract_ff_answer(result)
+        else:
+            raise ValueError(f'Invalid task type: {task_type}')
+
+    def match(self, gold: str, pred: str) -> float:
+        return exact_match(gold=gold, pred=pred)
+
+    def compute_metric(self, review_res_list: list) -> float:
+        """
+        Compute evaluation result by specific metric.
+
+        Args:
+            review_res_list: review score list, e.g. [0, 1, 1, 0, ...]
+
+        Returns:
+            The metric score.
+        """
+        items = [(score, 1.0) for score in review_res_list]
+        return weighted_mean(items)
+
+    def gen_report(self, subset_score_map: dict, report_name: str = None) -> dict:
+        """
+        Generate the report for the model output.
+
+        Args:
+            subset_score_map: The subset-score mapping. e.g. {subset_name: (score, num), ...}
+            report_name: The user-defined report name.
+
+        Returns: A dict of metric calculation results. The format is like:
+        {
+            "name":"BBH",
+            "metric":"WeightedAverageAccuracy",
+            "score":0.3389,
+            "category":[
+                {
+                    "name":"DEFAULT",
+                    "score":0.3389,
+                    "subset":[
+                        {
+                            "name":"BBH",
+                            "score":0.3389
+                        },
+                    ]
+                }
+            ],
+            "total_num":100
+        }
+        """
+        total_num: int = sum([num for _, num in subset_score_map.values()])
+        weighted_avg_acc: float = sum([score * num for score, num in subset_score_map.values()]) / total_num
+        weighted_avg_acc = normalize_score(score=weighted_avg_acc)
+        cate_avg_list = [{'name': subset_name, 'score': normalize_score(score=score)} for subset_name, (score, _) in subset_score_map.items()]
+
+        category_d = dict(name='DEFAULT',
+                          score=weighted_avg_acc,
+                          subset=cate_avg_list)
+
+        res_map = dict(name=report_name or 'bbh',
+                       metric=self.metric_list[0]['name'],
+                       score=weighted_avg_acc,
+                       category=[category_d],
+                       total_num=total_num)
+
+        return res_map
+
+    @classmethod
+    def _extract_mc_answer(cls, ans: str) -> str:
+        """
+        Extract the answer from the model output for Multiple choice task.
+        """
+        ans_line = ans.split('answer is ')
+        if len(ans_line) != 1:
+            ans = ans_line[1].strip()
+        match = re.search(r'\(([A-Z])\)*', ans)
+        if match:
+            return match.group(1)
+        match = re.search(r'([A-Z])', ans)
+        if match:
+            return match.group(1)
+        return ans
+
+    @classmethod
+    def _extract_ff_answer(cls, ans: str):
+        """
+        Extract the answer from the model output for Free-form task.
+        """
+        res = ResponseParser.parse_first_option(ans)
+        if res:
+            return res
+
+        ans_line = ans.split('answer is ')
+        if len(ans_line) != 1:
+            ans = ans_line[1].strip()
+        ans = ans.split('\n')[0]
+        if ans.endswith('.'):
+            ans = ans[:-1]
+        return ans