evalscope 0.10.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -2,10 +2,11 @@
 import os.path
 import random
 from abc import ABC, abstractmethod
-from typing import Any, List, Optional
+from collections import defaultdict
+from typing import Any, List, Optional, Union
 
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
-from evalscope.metrics import Metric
+from evalscope.metrics.named_metrics import metric_registry
 from evalscope.report import Report, ReportGenerator
 from evalscope.utils.logger import get_logger
 
@@ -16,12 +17,16 @@ class DataAdapter(ABC):
 
     def __init__(self,
                  name: str,
+                 dataset_id: str,
                  subset_list: list,
-                 metric_list: List[Metric],
+                 metric_list: List[str],
                  few_shot_num: Optional[int] = 0,
                  train_split: Optional[str] = None,
                  eval_split: Optional[str] = None,
                  prompt_template: Optional[str] = None,
+                 system_prompt: Optional[str] = None,
+                 query_template: Optional[str] = None,
+                 pretty_name: Optional[str] = None,
                  **kwargs):
         """
         Data Adapter for the benchmark. You need to implement the following methods:
@@ -31,6 +36,7 @@ class DataAdapter(ABC):
             - match
         Args:
             name: str, the name of the benchmark.
+            dataset_id: str, the dataset id on ModelScope or local path for the benchmark.
             subset_list: list of subset names for the dataset.
             metric_list: list, the metric list to evaluate the model on specific benchmark.
             few_shot_num: int, number of few-shot examples. Default: 0
@@ -41,20 +47,23 @@ class DataAdapter(ABC):
                     the form of A or B or C or D, do not output explanation:`
         """
         self.name = name
+        self.dataset_id = dataset_id
         self.subset_list = subset_list
         self.metric_list = metric_list
         self.few_shot_num = few_shot_num
         self.train_split = train_split
         self.eval_split = eval_split
         self.prompt_template = prompt_template
+        self.system_prompt = system_prompt
+        self.query_template = query_template
+        self.pretty_name = pretty_name
         self.config_kwargs = kwargs
         self.category_map = kwargs.get('category_map', {})
 
     def load(self,
-             dataset_name_or_path: str,
+             dataset_name_or_path: str = None,
              subset_list: list = None,
              work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
              **kwargs) -> dict:
         """
         Load the dataset. Remote and local datasets are supported.
@@ -64,27 +73,45 @@ class DataAdapter(ABC):
             train_dataset, test_dataset: Iterable dataset, object each item of which is a dict.
 
         """
-        dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
+        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
         subset_list = subset_list or self.subset_list
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                    subsets: {subset_list}')
             data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
-            if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
         else:
-            from modelscope.msdatasets import MsDataset
+            data_dict = self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
+        if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
+            raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+        return data_dict
+
+    def load_from_hub(self, dataset_name_or_path: str, subset_list: list, work_dir: str, **kwargs) -> dict:
+        from modelscope.msdatasets import MsDataset
+
+        datasets_hub: str = kwargs.pop('datasets_hub', HubType.MODELSCOPE)
+        split_as_subset: bool = kwargs.pop('split_as_subset', False)
+        # Load dataset from remote
+        logger.info(
+            f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
 
-            # Load dataset from remote
-            logger.info(
-                f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
-            data_dict = {}
-            split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-            if len(split_list) == 0:
-                logger.error(f'Got empty split list: {split_list}')
+        data_dict = {}
+        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
+        if len(split_list) == 0:
+            logger.error(f'Got empty split list: {split_list}')
 
+        if split_as_subset:
+            for sub_name in subset_list:
+                data_dict[sub_name] = {}
+                # e.g. train: few-shot, test: target dataset to evaluate
+                for split in split_list:
+                    dataset = MsDataset.load(
+                        dataset_name=dataset_name_or_path,
+                        split=sub_name,  # load subset from split
+                        cache_dir=work_dir,
+                        hub=datasets_hub,
+                        **kwargs)
+                    data_dict[sub_name].update({split: dataset})
+        else:
             for sub_name in subset_list:
                 data_dict[sub_name] = {}
                 # e.g. train: few-shot, test: target dataset to evaluate
@@ -96,17 +123,48 @@ class DataAdapter(ABC):
                         cache_dir=work_dir,
                         hub=datasets_hub,
                         **kwargs)
-
                     data_dict[sub_name].update({split: dataset})
 
         return data_dict
 
-    def load_from_disk(self, *args, **kwargs) -> dict:
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
+        Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
+        """
+        from modelscope.msdatasets import MsDataset
+
+        logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
+                subsets: {subset_list}')
+        data_dict = {}
+        subset_list = subset_list or self.subset_list
+        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
+        for sub_name in subset_list:
+            data_dict[sub_name] = {}
+            # e.g. train: few-shot, test: target dataset to evaluate
+            for split in split_list:
+                dataset = MsDataset.load(
+                    dataset_name=dataset_name_or_path, subset_name=sub_name, split=split, cache_dir=work_dir, **kwargs)
+                data_dict[sub_name].update({split: dataset})
+        return data_dict
+
+    def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
+        """
+        Reformat the dataset subset with subset_key and format.
         """
-        return {}
+        res_dict: dict = defaultdict(lambda: defaultdict(list), {key: defaultdict(list) for key in self.subset_list})
+
+        for sub_name, sub_data_dict in data_dict.items():
+            for split in [self.train_split, self.eval_split]:
+                if split is None:
+                    continue
+                for sample_d in sub_data_dict[split]:
+                    new_subset_name = format.format(sample_d[subset_key])
+                    if new_subset_name not in self.subset_list:
+                        continue
+                    res_dict[new_subset_name][split].append(sample_d)
+        return res_dict
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -133,7 +191,7 @@ class DataAdapter(ABC):
 
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
-            if self.few_shot_num and self.few_shot_num > 0:
+            if self.train_split and self.few_shot_num and self.few_shot_num > 0:
                 few_shot_random: bool = self.config_kwargs.get('few_shot_random', True)
                 few_shot_data = self.get_fewshot_examples([item for item in sub_data_dict[self.train_split]],
                                                           self.few_shot_num,
@@ -156,7 +214,7 @@ class DataAdapter(ABC):
         else:
             return data_list[:k]
 
-    def compute_metric(self, review_res_list: list) -> List[dict]:
+    def compute_metric(self, review_res_list: Union[dict, list], **kwargs) -> List[dict]:
         """
         Compute evaluation result by specific metrics.
 
@@ -170,14 +228,15 @@ class DataAdapter(ABC):
             raise ValueError('No metric list found for the benchmark.')
 
         res_list = []
-        for metric in self.metric_list:
+        for metric_str in self.metric_list:
+            metric = metric_registry.get(metric_str)
             metric_name = metric.name
             metric_func = metric.object
-            res_list.append({
-                'metric_name': metric_name,
-                'score': metric_func(review_res_list),
-                'num': len(review_res_list)
-            })
+            if isinstance(review_res_list, dict):
+                review_res = review_res_list.get(metric_name, [])
+            else:
+                review_res = review_res_list
+            res_list.append({'metric_name': metric_name, 'score': metric_func(review_res), 'num': len(review_res)})
         return res_list
 
     def gen_report(self, subset_score_map: dict, report_name: str = None, **kwargs) -> Report:

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -0,0 +1,71 @@
+import math
+import os
+import re
+from typing import Any, Optional
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, EvalType, HubType
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='data_collection',
+    dataset_id='',  # dataset_id need to be set
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='',
+)
+class DataCollectionAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        """
+        Data adapter for collection dataset.
+        """
+        super().__init__(**kwargs)
+
+    def load(self,
+             dataset_name_or_path: str = None,
+             subset_list: list = None,
+             work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
+             datasets_hub: str = HubType.MODELSCOPE,
+             **kwargs) -> dict:
+        """
+        Load the dataset. Remote and local datasets are supported.
+        """
+        dataset_name_or_path = os.path.expanduser(dataset_name_or_path or self.dataset_id)
+        subset_list = subset_list or self.subset_list
+
+        # Try to load dataset from local disk
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset = jsonl_to_list(dataset_name_or_path)
+            if len(dataset) == 0:
+                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+        else:
+            from modelscope.msdatasets import MsDataset
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path}')
+
+            dataset = MsDataset.load(dataset_name=dataset_name_or_path, cache_dir=work_dir, hub=datasets_hub, **kwargs)
+
+            dataset = dataset[self.eval_split].to_list()
+
+        return dataset
+
+    def get_gold_answer(self, input_d: Any) -> Any:
+        return super().get_gold_answer(input_d)
+
+    def match(self, gold: Any, pred: Any) -> Any:
+        return super().match(gold, pred)
+
+    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
+        return super().parse_pred_result(result, raw_input_d, eval_type)

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -0,0 +1,125 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import csv
+import os
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics.metrics import exact_match
+from evalscope.models import MultiChoiceModelAdapter
+from evalscope.utils import ResponseParser
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='general_mcq',
+    dataset_id='general_mcq',
+    model_adapter=MultiChoiceModelAdapter,
+    subset_list=['default'],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split='dev',
+    eval_split='val',
+    prompt_template='请回答问题，并选出其中的正确答案\n{query}',
+    query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
+class GeneralMCQAdapter(DataAdapter):
+
+    choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
+        data_dict = {}
+        for subset_name in subset_list:
+            for split_name in [self.train_split, self.eval_split]:
+                if os.path.exists(dataset_name_or_path):
+                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}_{split_name}.csv')
+                else:
+                    file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}.csv')
+                if os.path.exists(file_path):
+                    with open(file_path, encoding='utf-8') as f:
+                        rows = []
+                        reader = csv.reader(f)
+                        header = next(reader)
+                        for row in reader:
+                            item = dict(zip(header, row))
+                            rows.append(item)
+
+                        if subset_name in data_dict:
+                            data_dict[subset_name].update({split_name: rows})
+                        else:
+                            data_dict[subset_name] = {split_name: rows}
+
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from raw input, unify the prompt format for C-Eval benchmark.
+
+        Args:
+            input_d (dict): The raw input. A single data format of the C-Eval:
+
+            {'id': 0,
+            'question': '下列关于税法基本原则的表述中，不正确的是____。',
+            'A': '税收法定原则包括税收要件法定原则和税务合法性原则',
+            'B': '税收公平原则源于法律上的平等性原则',
+            'C': '税收效率原则包含经济效率和行政效率两个方面',
+            'D': '税务机关按法定程序依法征税，可以自由做出减征、停征或免征税款的决定',
+            'answer': 'D'}
+
+        Returns:
+            {'data': ['prompt ...']}
+        """
+
+        few_shot_prompts = [self._format_example(input_d=sample, include_answer=True) for sample in few_shot_list]
+
+        if len(few_shot_prompts) > 0:
+            context: str = '\n'.join(few_shot_prompts) + '\n'
+        else:
+            context = ''
+        context = context.strip() + self._format_example(input_d=input_d, include_answer=False)
+
+        full_prompt = self.prompt_template.format(query=context)
+
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('answer', '')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d (dict): The raw input. Depending on the dataset.
+            eval_type: `checkpoint` or `service` or `custom`. Default is `checkpoint`.
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        if eval_type == EvalType.CHECKPOINT:
+            return result
+        elif eval_type == EvalType.SERVICE:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        elif eval_type == EvalType.CUSTOM:
+            return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        else:
+            raise ValueError(f'Invalid eval_type: {eval_type}')
+
+    def match(self, gold: str, pred: str) -> float:
+        return exact_match(gold=gold, pred=pred)
+
+    def _format_example(self, input_d: dict, include_answer=True):
+        choices_str = '\n'.join([f'{choice}. {input_d[choice]}' for choice in self.choices if choice in input_d])
+
+        if include_answer:
+            return self.query_template.format(
+                question=input_d['question'], choices=choices_str, answer=input_d['answer'])
+        else:
+            return self.query_template.format(question=input_d['question'], choices=choices_str, answer='').rstrip()

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -5,7 +5,7 @@ from collections import defaultdict
 from typing import List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import AverageBLEU, bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
+from evalscope.metrics import bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -18,10 +18,11 @@ logger = get_logger()
     dataset_id='general_qa',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=[AverageBLEU],
+    metric_list=['AverageBLEU'],
     few_shot_num=0,
     train_split=None,
     eval_split='test',
+    prompt_template='请回答问题\n{query}',
 )
 class GeneralQAAdapter(DataAdapter):
     # TODO: set few_shot_num
@@ -30,16 +31,16 @@ class GeneralQAAdapter(DataAdapter):
 
         super().__init__(**kwargs)
 
-    def load(self, dataset_name_or_path: str, subset_list: list = None, **kwargs) -> dict:
+    def load(self, **kwargs) -> dict:
 
-        data_file_list = glob.glob(os.path.join(dataset_name_or_path, '*.jsonl'))
+        data_file_list = glob.glob(os.path.join(self.dataset_id, '*.jsonl'))
         data_list = []
 
         try:
             for file_path in data_file_list:
                 data_list.extend(jsonl_to_list(file_path))
         except Exception as e:
-            raise ValueError(f'Failed to load data from {dataset_name_or_path}, got error: {e}')
+            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
         data_dict = {'default': {'test': data_list}}
 
@@ -62,11 +63,9 @@ class GeneralQAAdapter(DataAdapter):
             logger.warning('The history is not included in the prompt for GeneralQA. \
                            To be supported in the future.')
 
-        prompt = input_d.get('question', '') or input_d.get('query', '')
-
-        # if len(history) > 0:
-        #     prompt = '\n'.join(history) + '\n' + prompt
-        return {'data': [prompt], 'system_prompt': self.prompt_template}
+        query = input_d.get('question', '') or input_d.get('query', '')
+        prompt = self.prompt_template.format(query=query)
+        return {'data': [prompt], 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
         """
@@ -107,7 +106,7 @@ class GeneralQAAdapter(DataAdapter):
         res.update(bleu_dict)
         return res
 
-    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
         """
         compute weighted mean of the bleu score of all samples
 

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -3,10 +3,9 @@ import random
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys, EvalType
-from evalscope.metrics import Pass1, exact_match
+from evalscope.constants import EvalType
+from evalscope.metrics import exact_match
 from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils.utils import ResponseParser
 
 
 @Benchmark.register(
@@ -14,11 +13,11 @@ from evalscope.utils.utils import ResponseParser
     dataset_id='modelscope/gpqa',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
-    metric_list=[Pass1],
+    metric_list=['AveragePass@1'],
     few_shot_num=5,
-    train_split='train',
+    train_split=None,
     eval_split='train',  # only have train split
-    prompt_template='',
+    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )
 class GPQAAdapter(DataAdapter):
 
@@ -48,9 +47,10 @@ class GPQAAdapter(DataAdapter):
         """  # noqa: E501
         processed_input_d = self.__process_input(input_d)
         input_d['answer'] = processed_input_d['answer']  # add answer to input_d for answer extraction
-        prompt = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}Let's think step by step: "  # noqa: E501
+        query = self.prompt_prefix + f"{input_d['Question']}\n{self.__form_options(processed_input_d['choices'])}"  # noqa: E501
 
-        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
+        prompt = self.prompt_template.format(query=query)
+        return {'data': [prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
 
     def __process_input(self, input_d: dict) -> dict:
 
@@ -94,10 +94,28 @@ class GPQAAdapter(DataAdapter):
         """
         Parse the predicted result and extract proper answer.
         """
-        return ResponseParser.parse_first_option_with_choices(result, self.choices)
+        return GPQAAdapter.get_multiple_choice_answer(result)
 
     def match(self, gold: str, pred: str) -> float:
         """
         Match the gold answer and the predicted answer.
         """
         return exact_match(gold=gold, pred=pred)
+
+    @staticmethod
+    def get_multiple_choice_answer(pred: str):
+        tmp = re.findall(r'\b(A|B|C|D)\b', pred.upper())
+        if tmp:
+            pred = tmp
+        else:
+            pred = [pred.strip().strip('.')]
+
+        if len(pred) == 0:
+            pred = ''
+        else:
+            pred = pred[-1]
+
+        # Remove the period at the end, again!
+        pred = pred.rstrip('.').rstrip('/')
+
+        return pred

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -6,7 +6,6 @@ import os
 import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import AverageAccuracy
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -19,11 +18,11 @@ logger = get_logger()
     dataset_id='modelscope/gsm8k',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['main'],
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=4,
-    train_split='train',
+    train_split=None,
     eval_split='test',
-    prompt_template='',
+    prompt_template="Question: {query}\nLet's think step by step\nAnswer:",
 )
 class GSM8KAdapter(DataAdapter):
 
@@ -73,10 +72,11 @@ class GSM8KAdapter(DataAdapter):
             }
         """
         use_fewshot = self.few_shot_num > 0
+        context = self._generate_prompt(use_fewshot=use_fewshot)
 
-        full_prompt = self._generate_prompt(input_d, few_shot_list=few_shot_list, use_fewshot=use_fewshot)
+        full_prompt = context + self.prompt_template.format(query=input_d['question'])
 
-        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Extract the gold answer from the input dict.
@@ -123,7 +123,7 @@ class GSM8KAdapter(DataAdapter):
         return number_equal(gold_ans=gold, pred_ans=pred)
 
     @classmethod
-    def _generate_prompt(cls, input_d: dict, few_shot_list: list, use_fewshot: bool = True) -> str:
+    def _generate_prompt(cls, use_fewshot: bool = True) -> str:
         if use_fewshot:
             # Use 4-shot examples by system
             context = (
@@ -135,14 +135,9 @@ class GSM8KAdapter(DataAdapter):
                 "When Bella buys 2/5 times more marbles, she'll have increased the number of marbles by 2/5*60 = 24\nThe total number of marbles she'll have is 60+24 = 84\nIf Bella currently has 60 marbles, and she has two times as many marbles as frisbees, she has 60/2 = 30 frisbees.\nIf Bella buys 2/5 times more frisbees, she'll have 2/5*30 = 12 more frisbees.\nThe total number of frisbees she'll have will increase to 30+12 = 42\nBella also has 20 more frisbees than deck cards, meaning she has 30-20 = 10 deck cards\nIf she buys 2/5 times more deck cards, she'll have 2/5*10 = 4 more deck cards.\nThe total number of deck cards she'll have is 10+4 = 14\nTogether, Bella will have a total of 14+42+84 = 140 items\nThe answer is 140\n\n"
                 "Question: A group of 4 fruit baskets contains 9 apples, 15 oranges, and 14 bananas in the first three baskets and 2 less of each fruit in the fourth basket. How many fruits are there?\nLet's think step by step\n"
                 'For the first three baskets, the number of apples and oranges in one basket is 9+15=24\nIn total, together with bananas, the number of fruits in one basket is 24+14=38 for the first three baskets.\nSince there are three baskets each having 38 fruits, there are 3*38=114 fruits in the first three baskets.\nThe number of apples in the fourth basket is 9-2=7\nThere are also 15-2=13 oranges in the fourth basket\nThe combined number of oranges and apples in the fourth basket is 13+7=20\nThe fourth basket also contains 14-2=12 bananas.\nIn total, the fourth basket has 20+12=32 fruits.\nThe four baskets together have 32+114=146 fruits.\nThe answer is 146\n\n'
-                f"Question: {input_d['question']}\nLet's think step by step\nAnswer:")
-            # context = input_d['question']
-            # fewshot_prompts = ['Question: ' + item_d['question'] + '\nAnswer: ' + item_d['answer'] for item_d in few_shot_list]
-            # fewshot_prompts = fewshot_prompts + ['Question: ' + context + '\nAnswer:']
-            # context = '\n\n'.join(fewshot_prompts)
+            )
         else:
-            context = input_d['question']
-            context = 'Question: ' + context + '\nAnswer:'
+            context = ''
         return context
 
     @staticmethod

evalscope/benchmarks/hellaswag/hellaswag_adapter.py

@@ -5,7 +5,7 @@ import re
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.metrics import exact_match
 from evalscope.models import ContinuationLogitsModelAdapter
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -21,7 +21,7 @@ logger = get_logger()
     dataset_id='modelscope/hellaswag',
     model_adapter=ContinuationLogitsModelAdapter,
     subset_list=['default'],
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split='train',
     eval_split='validation',
@@ -89,11 +89,7 @@ class HellaSwagAdapter(DataAdapter):
 
         ctx_continuation_pair_list = [(context.strip(), ' ' + cont.strip()) for cont in endings]
 
-        return {
-            'data': ctx_continuation_pair_list,
-            'multi_choices': self.choices,
-            'system_prompt': self.prompt_template
-        }
+        return {'data': ctx_continuation_pair_list, 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice