evalscope 0.7.2__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/benchmarks/mmlu/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.mmlu.mmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST, MMLUAdapter
+from evalscope.benchmarks.mmlu.mmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
+from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter
 from evalscope.benchmarks.mmlu.mmlu_adapter import MMLUAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/mmlu/mmlu.py

@@ -1,3 +1,4 @@
+# isort: skip_file
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
 #
@@ -14,14 +15,11 @@
 # limitations under the License.
 # flake8: noqa
 
-import os
-
 import datasets
+import os
 import pandas as pd
-
 """The MMLU dataset on ModelScope hub. READ ONLY, DO NOT MODIFY."""
 
-
 _CITATION = """\
 @article{hendryckstest2021,
   title={Measuring Massive Multitask Language Understanding},
@@ -105,29 +103,23 @@ task_list = [
 
 
 class MMLUConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
         super().__init__(version=datasets.Version('1.0.0'), **kwargs)
 
 
 class MMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        MMLUConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [MMLUConfig(name=task_name, ) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                'input': datasets.Value('string'),
-                'A': datasets.Value('string'),
-                'B': datasets.Value('string'),
-                'C': datasets.Value('string'),
-                'D': datasets.Value('string'),
-                'target': datasets.Value('string'),
-            }
-        )
+        features = datasets.Features({
+            'input': datasets.Value('string'),
+            'A': datasets.Value('string'),
+            'B': datasets.Value('string'),
+            'C': datasets.Value('string'),
+            'D': datasets.Value('string'),
+            'target': datasets.Value('string'),
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -143,25 +135,19 @@ class MMLU(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir, 'data', 'test', f'{task_name}_test.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'data', 'test', f'{task_name}_test.csv'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split.VALIDATION,
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir, 'data', 'val', f'{task_name}_val.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'data', 'val', f'{task_name}_val.csv'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split.TRAIN,
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir, 'data', 'dev', f'{task_name}_dev.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'data', 'dev', f'{task_name}_dev.csv'),
                 },
             ),
         ]

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -4,8 +4,9 @@ import os
 
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score, ResponseParser
+from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
@@ -72,65 +73,65 @@ SUBSET_LIST = [
     'college_biology',
 ]
 
-
-SUBJECT_MAPPING = {'abstract_algebra': ['Abstract Algebra', 'math', 'STEM'],
-                   'anatomy': ['Anatomy', 'health', 'Other'],
-                   'astronomy': ['Astronomy', 'physics', 'STEM'],
-                   'business_ethics': ['Business Ethics', 'business', 'Other'],
-                   'clinical_knowledge': ['Clinical Knowledge', 'health', 'Other'],
-                   'college_biology': ['College Biology', 'biology', 'STEM'],
-                   'college_chemistry': ['College Chemistry', 'chemistry', 'STEM'],
-                   'college_computer_science': ['College Computer Science', 'computer science', 'STEM'],
-                   'college_mathematics': ['College Mathematics', 'math', 'STEM'],
-                   'college_medicine': ['College Medicine', 'health', 'Other'],
-                   'college_physics': ['College Physics', 'physics', 'STEM'],
-                   'computer_security': ['Computer Security', 'computer science', 'STEM'],
-                   'conceptual_physics': ['Conceptual Physics', 'physics', 'STEM'],
-                   'econometrics': ['Econometrics', 'economics', 'Social Science'],
-                   'electrical_engineering': ['Electrical Engineering', 'engineering', 'STEM'],
-                   'elementary_mathematics': ['Elementary Mathematics', 'math', 'STEM'],
-                   'formal_logic': ['Formal Logic', 'philosophy', 'Humanities'],
-                   'global_facts': ['Global Facts', 'other', 'Other'],
-                   'high_school_biology': ['High School Biology', 'biology', 'STEM'],
-                   'high_school_chemistry': ['High School Chemistry', 'chemistry', 'STEM'],
-                   'high_school_computer_science': ['High School Computer Science', 'computer science', 'STEM'],
-                   'high_school_european_history': ['High School European History', 'history', 'Humanities'],
-                   'high_school_geography': ['High School Geography', 'geography', 'Social Science'],
-                   'high_school_government_and_politics': ['High School Government And Politics', 'politics', 'Social Science'],
-                   'high_school_macroeconomics': ['High School Macroeconomics', 'economics', 'Social Science'],
-                   'high_school_mathematics': ['High School Mathematics', 'math', 'STEM'],
-                   'high_school_microeconomics': ['High School Microeconomics', 'economics', 'Social Science'],
-                   'high_school_physics': ['High School Physics', 'physics', 'STEM'],
-                   'high_school_psychology': ['High School Psychology', 'psychology', 'Social Science'],
-                   'high_school_statistics': ['High School Statistics', 'math', 'STEM'],
-                   'high_school_us_history': ['High School Us History', 'history', 'Humanities'],
-                   'high_school_world_history': ['High School World History', 'history', 'Humanities'],
-                   'human_aging': ['Human Aging', 'health', 'Other'],
-                   'human_sexuality': ['Human Sexuality', 'culture', 'Social Science'],
-                   'international_law': ['International Law', 'law', 'Humanities'],
-                   'jurisprudence': ['Jurisprudence', 'law', 'Humanities'],
-                   'logical_fallacies': ['Logical Fallacies', 'philosophy', 'Humanities'],
-                   'machine_learning': ['Machine Learning', 'computer science', 'STEM'],
-                   'management': ['Management', 'business', 'Other'],
-                   'marketing': ['Marketing', 'business', 'Other'],
-                   'medical_genetics': ['Medical Genetics', 'health', 'Other'],
-                   'miscellaneous': ['Miscellaneous', 'other', 'Other'],
-                   'moral_disputes': ['Moral Disputes', 'philosophy', 'Humanities'],
-                   'moral_scenarios': ['Moral Scenarios', 'philosophy', 'Humanities'],
-                   'nutrition': ['Nutrition', 'health', 'Other'],
-                   'philosophy': ['Philosophy', 'philosophy', 'Humanities'],
-                   'prehistory': ['Prehistory', 'history', 'Humanities'],
-                   'professional_accounting': ['Professional Accounting', 'other', 'Other'],
-                   'professional_law': ['Professional Law', 'law', 'Humanities'],
-                   'professional_medicine': ['Professional Medicine', 'health', 'Other'],
-                   'professional_psychology': ['Professional Psychology', 'psychology', 'Social Science'],
-                   'public_relations': ['Public Relations', 'politics', 'Social Science'],
-                   'security_studies': ['Security Studies', 'politics', 'Social Science'],
-                   'sociology': ['Sociology', 'culture', 'Social Science'],
-                   'us_foreign_policy': ['Us Foreign Policy', 'politics', 'Social Science'],
-                   'virology': ['Virology', 'health', 'Other'],
-                   'world_religions': ['World Religions', 'philosophy', 'Humanities'],
-                   }
+SUBJECT_MAPPING = {
+    'abstract_algebra': ['Abstract Algebra', 'math', 'STEM'],
+    'anatomy': ['Anatomy', 'health', 'Other'],
+    'astronomy': ['Astronomy', 'physics', 'STEM'],
+    'business_ethics': ['Business Ethics', 'business', 'Other'],
+    'clinical_knowledge': ['Clinical Knowledge', 'health', 'Other'],
+    'college_biology': ['College Biology', 'biology', 'STEM'],
+    'college_chemistry': ['College Chemistry', 'chemistry', 'STEM'],
+    'college_computer_science': ['College Computer Science', 'computer science', 'STEM'],
+    'college_mathematics': ['College Mathematics', 'math', 'STEM'],
+    'college_medicine': ['College Medicine', 'health', 'Other'],
+    'college_physics': ['College Physics', 'physics', 'STEM'],
+    'computer_security': ['Computer Security', 'computer science', 'STEM'],
+    'conceptual_physics': ['Conceptual Physics', 'physics', 'STEM'],
+    'econometrics': ['Econometrics', 'economics', 'Social Science'],
+    'electrical_engineering': ['Electrical Engineering', 'engineering', 'STEM'],
+    'elementary_mathematics': ['Elementary Mathematics', 'math', 'STEM'],
+    'formal_logic': ['Formal Logic', 'philosophy', 'Humanities'],
+    'global_facts': ['Global Facts', 'other', 'Other'],
+    'high_school_biology': ['High School Biology', 'biology', 'STEM'],
+    'high_school_chemistry': ['High School Chemistry', 'chemistry', 'STEM'],
+    'high_school_computer_science': ['High School Computer Science', 'computer science', 'STEM'],
+    'high_school_european_history': ['High School European History', 'history', 'Humanities'],
+    'high_school_geography': ['High School Geography', 'geography', 'Social Science'],
+    'high_school_government_and_politics': ['High School Government And Politics', 'politics', 'Social Science'],
+    'high_school_macroeconomics': ['High School Macroeconomics', 'economics', 'Social Science'],
+    'high_school_mathematics': ['High School Mathematics', 'math', 'STEM'],
+    'high_school_microeconomics': ['High School Microeconomics', 'economics', 'Social Science'],
+    'high_school_physics': ['High School Physics', 'physics', 'STEM'],
+    'high_school_psychology': ['High School Psychology', 'psychology', 'Social Science'],
+    'high_school_statistics': ['High School Statistics', 'math', 'STEM'],
+    'high_school_us_history': ['High School Us History', 'history', 'Humanities'],
+    'high_school_world_history': ['High School World History', 'history', 'Humanities'],
+    'human_aging': ['Human Aging', 'health', 'Other'],
+    'human_sexuality': ['Human Sexuality', 'culture', 'Social Science'],
+    'international_law': ['International Law', 'law', 'Humanities'],
+    'jurisprudence': ['Jurisprudence', 'law', 'Humanities'],
+    'logical_fallacies': ['Logical Fallacies', 'philosophy', 'Humanities'],
+    'machine_learning': ['Machine Learning', 'computer science', 'STEM'],
+    'management': ['Management', 'business', 'Other'],
+    'marketing': ['Marketing', 'business', 'Other'],
+    'medical_genetics': ['Medical Genetics', 'health', 'Other'],
+    'miscellaneous': ['Miscellaneous', 'other', 'Other'],
+    'moral_disputes': ['Moral Disputes', 'philosophy', 'Humanities'],
+    'moral_scenarios': ['Moral Scenarios', 'philosophy', 'Humanities'],
+    'nutrition': ['Nutrition', 'health', 'Other'],
+    'philosophy': ['Philosophy', 'philosophy', 'Humanities'],
+    'prehistory': ['Prehistory', 'history', 'Humanities'],
+    'professional_accounting': ['Professional Accounting', 'other', 'Other'],
+    'professional_law': ['Professional Law', 'law', 'Humanities'],
+    'professional_medicine': ['Professional Medicine', 'health', 'Other'],
+    'professional_psychology': ['Professional Psychology', 'psychology', 'Social Science'],
+    'public_relations': ['Public Relations', 'politics', 'Social Science'],
+    'security_studies': ['Security Studies', 'politics', 'Social Science'],
+    'sociology': ['Sociology', 'culture', 'Social Science'],
+    'us_foreign_policy': ['Us Foreign Policy', 'politics', 'Social Science'],
+    'virology': ['Virology', 'health', 'Other'],
+    'world_religions': ['World Religions', 'philosophy', 'Humanities'],
+}
 
 
 class MMLUAdapter(DataAdapter):
@@ -160,12 +161,13 @@ class MMLUAdapter(DataAdapter):
             logger.warning(f'few_shot_num <= 5 for MMLU, but got {few_shot_num}. Use 5-shot by default.')
             few_shot_num = 5
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -227,8 +229,7 @@ class MMLUAdapter(DataAdapter):
 
         """
         prompt = 'The following are multiple choice questions (with answers) about {}.\n\n'.format(
-            self._format_subject(subset_name)
-        )
+            self._format_subject(subset_name))
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
@@ -335,19 +336,26 @@ class MMLUAdapter(DataAdapter):
             domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
                                       sum([num for _, _, num in domain_res_list])
             domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({'name': domain_name,
-                                  'score': domain_weighted_avg_acc,
-                                  'subset': [{'name': subset_name, 'score': normalize_score(score=subset_score)}
-                                             for subset_name, subset_score, _ in domain_res_list]})
+            category_list.append({
+                'name':
+                domain_name,
+                'score':
+                domain_weighted_avg_acc,
+                'subset': [{
+                    'name': subset_name,
+                    'score': normalize_score(score=subset_score)
+                } for subset_name, subset_score, _ in domain_res_list]
+            })
 
         category_list = sorted(category_list, key=lambda x: x['name'])
 
         # Get final dict of report
-        res_map = dict(name=report_name or 'mmlu',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=category_list,
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'mmlu',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=category_list,
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/race/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.race.race_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST, RACEAdapter
+from evalscope.benchmarks.race.race_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
+from evalscope.benchmarks.race.race_adapter import RACEAdapter
 from evalscope.benchmarks.race.race_adapter import RACEAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/race/race.py

@@ -11,12 +11,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-
 import datasets
+import os
 import pandas as pd
 
-
 _CITATION = """\
 @inproceedings{lai-etal-2017-race,
     title = "{RACE}: Large-scale {R}e{A}ding Comprehension Dataset From Examinations",
@@ -40,39 +38,33 @@ _DESCRIPTION = """\
 RACE is a large-scale reading comprehension dataset with more than 28,000 passages and nearly 100,000 questions.
 """
 
-_HOMEPAGE = "https://modelscope.cn/datasets/modelscope/race/summary"
+_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/race/summary'
 
-_URL = "https://modelscope.cn/api/v1/datasets/modelscope/race/repo?Revision=master&FilePath=race.zip"
+_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/race/repo?Revision=master&FilePath=race.zip'
 
 task_list = [
-    "high",
-    "middle",
+    'high',
+    'middle',
 ]
 
 
 class RACEConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.0"), **kwargs)
+        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
 
 
 class RACE(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        RACEConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [RACEConfig(name=task_name, ) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                "example_id": datasets.Value("string"),
-                "article": datasets.Value("string"),
-                "answer": datasets.Value("string"),
-                "question": datasets.Value("string"),
-                "options": [datasets.Value("string")],
-            }
-        )
+        features = datasets.Features({
+            'example_id': datasets.Value('string'),
+            'article': datasets.Value('string'),
+            'answer': datasets.Value('string'),
+            'question': datasets.Value('string'),
+            'options': [datasets.Value('string')],
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -87,32 +79,26 @@ class RACE(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"race/test/{task_name}-00000-of-00001.parquet"
-                    ),
+                    'filepath': os.path.join(data_dir, f'race/test/{task_name}-00000-of-00001.parquet'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split.VALIDATION,
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"race/val/{task_name}-00000-of-00001.parquet"
-                    ),
+                    'filepath': os.path.join(data_dir, f'race/val/{task_name}-00000-of-00001.parquet'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split.TRAIN,
                 gen_kwargs={
-                    "filepath": os.path.join(
-                        data_dir, f"race/train/{task_name}-00000-of-00001.parquet"
-                    ),
+                    'filepath': os.path.join(data_dir, f'race/train/{task_name}-00000-of-00001.parquet'),
                 },
             ),
         ]
 
     def _generate_examples(self, filepath):
         df = pd.read_parquet(filepath)
-        df.columns = ["example_id", "article", "answer", "question", "options"]
+        df.columns = ['example_id', 'article', 'answer', 'question', 'options']
 
-        for i, instance in enumerate(df.to_dict(orient="records")):
-            yield i, instance
+        for i, instance in enumerate(df.to_dict(orient='records')):
+            yield i, instance

evalscope/benchmarks/race/race_adapter.py

@@ -1,26 +1,22 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
 import json
+import os
+
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score, jsonl_to_list
+from evalscope.utils import jsonl_to_list, normalize_score
 from evalscope.utils.logger import get_logger
+
 # flake8: noqa
 
 logger = get_logger()
 
 DATASET_ID = 'modelscope/race'
 
-SUBSET_LIST = [
-    "high",
-    "middle"
-]
-
+SUBSET_LIST = ['high', 'middle']
 
-SUBJECT_MAPPING = {"high": "High",
-                   "middle": "Middle"
-}
+SUBJECT_MAPPING = {'high': 'High', 'middle': 'Middle'}
 
 
 class RACEAdapter(DataAdapter):
@@ -49,12 +45,13 @@ class RACEAdapter(DataAdapter):
             logger.warning(f'few_shot_num <= 3 for RACE, but got {few_shot_num}. Use 3-shot by default.')
             few_shot_num = 3
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -92,8 +89,7 @@ class RACEAdapter(DataAdapter):
 
         """
         prompt = 'The following are multiple choice reading comprehension questions (with answers).\n\n'.format(
-            self._format_subject(subset_name)
-        )
+            self._format_subject(subset_name))
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
@@ -122,9 +118,9 @@ class RACEAdapter(DataAdapter):
         """
         if eval_type == 'checkpoint':
             return result
-        elif eval_type == 'service':        # TODO: to be implemented
+        elif eval_type == 'service':  # TODO: to be implemented
             return result
-        elif eval_type == 'custom':         # TODO: to be implemented
+        elif eval_type == 'custom':  # TODO: to be implemented
             return result
         else:
             raise ValueError(f'Unknown eval_type: {eval_type}')
@@ -191,17 +187,24 @@ class RACEAdapter(DataAdapter):
             domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
                                      sum([num for _, _, num in domain_res_list])
             domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({'name': domain_name,
-                                  'score': normalize_score(score=domain_weighted_avg_acc),
-                                  'subset': [{'name': subset_name, 'score': subset_score}
-                                             for subset_name, subset_score, _ in domain_res_list]})
+            category_list.append({
+                'name':
+                domain_name,
+                'score':
+                normalize_score(score=domain_weighted_avg_acc),
+                'subset': [{
+                    'name': subset_name,
+                    'score': subset_score
+                } for subset_name, subset_score, _ in domain_res_list]
+            })
 
         # Get final dict of report
-        res_map = dict(name=report_name or 'race',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=category_list,
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'race',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=category_list,
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/race/samples.jsonl

@@ -2,4 +2,4 @@
 {'example_id': 'middle3329.txt', 'article': 'Do you know why diff...ng at all.', 'answer': 'B', 'question': 'Those pests with dif...of danger.', 'options': ['change their colours', 'hide in the day time...r at night', 'move quietly', 'hide at night and ap...e day time']}
 {'example_id': 'middle3614.txt', 'article': 'The seahorse is a ve...o the sea.', 'answer': 'B', 'question': 'A seahorse eats   _  .', 'options': ['sea weed', 'small fish', 'water', 'nothing']}
 {'example_id': 'middle6632.txt', 'article': 'Kids have unbelievab...h at her."', 'answer': 'D', 'question': 'Which is NOT mention...e passage?', 'options': ['Robots keep secrets.', 'Robots give suggestions.', 'Robots do chores.', 'Robots make movies.']}
-{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}
+{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}

evalscope/benchmarks/trivia_qa/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter, DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import DATASET_ID, SUBSET_LIST
+from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter
 from evalscope.benchmarks.trivia_qa.trivia_qa_adapter import TriviaQaAdapter as DataAdapterClass
-from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import ChatGenerationModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/trivia_qa/samples.jsonl

@@ -2,4 +2,4 @@
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Which Lloyd Webber musical premiered in the US on 10th December 1993?"}], "ideal": ["Sunset Blvd", "West Sunset Boulevard", "Sunset Boulevard", "Sunset Bulevard", "Sunset Blvd.", "sunset boulevard", "sunset bulevard", "west sunset boulevard", "sunset blvd"]}
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Who was the next British Prime Minister after Arthur Balfour?"}], "ideal": ["Sir Henry Campbell-Bannerman", "Campbell-Bannerman", "Campbell Bannerman", "Sir Henry Campbell Bannerman", "Henry Campbell Bannerman", "Henry Campbell-Bannerman", "henry campbell bannerman", "sir henry campbell bannerman", "campbell bannerman"]}
 {"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "Who had a 70s No 1 hit with Kiss You All Over?"}], "ideal": ["Internal exile", "Exiles", "Transported for life", "Exile (politics and government)", "Voluntary exile", "Sent into exile", "Exile and Banishment", "Self-exile", "Forced exile", "Exile", "Exile in Greek tragedy", "Banish", "Banishment", "exiles", "voluntary exile", "forced exile", "banish", "self exile", "exile politics and government", "exile in greek tragedy", "sent into exile", "banishment", "transported for life", "exile", "internal exile", "exile and banishment"]}
-{"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "What claimed the life of singer Kathleen Ferrier?"}], "ideal": ["Cancer pathology", "Deaths by cancer", "Anti-cancer", "Cancer (disease)", "Cancerophobia", "Malignant lesion", "Cancer medication", "Malignant tumors", "Cancer signs", "Malignant neoplasm", "Invasive (cancer)", "Malignant Neoplasms", "Malignant growth", "Sporadic cancer", "Malignant cancer", "Tumour virus", "Cancer en cuirasse", "Microtumor", "Malignant neoplasms", "Malignant tumour", "Carcinophobia", "Malignacy", "Cancer patient", "Epithelial cancers", "Solid cancer", "Cancers", "Tumor medication", "Malignant neoplastic disease", "AIDS-related cancer", "Invasive cancer", "Cancer therapy", "Cancerous tumor", "Cancer", "Financial toxicity", "Cancer diagnosis", "Cancer (medicine)", "Malignant tumor", "Cancerous", "Borderline (cancer)", "Signs of cancer", "Malignancies", "Cancer aromatase", "aids related cancer", "sporadic cancer", "cancer disease", "malignant tumors", "cancers", "carcinophobia", "cancer", "cancer diagnosis", "malignant neoplastic disease", "malignant neoplasm", "tumour virus", "cancer medicine", "deaths by cancer", "malignant tumour", "epithelial cancers", "solid cancer", "cancerous", "borderline cancer", "invasive cancer", "anti cancer", "cancer pathology", "cancer signs", "cancer aromatase", "cancer therapy", "financial toxicity", "cancerophobia", "cancer en cuirasse", "cancer patient", "cancerous tumor", "malignant cancer", "malignant neoplasms", "tumor medication", "signs of cancer", "malignacy", "malignant tumor", "cancer medication", "microtumor", "malignancies", "malignant lesion", "malignant growth"]}
+{"input": [{"role": "system", "content": "Follow the given examples and answer the question."}, {"role": "user", "content": "What claimed the life of singer Kathleen Ferrier?"}], "ideal": ["Cancer pathology", "Deaths by cancer", "Anti-cancer", "Cancer (disease)", "Cancerophobia", "Malignant lesion", "Cancer medication", "Malignant tumors", "Cancer signs", "Malignant neoplasm", "Invasive (cancer)", "Malignant Neoplasms", "Malignant growth", "Sporadic cancer", "Malignant cancer", "Tumour virus", "Cancer en cuirasse", "Microtumor", "Malignant neoplasms", "Malignant tumour", "Carcinophobia", "Malignacy", "Cancer patient", "Epithelial cancers", "Solid cancer", "Cancers", "Tumor medication", "Malignant neoplastic disease", "AIDS-related cancer", "Invasive cancer", "Cancer therapy", "Cancerous tumor", "Cancer", "Financial toxicity", "Cancer diagnosis", "Cancer (medicine)", "Malignant tumor", "Cancerous", "Borderline (cancer)", "Signs of cancer", "Malignancies", "Cancer aromatase", "aids related cancer", "sporadic cancer", "cancer disease", "malignant tumors", "cancers", "carcinophobia", "cancer", "cancer diagnosis", "malignant neoplastic disease", "malignant neoplasm", "tumour virus", "cancer medicine", "deaths by cancer", "malignant tumour", "epithelial cancers", "solid cancer", "cancerous", "borderline cancer", "invasive cancer", "anti cancer", "cancer pathology", "cancer signs", "cancer aromatase", "cancer therapy", "financial toxicity", "cancerophobia", "cancer en cuirasse", "cancer patient", "cancerous tumor", "malignant cancer", "malignant neoplasms", "tumor medication", "signs of cancer", "malignacy", "malignant tumor", "cancer medication", "microtumor", "malignancies", "malignant lesion", "malignant growth"]}