evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,9 +1,10 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import csv
 import os
+
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.metrics.metrics import exact_match, weighted_mean
-from evalscope.utils import normalize_score, ResponseParser
+from evalscope.utils import ResponseParser, normalize_score
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -67,58 +68,60 @@ SUBSET_LIST = [
     'physician',
 ]
 
-SUBJECT_MAPPING = {'computer_network': ['Computer Network', '计算机网络', 'STEM'],
-                   'operating_system': ['Operating System', '操作系统', 'STEM'],
-                   'computer_architecture': ['Computer Architecture', '计算机组成', 'STEM'],
-                   'college_programming': ['College Programming', '大学编程', 'STEM'],
-                   'college_physics': ['College Physics', '大学物理', 'STEM'],
-                   'college_chemistry': ['College Chemistry', '大学化学', 'STEM'],
-                   'advanced_mathematics': ['Advanced Mathematics', '高等数学', 'STEM'],
-                   'probability_and_statistics': ['Probability and Statistics', '概率统计', 'STEM'],
-                   'discrete_mathematics': ['Discrete Mathematics', '离散数学', 'STEM'],
-                   'electrical_engineer': ['Electrical Engineer', '注册电气工程师', 'STEM'],
-                   'metrology_engineer': ['Metrology Engineer', '注册计量师', 'STEM'],
-                   'high_school_mathematics': ['High School Mathematics', '高中数学', 'STEM'],
-                   'high_school_physics': ['High School Physics', '高中物理', 'STEM'],
-                   'high_school_chemistry': ['High School Chemistry', '高中化学', 'STEM'],
-                   'high_school_biology': ['High School Biology', '高中生物', 'STEM'],
-                   'middle_school_mathematics': ['Middle School Mathematics', '初中数学', 'STEM'],
-                   'middle_school_biology': ['Middle School Biology', '初中生物', 'STEM'],
-                   'middle_school_physics': ['Middle School Physics', '初中物理', 'STEM'],
-                   'middle_school_chemistry': ['Middle School Chemistry', '初中化学', 'STEM'],
-                   'veterinary_medicine': ['Veterinary Medicine', '兽医学', 'STEM'],
-                   'college_economics': ['College Economics', '大学经济学', 'Social Science'],
-                   'business_administration': ['Business Administration', '工商管理', 'Social Science'],
-                   'marxism': ['Marxism', '马克思主义基本原理', 'Social Science'],
-                   'mao_zedong_thought': ['Mao Zedong Thought', '毛泽东思想和中国特色社会主义理论体系概论', 'Social Science'],
-                   'education_science': ['Education Science', '教育学', 'Social Science'],
-                   'teacher_qualification': ['Teacher Qualification', '教师资格', 'Social Science'],
-                   'high_school_politics': ['High School Politics', '高中政治', 'Social Science'],
-                   'high_school_geography': ['High School Geography', '高中地理', 'Social Science'],
-                   'middle_school_politics': ['Middle School Politics', '初中政治', 'Social Science'],
-                   'middle_school_geography': ['Middle School Geography', '初中地理', 'Social Science'],
-                   'modern_chinese_history': ['Modern Chinese History', '近代史纲要', 'Humanities'],
-                   'ideological_and_moral_cultivation': ['Ideological and Moral Cultivation', '思想道德修养与法律基础', 'Humanities'],
-                   'logic': ['Logic', '逻辑学', 'Humanities'],
-                   'law': ['Law', '法学', 'Humanities'],
-                   'chinese_language_and_literature': ['Chinese Language and Literature', '中国语言文学', 'Humanities'],
-                   'art_studies': ['Art Studies', '艺术学', 'Humanities'],
-                   'professional_tour_guide': ['Professional Tour Guide', '导游资格', 'Humanities'],
-                   'legal_professional': ['Legal Professional', '法律职业资格', 'Humanities'],
-                   'high_school_chinese': ['High School Chinese', '高中语文', 'Humanities'],
-                   'high_school_history': ['High School History', '高中历史', 'Humanities'],
-                   'middle_school_history': ['Middle School History', '初中历史', 'Humanities'],
-                   'civil_servant': ['Civil Servant', '公务员', 'Other'],
-                   'sports_science': ['Sports Science', '体育学', 'Other'],
-                   'plant_protection': ['Plant Protection', '植物保护', 'Other'],
-                   'basic_medicine': ['Basic Medicine', '基础医学', 'Other'],
-                   'clinical_medicine': ['Clinical Medicine', '临床医学', 'Other'],
-                   'urban_and_rural_planner': ['Urban and Rural Planner', '注册城乡规划师', 'Other'],
-                   'accountant': ['Accountant', '注册会计师', 'Other'],
-                   'fire_engineer': ['Fire Engineer', '注册消防工程师', 'Other'],
-                   'environmental_impact_assessment_engineer': ['Environmental Impact Assessment Engineer', '环境影响评价工程师', 'Other'],
-                   'tax_accountant': ['Tax Accountant', '税务师', 'Other'],
-                   'physician': ['Physician', '医师资格', 'Other']}
+SUBJECT_MAPPING = {
+    'computer_network': ['Computer Network', '计算机网络', 'STEM'],
+    'operating_system': ['Operating System', '操作系统', 'STEM'],
+    'computer_architecture': ['Computer Architecture', '计算机组成', 'STEM'],
+    'college_programming': ['College Programming', '大学编程', 'STEM'],
+    'college_physics': ['College Physics', '大学物理', 'STEM'],
+    'college_chemistry': ['College Chemistry', '大学化学', 'STEM'],
+    'advanced_mathematics': ['Advanced Mathematics', '高等数学', 'STEM'],
+    'probability_and_statistics': ['Probability and Statistics', '概率统计', 'STEM'],
+    'discrete_mathematics': ['Discrete Mathematics', '离散数学', 'STEM'],
+    'electrical_engineer': ['Electrical Engineer', '注册电气工程师', 'STEM'],
+    'metrology_engineer': ['Metrology Engineer', '注册计量师', 'STEM'],
+    'high_school_mathematics': ['High School Mathematics', '高中数学', 'STEM'],
+    'high_school_physics': ['High School Physics', '高中物理', 'STEM'],
+    'high_school_chemistry': ['High School Chemistry', '高中化学', 'STEM'],
+    'high_school_biology': ['High School Biology', '高中生物', 'STEM'],
+    'middle_school_mathematics': ['Middle School Mathematics', '初中数学', 'STEM'],
+    'middle_school_biology': ['Middle School Biology', '初中生物', 'STEM'],
+    'middle_school_physics': ['Middle School Physics', '初中物理', 'STEM'],
+    'middle_school_chemistry': ['Middle School Chemistry', '初中化学', 'STEM'],
+    'veterinary_medicine': ['Veterinary Medicine', '兽医学', 'STEM'],
+    'college_economics': ['College Economics', '大学经济学', 'Social Science'],
+    'business_administration': ['Business Administration', '工商管理', 'Social Science'],
+    'marxism': ['Marxism', '马克思主义基本原理', 'Social Science'],
+    'mao_zedong_thought': ['Mao Zedong Thought', '毛泽东思想和中国特色社会主义理论体系概论', 'Social Science'],
+    'education_science': ['Education Science', '教育学', 'Social Science'],
+    'teacher_qualification': ['Teacher Qualification', '教师资格', 'Social Science'],
+    'high_school_politics': ['High School Politics', '高中政治', 'Social Science'],
+    'high_school_geography': ['High School Geography', '高中地理', 'Social Science'],
+    'middle_school_politics': ['Middle School Politics', '初中政治', 'Social Science'],
+    'middle_school_geography': ['Middle School Geography', '初中地理', 'Social Science'],
+    'modern_chinese_history': ['Modern Chinese History', '近代史纲要', 'Humanities'],
+    'ideological_and_moral_cultivation': ['Ideological and Moral Cultivation', '思想道德修养与法律基础', 'Humanities'],
+    'logic': ['Logic', '逻辑学', 'Humanities'],
+    'law': ['Law', '法学', 'Humanities'],
+    'chinese_language_and_literature': ['Chinese Language and Literature', '中国语言文学', 'Humanities'],
+    'art_studies': ['Art Studies', '艺术学', 'Humanities'],
+    'professional_tour_guide': ['Professional Tour Guide', '导游资格', 'Humanities'],
+    'legal_professional': ['Legal Professional', '法律职业资格', 'Humanities'],
+    'high_school_chinese': ['High School Chinese', '高中语文', 'Humanities'],
+    'high_school_history': ['High School History', '高中历史', 'Humanities'],
+    'middle_school_history': ['Middle School History', '初中历史', 'Humanities'],
+    'civil_servant': ['Civil Servant', '公务员', 'Other'],
+    'sports_science': ['Sports Science', '体育学', 'Other'],
+    'plant_protection': ['Plant Protection', '植物保护', 'Other'],
+    'basic_medicine': ['Basic Medicine', '基础医学', 'Other'],
+    'clinical_medicine': ['Clinical Medicine', '临床医学', 'Other'],
+    'urban_and_rural_planner': ['Urban and Rural Planner', '注册城乡规划师', 'Other'],
+    'accountant': ['Accountant', '注册会计师', 'Other'],
+    'fire_engineer': ['Fire Engineer', '注册消防工程师', 'Other'],
+    'environmental_impact_assessment_engineer': ['Environmental Impact Assessment Engineer', '环境影响评价工程师', 'Other'],
+    'tax_accountant': ['Tax Accountant', '税务师', 'Other'],
+    'physician': ['Physician', '医师资格', 'Other']
+}
 
 
 class CEVALAdapter(DataAdapter):
@@ -148,12 +151,13 @@ class CEVALAdapter(DataAdapter):
             logger.warning(f'few_shot_num <= 5 for C-Eval, but got {few_shot_num}. Use 5-shot by default.')
             few_shot_num = 5
 
-        super().__init__(subset_list=subset_list,
-                         metric_list=metric_list,
-                         few_shot_num=few_shot_num,
-                         train_split=train_split,
-                         eval_split=eval_split,
-                         **kwargs)
+        super().__init__(
+            subset_list=subset_list,
+            metric_list=metric_list,
+            few_shot_num=few_shot_num,
+            train_split=train_split,
+            eval_split=eval_split,
+            **kwargs)
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = {}
@@ -211,7 +215,7 @@ class CEVALAdapter(DataAdapter):
         full_prompt: str = context.strip() + self._format_example(input_d=input_d, include_answer=False)
 
         subject_name: str = SUBJECT_MAPPING.get(subset_name)[1] if SUBJECT_MAPPING.get(subset_name) else subset_name
-        full_prompt = f"以下是中国关于{subject_name}考试的单项选择题，请选出其中的正确答案。\n" + full_prompt
+        full_prompt = f'以下是中国关于{subject_name}考试的单项选择题，请选出其中的正确答案。\n' + full_prompt
 
         return {'data': [full_prompt], 'multi_choices': self.choices}
 
@@ -311,19 +315,26 @@ class CEVALAdapter(DataAdapter):
             domain_weighted_avg_acc = sum([score * num for _, score, num in domain_res_list]) / \
                                       sum([num for _, _, num in domain_res_list])
             domain_weighted_avg_acc = normalize_score(score=domain_weighted_avg_acc)
-            category_list.append({'name': domain_name,
-                                  'score': domain_weighted_avg_acc,
-                                  'subset': [{'name': subset_name, 'score': normalize_score(score=subset_score)}
-                                             for subset_name, subset_score, _ in domain_res_list]})
+            category_list.append({
+                'name':
+                domain_name,
+                'score':
+                domain_weighted_avg_acc,
+                'subset': [{
+                    'name': subset_name,
+                    'score': normalize_score(score=subset_score)
+                } for subset_name, subset_score, _ in domain_res_list]
+            })
 
         category_list = sorted(category_list, key=lambda x: x['name'])
 
         # Get final dict of report
-        res_map = dict(name=report_name or 'ceval',
-                       metric=self.metric_list[0]['name'],
-                       score=weighted_avg_acc,
-                       category=category_list,
-                       total_num=total_num)
+        res_map = dict(
+            name=report_name or 'ceval',
+            metric=self.metric_list[0]['name'],
+            score=weighted_avg_acc,
+            category=category_list,
+            total_num=total_num)
 
         return res_map
 

evalscope/benchmarks/ceval/ceval_exam.py

@@ -2,12 +2,11 @@
 # Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
 #
 # Licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License
-import os
-
 import datasets
+import os
 import pandas as pd
-# flake8: noqa
 
+# flake8: noqa
 """DO NOT EDIT unless you are contributing a new dataset."""
 
 _CITATION = """\
@@ -29,7 +28,6 @@ _LICENSE = 'Creative Commons Attribution-NonCommercial-ShareAlike 4.0 Internatio
 
 _URL = r'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/c-eval/ceval-exam.zip'
 
-
 task_list = [
     'computer_network',
     'operating_system',
@@ -85,32 +83,27 @@ task_list = [
     'physician',
 ]
 
+
 class CevalExamConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
         super().__init__(version=datasets.Version('1.0.0'), **kwargs)
 
 
 class CevalExam(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        CevalExamConfig(
-            name=task_name,
-        )
-        for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [CevalExamConfig(name=task_name, ) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                'id': datasets.Value('int32'),
-                'question': datasets.Value('string'),
-                'A': datasets.Value('string'),
-                'B': datasets.Value('string'),
-                'C': datasets.Value('string'),
-                'D': datasets.Value('string'),
-                'answer': datasets.Value('string'),
-                'explanation':datasets.Value('string'),
-            }
-        )
+        features = datasets.Features({
+            'id': datasets.Value('int32'),
+            'question': datasets.Value('string'),
+            'A': datasets.Value('string'),
+            'B': datasets.Value('string'),
+            'C': datasets.Value('string'),
+            'D': datasets.Value('string'),
+            'answer': datasets.Value('string'),
+            'explanation': datasets.Value('string'),
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -126,25 +119,19 @@ class CevalExam(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir, 'test', f'{task_name}_test.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'test', f'{task_name}_test.csv'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split('val'),
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir,  'val', f'{task_name}_val.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'val', f'{task_name}_val.csv'),
                 },
             ),
             datasets.SplitGenerator(
                 name=datasets.Split('dev'),
                 gen_kwargs={
-                    'filepath': os.path.join(
-                        data_dir, 'dev', f'{task_name}_dev.csv'
-                    ),
+                    'filepath': os.path.join(data_dir, 'dev', f'{task_name}_dev.csv'),
                 },
             ),
         ]

evalscope/benchmarks/cmmlu/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.benchmarks.cmmlu.cmmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST, CMMLUAdapter
+from evalscope.benchmarks.cmmlu.cmmlu_adapter import DATASET_ID, SUBJECT_MAPPING, SUBSET_LIST
+from evalscope.benchmarks.cmmlu.cmmlu_adapter import CMMLUAdapter
 from evalscope.benchmarks.cmmlu.cmmlu_adapter import CMMLUAdapter as DataAdapterClass
-from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass    # noqa
+from evalscope.models.model_adapter import MultiChoiceModelAdapter as ModelAdapterClass  # noqa

evalscope/benchmarks/cmmlu/cmmlu.py

@@ -14,11 +14,10 @@
 # limitations under the License.
 # flake8: noqa
 
-import os
 import datasets
+import os
 import pandas as pd
 
-
 _CITATION = """\
 @misc{li2023cmmlu,
       title={CMMLU: Measuring massive multitask language understanding in Chinese},
@@ -34,107 +33,103 @@ _DESCRIPTION = """\
 CMMLU is a comprehensive Chinese assessment suite specifically designed to evaluate the advanced knowledge and reasoning abilities of LLMs within the Chinese language and cultural context.
 """
 
-_HOMEPAGE = "https://modelscope.cn/datasets/modelscope/cmmlu/summary"
-
+_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/cmmlu/summary'
 
 # _URL = r"https://huggingface.co/datasets/haonan-li/cmmlu/resolve/main/cmmlu_v1_0_1.zip"
-_URL = r"https://modelscope.cn/api/v1/datasets/modelscope/cmmlu/repo?Revision=master&FilePath=cmmlu_v1_0_1.zip"
+_URL = r'https://modelscope.cn/api/v1/datasets/modelscope/cmmlu/repo?Revision=master&FilePath=cmmlu_v1_0_1.zip'
 
 # contains 67 sub-tasks
 task_list = [
-     'agronomy',
-     'anatomy',
-     'ancient_chinese',
-     'arts',
-     'astronomy',
-     'business_ethics',
-     'chinese_civil_service_exam',
-     'chinese_driving_rule',
-     'chinese_food_culture',
-     'chinese_foreign_policy',
-     'chinese_history',
-     'chinese_literature',
-     'chinese_teacher_qualification',
-     'clinical_knowledge',
-     'college_actuarial_science',
-     'college_education',
-     'college_engineering_hydrology',
-     'college_law',
-     'college_mathematics',
-     'college_medical_statistics',
-     'college_medicine',
-     'computer_science',
-     'computer_security',
-     'conceptual_physics',
-     'construction_project_management',
-     'economics',
-     'education',
-     'electrical_engineering',
-     'elementary_chinese',
-     'elementary_commonsense',
-     'elementary_information_and_technology',
-     'elementary_mathematics',
-     'ethnology',
-     'food_science',
-     'genetics',
-     'global_facts',
-     'high_school_biology',
-     'high_school_chemistry',
-     'high_school_geography',
-     'high_school_mathematics',
-     'high_school_physics',
-     'high_school_politics',
-     'human_sexuality',
-     'international_law',
-     'journalism',
-     'jurisprudence',
-     'legal_and_moral_basis',
-     'logical',
-     'machine_learning',
-     'management',
-     'marketing',
-     'marxist_theory',
-     'modern_chinese',
-     'nutrition',
-     'philosophy',
-     'professional_accounting',
-     'professional_law',
-     'professional_medicine',
-     'professional_psychology',
-     'public_relations',
-     'security_study',
-     'sociology',
-     'sports_science',
-     'traditional_chinese_medicine',
-     'virology',
-     'world_history',
-     'world_religions',
+    'agronomy',
+    'anatomy',
+    'ancient_chinese',
+    'arts',
+    'astronomy',
+    'business_ethics',
+    'chinese_civil_service_exam',
+    'chinese_driving_rule',
+    'chinese_food_culture',
+    'chinese_foreign_policy',
+    'chinese_history',
+    'chinese_literature',
+    'chinese_teacher_qualification',
+    'clinical_knowledge',
+    'college_actuarial_science',
+    'college_education',
+    'college_engineering_hydrology',
+    'college_law',
+    'college_mathematics',
+    'college_medical_statistics',
+    'college_medicine',
+    'computer_science',
+    'computer_security',
+    'conceptual_physics',
+    'construction_project_management',
+    'economics',
+    'education',
+    'electrical_engineering',
+    'elementary_chinese',
+    'elementary_commonsense',
+    'elementary_information_and_technology',
+    'elementary_mathematics',
+    'ethnology',
+    'food_science',
+    'genetics',
+    'global_facts',
+    'high_school_biology',
+    'high_school_chemistry',
+    'high_school_geography',
+    'high_school_mathematics',
+    'high_school_physics',
+    'high_school_politics',
+    'human_sexuality',
+    'international_law',
+    'journalism',
+    'jurisprudence',
+    'legal_and_moral_basis',
+    'logical',
+    'machine_learning',
+    'management',
+    'marketing',
+    'marxist_theory',
+    'modern_chinese',
+    'nutrition',
+    'philosophy',
+    'professional_accounting',
+    'professional_law',
+    'professional_medicine',
+    'professional_psychology',
+    'public_relations',
+    'security_study',
+    'sociology',
+    'sports_science',
+    'traditional_chinese_medicine',
+    'virology',
+    'world_history',
+    'world_religions',
 ]
 
 
 class CMMLUConfig(datasets.BuilderConfig):
+
     def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version("1.0.1"), **kwargs)
+        super().__init__(version=datasets.Version('1.0.1'), **kwargs)
         # V1.0.1 Fix: One comma missing in word_religions.csv
         # V1.0.0 Init version
 
 
 class CMMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [
-        CMMLUConfig(name=task_name) for task_name in task_list
-    ]
+    BUILDER_CONFIGS = [CMMLUConfig(name=task_name) for task_name in task_list]
 
     def _info(self):
-        features = datasets.Features(
-            {
-                "Question": datasets.Value("string"),
-                "A": datasets.Value("string"),
-                "B": datasets.Value("string"),
-                "C": datasets.Value("string"),
-                "D": datasets.Value("string"),
-                "Answer": datasets.Value("string"),
-            }
-        )
+        features = datasets.Features({
+            'Question': datasets.Value('string'),
+            'A': datasets.Value('string'),
+            'B': datasets.Value('string'),
+            'C': datasets.Value('string'),
+            'D': datasets.Value('string'),
+            'Answer': datasets.Value('string'),
+        })
         return datasets.DatasetInfo(
             description=_DESCRIPTION,
             features=features,
@@ -149,18 +144,18 @@ class CMMLU(datasets.GeneratorBasedBuilder):
             datasets.SplitGenerator(
                 name=datasets.Split.TEST,
                 gen_kwargs={
-                    "filepath": os.path.join(data_dir, f"test/{task_name}.csv"),
+                    'filepath': os.path.join(data_dir, f'test/{task_name}.csv'),
                 },
             ),
             datasets.SplitGenerator(
-                name=datasets.Split("dev"),
+                name=datasets.Split('dev'),
                 gen_kwargs={
-                    "filepath": os.path.join(data_dir, f"dev/{task_name}.csv"),
+                    'filepath': os.path.join(data_dir, f'dev/{task_name}.csv'),
                 },
             ),
         ]
 
     def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, header=0, index_col=0, encoding="utf-8")
-        for i, instance in enumerate(df.to_dict(orient="records")):
-            yield i, instance
+        df = pd.read_csv(filepath, header=0, index_col=0, encoding='utf-8')
+        for i, instance in enumerate(df.to_dict(orient='records')):
+            yield i, instance