evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/cmmlu/cmmlu.py

@@ -1,161 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-
-import datasets
-import os
-import pandas as pd
-
-_CITATION = """\
-@misc{li2023cmmlu,
-      title={CMMLU: Measuring massive multitask language understanding in Chinese},
-      author={Haonan Li and Yixuan Zhang and Fajri Koto and Yifei Yang and Hai Zhao and Yeyun Gong and Nan Duan and Timothy Baldwin},
-      year={2023},
-      eprint={2306.09212},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-"""
-
-_DESCRIPTION = """\
-CMMLU is a comprehensive Chinese assessment suite specifically designed to evaluate the advanced knowledge and reasoning abilities of LLMs within the Chinese language and cultural context.
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/cmmlu/summary'
-
-# _URL = r"https://huggingface.co/datasets/haonan-li/cmmlu/resolve/main/cmmlu_v1_0_1.zip"
-_URL = r'https://modelscope.cn/api/v1/datasets/modelscope/cmmlu/repo?Revision=master&FilePath=cmmlu_v1_0_1.zip'
-
-# contains 67 sub-tasks
-task_list = [
-    'agronomy',
-    'anatomy',
-    'ancient_chinese',
-    'arts',
-    'astronomy',
-    'business_ethics',
-    'chinese_civil_service_exam',
-    'chinese_driving_rule',
-    'chinese_food_culture',
-    'chinese_foreign_policy',
-    'chinese_history',
-    'chinese_literature',
-    'chinese_teacher_qualification',
-    'clinical_knowledge',
-    'college_actuarial_science',
-    'college_education',
-    'college_engineering_hydrology',
-    'college_law',
-    'college_mathematics',
-    'college_medical_statistics',
-    'college_medicine',
-    'computer_science',
-    'computer_security',
-    'conceptual_physics',
-    'construction_project_management',
-    'economics',
-    'education',
-    'electrical_engineering',
-    'elementary_chinese',
-    'elementary_commonsense',
-    'elementary_information_and_technology',
-    'elementary_mathematics',
-    'ethnology',
-    'food_science',
-    'genetics',
-    'global_facts',
-    'high_school_biology',
-    'high_school_chemistry',
-    'high_school_geography',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_politics',
-    'human_sexuality',
-    'international_law',
-    'journalism',
-    'jurisprudence',
-    'legal_and_moral_basis',
-    'logical',
-    'machine_learning',
-    'management',
-    'marketing',
-    'marxist_theory',
-    'modern_chinese',
-    'nutrition',
-    'philosophy',
-    'professional_accounting',
-    'professional_law',
-    'professional_medicine',
-    'professional_psychology',
-    'public_relations',
-    'security_study',
-    'sociology',
-    'sports_science',
-    'traditional_chinese_medicine',
-    'virology',
-    'world_history',
-    'world_religions',
-]
-
-
-class CMMLUConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.1'), **kwargs)
-        # V1.0.1 Fix: One comma missing in word_religions.csv
-        # V1.0.0 Init version
-
-
-class CMMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [CMMLUConfig(name=task_name) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'Question': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'Answer': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'test/{task_name}.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('dev'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'dev/{task_name}.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, header=0, index_col=0, encoding='utf-8')
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/cmmlu/samples.jsonl

@@ -1,5 +0,0 @@
-{'input': '毛毛骑在牛背上过河，他共有甲、乙、丙、丁4头牛，甲过河要20分钟，乙过河要30分钟，丙过河要40分钟，丁过河要50分钟。毛毛每次只能赶2头牛过河，要把4头牛都赶到对岸去，最少要多少分钟?', 'A': '190', 'B': '180', 'C': '170', 'D': '160', 'target': 'D'}
-{'input': '下列关于重力的说法正确的是', 'A': '在地球周围的物体都要受到重力作用，与其运动状态无关', 'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变', 'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下', 'D': '在地球表面各处的重力方向都是相同的', 'target': 'A'}
-{'input': '心脏的静脉血回心的主要途径是', 'A': '心小静脉', 'B': '冠状窦', 'C': '心中静脉', 'D': '心前静脉', 'target': 'B'}
-{'input': "以西蒙为代表的决策理论学派提出的决策准则是", 'A': '最优化', 'B': '公平', 'C': '民主化', 'D': '满意', 'target': 'D'}
-{'input': '20世纪初，英国首相阿斯奎斯说：“我们现在有一个牢固确立了两百年的传统，即归根到底，王位的占有者接受其大臣的建议并据此行事。”这一传统的确立，使一个以小农业和手工业生产为主的国家变成了一个典型的资本主义国家，成为欧洲各国效仿的对象。各国效仿的理由是', 'A': '英国“光荣革命”宣告了欧洲新社会政治制度的诞生', 'B': '殖民主义深刻影响了英国“世界工厂”的地位', 'C': '英国经济上的成就得益于其制度设计', 'D': '英国启蒙思想奠定了资产阶级民主主义政治的理论基础', 'target': 'C'}

evalscope/benchmarks/competition_math/competition_math.py

@@ -1,79 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-"""Mathematics Aptitude Test of Heuristics (MATH) dataset."""
-
-import datasets
-import json
-import os
-
-_CITATION = """\
-@article{hendrycksmath2021,
-  title={Measuring Mathematical Problem Solving With the MATH Dataset},
-  author={Dan Hendrycks
-    and Collin Burns
-    and Saurav Kadavath
-    and Akul Arora
-    and Steven Basart
-    and Eric Tang
-    and Dawn Song
-    and Jacob Steinhardt},
-  journal={arXiv preprint arXiv:2103.03874},
-  year={2021}
-}
-"""
-
-_DESCRIPTION = """\
-The Mathematics Aptitude Test of Heuristics (MATH) dataset consists of problems
-from mathematics competitions, including the AMC 10, AMC 12, AIME, and more.
-Each problem in MATH has a full step-by-step solution, which can be used to teach
-models to generate answer derivations and explanations.
-"""
-
-_HOMEPAGE = 'https://github.com/hendrycks/math'
-
-_LICENSE = 'https://github.com/hendrycks/math/blob/main/LICENSE'
-
-# Original data URL: "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar"
-_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/math/MATH.zip'
-
-
-class CompetitionMathDataset(datasets.GeneratorBasedBuilder):
-    """Mathematics Aptitude Test of Heuristics (MATH) dataset."""
-
-    VERSION = datasets.Version('1.0.0')
-
-    def _info(self):
-        features = datasets.Features({
-            'problem': datasets.Value('string'),
-            'level': datasets.Value('string'),
-            'type': datasets.Value('string'),
-            'solution': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            supervised_keys=None,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        download_dir = dl_manager.download_and_extract(_URL)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={'data_dir': dl_manager.iter_files(os.path.join(download_dir, 'MATH', 'train'))},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={'data_dir': dl_manager.iter_files(os.path.join(download_dir, 'MATH', 'test'))},
-            ),
-        ]
-
-    def _generate_examples(self, data_dir):
-        """Yields examples as (key, example) tuples."""
-        for id_, filepath in enumerate(data_dir):
-            with open(filepath, 'rb') as fin:
-                example = json.load(fin)
-                yield id_, example