evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/arc/ai2_arc.py

@@ -1,151 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright (c) Allen Institute, and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-"""AI2 ARC (Abstraction and Reasoning Corpus) for General Artificial Intelligence Benchmark."""
-"""AUTO GENERATED, DO NOT EDIT"""
-
-import datasets
-import json
-import os
-
-# flake8: noqa
-
-_CITATION = """\
-@article{allenai:arc,
-      author    = {Peter Clark  and Isaac Cowhey and Oren Etzioni and Tushar Khot and
-                    Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
-      title     = {Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
-      journal   = {arXiv:1803.05457v1},
-      year      = {2018},
-}
-"""
-
-_DESCRIPTION = """\
-A new dataset of 7,787 genuine grade-school level, multiple-choice science questions, assembled to encourage research in
- advanced question-answering. The dataset is partitioned into a Challenge Set and an Easy Set, where the former contains
- only questions answered incorrectly by both a retrieval-based algorithm and a word co-occurrence algorithm. We are also
- including a corpus of over 14 million science sentences relevant to the task,
- and an implementation of three neural baseline models for this dataset. We pose ARC as a challenge to the community.
-
-ARC-Easy:
-    train: 2251
-    test: 2376
-    validation: 570
-
-ARC-Challenge:
-    train: 1119
-    test: 1172
-    validation: 299
-"""
-
-_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/arc/ARC-V1-Feb2018.zip'
-
-# tasks: ['ARC-Easy', 'ARC-Challenge']
-
-
-class Ai2ArcConfig(datasets.BuilderConfig):
-    """BuilderConfig for Ai2ARC."""
-
-    def __init__(self, **kwargs):
-        """BuilderConfig for Ai2Arc.
-
-        Args:
-          **kwargs: keyword arguments forwarded to super.
-        """
-        super(Ai2ArcConfig, self).__init__(version=datasets.Version('1.0.0', ''), **kwargs)
-
-
-class Ai2Arc(datasets.GeneratorBasedBuilder):
-    """
-    The AI2 Reasoning Challenge (ARC) dataset.
-    Subset: ARC-Easy, ARC-Challenge.
-    """
-
-    VERSION = datasets.Version('1.0.0')
-    BUILDER_CONFIGS = [
-        Ai2ArcConfig(
-            name='ARC-Challenge',
-            description="""\
-          Challenge Set of 2590 “hard” questions (those that both a retrieval and a co-occurrence method fail to answer correctly)
-          """,
-        ),
-        Ai2ArcConfig(
-            name='ARC-Easy',
-            description="""\
-          Easy Set of 5197 questions
-          """,
-        ),
-    ]
-
-    def _info(self):
-        return datasets.DatasetInfo(
-            # This is the description that will appear on the datasets page.
-            description=_DESCRIPTION,
-            # datasets.features.FeatureConnectors
-            features=datasets.Features({
-                'id':
-                datasets.Value('string'),
-                'question':
-                datasets.Value('string'),
-                'choices':
-                datasets.features.Sequence({
-                    'text': datasets.Value('string'),
-                    'label': datasets.Value('string')
-                }),
-                'answerKey':
-                datasets.Value('string')
-                # These are the features of your dataset like images, labels ...
-            }),
-            # If there's a common (input, target) tuple from the features,
-            # specify them here. They'll be used if as_supervised=True in
-            # builder.as_dataset.
-            supervised_keys=None,
-            # Homepage of the dataset for documentation
-            homepage='https://allenai.org/data/arc',
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        # dl_manager is a datasets.download.DownloadManager that can be used to
-        # download and extract URLs
-        dl_dir = dl_manager.download_and_extract(_URL)
-        data_dir = os.path.join(dl_dir, 'ARC-V1-Feb2018-2')
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Train.jsonl')},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Test.jsonl')},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Dev.jsonl')},
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        with open(filepath, encoding='utf-8') as f:
-            for row in f:
-                data = json.loads(row)
-                answerkey = data['answerKey']
-                id_ = data['id']
-                question = data['question']['stem']
-                choices = data['question']['choices']
-                text_choices = [choice['text'] for choice in choices]
-                label_choices = [choice['label'] for choice in choices]
-                yield id_, {
-                    'id': id_,
-                    'answerKey': answerkey,
-                    'question': question,
-                    'choices': {
-                        'text': text_choices,
-                        'label': label_choices
-                    },
-                }

evalscope/benchmarks/benchmark.py

@@ -1,81 +0,0 @@
-import copy
-from collections import OrderedDict
-from dataclasses import dataclass, field, fields
-from typing import TYPE_CHECKING, Dict, List, Optional
-
-from evalscope.constants import OutputType
-
-if TYPE_CHECKING:
-    from evalscope.benchmarks import DataAdapter
-
-BENCHMARK_MAPPINGS = {}
-
-
-@dataclass
-class BenchmarkMeta:
-    name: str
-    dataset_id: str
-    data_adapter: 'DataAdapter'
-    model_adapter: Optional[str] = OutputType.GENERATION
-    output_types: Optional[List[str]] = field(default_factory=lambda: [OutputType.GENERATION])
-    subset_list: List[str] = field(default_factory=lambda: ['default'])
-    metric_list: List[str] = field(default_factory=list)
-    few_shot_num: int = 0
-    few_shot_random: bool = False
-    train_split: Optional[str] = None
-    eval_split: Optional[str] = None
-    prompt_template: Optional[str] = None
-    system_prompt: Optional[str] = None
-    query_template: Optional[str] = None
-    pretty_name: Optional[str] = None
-    description: Optional[str] = None
-    tags: Optional[List[str]] = field(default_factory=list)
-    filters: Optional[OrderedDict] = None
-    extra_params: Optional[Dict] = field(default_factory=dict)
-
-    def _update(self, args: dict):
-        if args.get('local_path'):
-            self.dataset_id = args['local_path']
-            del args['local_path']
-        self.__dict__.update(args)
-
-    def to_dict(self) -> dict:
-        return self.__dict__
-
-    def to_string_dict(self) -> dict:
-        cur_dict = copy.deepcopy(self.to_dict())
-        # cur_dict['data_adapter'] = self.data_adapter.__name__
-        del cur_dict['data_adapter']
-        return cur_dict
-
-    def get_data_adapter(self, config: dict = {}) -> 'DataAdapter':
-        if config:
-            self._update(config)
-
-        data_adapter = self.data_adapter(**self.to_dict())
-        return data_adapter
-
-
-class Benchmark:
-
-    def __init__(self):
-        pass
-
-    @classmethod
-    def get(cls, name: str) -> 'BenchmarkMeta':
-        if name not in BENCHMARK_MAPPINGS:
-            raise Exception(f'Unknown benchmark: {name}. Available tasks: {list(BENCHMARK_MAPPINGS.keys())}')
-        benchmark = BENCHMARK_MAPPINGS[name]
-        return benchmark
-
-    @classmethod
-    def register(cls, name: str, dataset_id: str, **kwargs):
-
-        def register_wrapper(data_adapter):
-            if name in BENCHMARK_MAPPINGS:
-                raise Exception(f'Benchmark {name} already registered')
-            BENCHMARK_MAPPINGS[name] = BenchmarkMeta(
-                name=name, data_adapter=data_adapter, dataset_id=dataset_id, **kwargs)
-            return data_adapter
-
-        return register_wrapper

evalscope/benchmarks/ceval/ceval_exam.py

@@ -1,146 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License
-import datasets
-import os
-import pandas as pd
-
-# flake8: noqa
-"""DO NOT EDIT unless you are contributing a new dataset."""
-
-_CITATION = """\
-@article{huang2023ceval,
-    title={C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models},
-    author={Huang, Yuzhen and Bai, Yuzhuo and Zhu, Zhihao and Zhang, Junlei and Zhang, Jinghan and Su, Tangjun and Liu, Junteng and Lv, Chuancheng and Zhang, Yikai and Lei, Jiayi and Fu, Yao and Sun, Maosong and He, Junxian},
-    journal={arXiv preprint arXiv:2305.08322},
-    year={2023}
-}
-"""
-
-_DESCRIPTION = """\
-C-Eval is a comprehensive Chinese evaluation suite for foundation models. It consists of 13948 multi-choice questions spanning 52 diverse disciplines and four difficulty levels.
-"""
-
-_HOMEPAGE = 'https://cevalbenchmark.com'
-
-_LICENSE = 'Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License'
-
-_URL = r'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/c-eval/ceval-exam.zip'
-
-task_list = [
-    'computer_network',
-    'operating_system',
-    'computer_architecture',
-    'college_programming',
-    'college_physics',
-    'college_chemistry',
-    'advanced_mathematics',
-    'probability_and_statistics',
-    'discrete_mathematics',
-    'electrical_engineer',
-    'metrology_engineer',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_chemistry',
-    'high_school_biology',
-    'middle_school_mathematics',
-    'middle_school_biology',
-    'middle_school_physics',
-    'middle_school_chemistry',
-    'veterinary_medicine',
-    'college_economics',
-    'business_administration',
-    'marxism',
-    'mao_zedong_thought',
-    'education_science',
-    'teacher_qualification',
-    'high_school_politics',
-    'high_school_geography',
-    'middle_school_politics',
-    'middle_school_geography',
-    'modern_chinese_history',
-    'ideological_and_moral_cultivation',
-    'logic',
-    'law',
-    'chinese_language_and_literature',
-    'art_studies',
-    'professional_tour_guide',
-    'legal_professional',
-    'high_school_chinese',
-    'high_school_history',
-    'middle_school_history',
-    'civil_servant',
-    'sports_science',
-    'plant_protection',
-    'basic_medicine',
-    'clinical_medicine',
-    'urban_and_rural_planner',
-    'accountant',
-    'fire_engineer',
-    'environmental_impact_assessment_engineer',
-    'tax_accountant',
-    'physician',
-]
-
-
-class CevalExamConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class CevalExam(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [CevalExamConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'id': datasets.Value('int32'),
-            'question': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-            'explanation': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'test', f'{task_name}_test.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('val'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'val', f'{task_name}_val.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('dev'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'dev', f'{task_name}_dev.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, encoding='utf-8')
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            if 'answer' not in instance.keys():
-                instance['answer'] = ''
-            if 'explanation' not in instance.keys():
-                instance['explanation'] = ''
-            yield i, instance

evalscope/benchmarks/cmmlu/cmmlu.py

@@ -1,161 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-
-import datasets
-import os
-import pandas as pd
-
-_CITATION = """\
-@misc{li2023cmmlu,
-      title={CMMLU: Measuring massive multitask language understanding in Chinese},
-      author={Haonan Li and Yixuan Zhang and Fajri Koto and Yifei Yang and Hai Zhao and Yeyun Gong and Nan Duan and Timothy Baldwin},
-      year={2023},
-      eprint={2306.09212},
-      archivePrefix={arXiv},
-      primaryClass={cs.CL}
-}
-"""
-
-_DESCRIPTION = """\
-CMMLU is a comprehensive Chinese assessment suite specifically designed to evaluate the advanced knowledge and reasoning abilities of LLMs within the Chinese language and cultural context.
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/cmmlu/summary'
-
-# _URL = r"https://huggingface.co/datasets/haonan-li/cmmlu/resolve/main/cmmlu_v1_0_1.zip"
-_URL = r'https://modelscope.cn/api/v1/datasets/modelscope/cmmlu/repo?Revision=master&FilePath=cmmlu_v1_0_1.zip'
-
-# contains 67 sub-tasks
-task_list = [
-    'agronomy',
-    'anatomy',
-    'ancient_chinese',
-    'arts',
-    'astronomy',
-    'business_ethics',
-    'chinese_civil_service_exam',
-    'chinese_driving_rule',
-    'chinese_food_culture',
-    'chinese_foreign_policy',
-    'chinese_history',
-    'chinese_literature',
-    'chinese_teacher_qualification',
-    'clinical_knowledge',
-    'college_actuarial_science',
-    'college_education',
-    'college_engineering_hydrology',
-    'college_law',
-    'college_mathematics',
-    'college_medical_statistics',
-    'college_medicine',
-    'computer_science',
-    'computer_security',
-    'conceptual_physics',
-    'construction_project_management',
-    'economics',
-    'education',
-    'electrical_engineering',
-    'elementary_chinese',
-    'elementary_commonsense',
-    'elementary_information_and_technology',
-    'elementary_mathematics',
-    'ethnology',
-    'food_science',
-    'genetics',
-    'global_facts',
-    'high_school_biology',
-    'high_school_chemistry',
-    'high_school_geography',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_politics',
-    'human_sexuality',
-    'international_law',
-    'journalism',
-    'jurisprudence',
-    'legal_and_moral_basis',
-    'logical',
-    'machine_learning',
-    'management',
-    'marketing',
-    'marxist_theory',
-    'modern_chinese',
-    'nutrition',
-    'philosophy',
-    'professional_accounting',
-    'professional_law',
-    'professional_medicine',
-    'professional_psychology',
-    'public_relations',
-    'security_study',
-    'sociology',
-    'sports_science',
-    'traditional_chinese_medicine',
-    'virology',
-    'world_history',
-    'world_religions',
-]
-
-
-class CMMLUConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.1'), **kwargs)
-        # V1.0.1 Fix: One comma missing in word_religions.csv
-        # V1.0.0 Init version
-
-
-class CMMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [CMMLUConfig(name=task_name) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'Question': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'Answer': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'test/{task_name}.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('dev'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'dev/{task_name}.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, header=0, index_col=0, encoding='utf-8')
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/cmmlu/samples.jsonl

@@ -1,5 +0,0 @@
-{'input': '毛毛骑在牛背上过河，他共有甲、乙、丙、丁4头牛，甲过河要20分钟，乙过河要30分钟，丙过河要40分钟，丁过河要50分钟。毛毛每次只能赶2头牛过河，要把4头牛都赶到对岸去，最少要多少分钟?', 'A': '190', 'B': '180', 'C': '170', 'D': '160', 'target': 'D'}
-{'input': '下列关于重力的说法正确的是', 'A': '在地球周围的物体都要受到重力作用，与其运动状态无关', 'B': '对某一物体而言，重力的大小是一个恒量，不随物体的地理位置而改变', 'C': '重力就是地球对物体的吸引力，重力的方向总是竖直向下', 'D': '在地球表面各处的重力方向都是相同的', 'target': 'A'}
-{'input': '心脏的静脉血回心的主要途径是', 'A': '心小静脉', 'B': '冠状窦', 'C': '心中静脉', 'D': '心前静脉', 'target': 'B'}
-{'input': "以西蒙为代表的决策理论学派提出的决策准则是", 'A': '最优化', 'B': '公平', 'C': '民主化', 'D': '满意', 'target': 'D'}
-{'input': '20世纪初，英国首相阿斯奎斯说：“我们现在有一个牢固确立了两百年的传统，即归根到底，王位的占有者接受其大臣的建议并据此行事。”这一传统的确立，使一个以小农业和手工业生产为主的国家变成了一个典型的资本主义国家，成为欧洲各国效仿的对象。各国效仿的理由是', 'A': '英国“光荣革命”宣告了欧洲新社会政治制度的诞生', 'B': '殖民主义深刻影响了英国“世界工厂”的地位', 'C': '英国经济上的成就得益于其制度设计', 'D': '英国启蒙思想奠定了资产阶级民主主义政治的理论基础', 'target': 'C'}

evalscope/benchmarks/competition_math/competition_math.py

@@ -1,79 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-"""Mathematics Aptitude Test of Heuristics (MATH) dataset."""
-
-import datasets
-import json
-import os
-
-_CITATION = """\
-@article{hendrycksmath2021,
-  title={Measuring Mathematical Problem Solving With the MATH Dataset},
-  author={Dan Hendrycks
-    and Collin Burns
-    and Saurav Kadavath
-    and Akul Arora
-    and Steven Basart
-    and Eric Tang
-    and Dawn Song
-    and Jacob Steinhardt},
-  journal={arXiv preprint arXiv:2103.03874},
-  year={2021}
-}
-"""
-
-_DESCRIPTION = """\
-The Mathematics Aptitude Test of Heuristics (MATH) dataset consists of problems
-from mathematics competitions, including the AMC 10, AMC 12, AIME, and more.
-Each problem in MATH has a full step-by-step solution, which can be used to teach
-models to generate answer derivations and explanations.
-"""
-
-_HOMEPAGE = 'https://github.com/hendrycks/math'
-
-_LICENSE = 'https://github.com/hendrycks/math/blob/main/LICENSE'
-
-# Original data URL: "https://people.eecs.berkeley.edu/~hendrycks/MATH.tar"
-_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/math/MATH.zip'
-
-
-class CompetitionMathDataset(datasets.GeneratorBasedBuilder):
-    """Mathematics Aptitude Test of Heuristics (MATH) dataset."""
-
-    VERSION = datasets.Version('1.0.0')
-
-    def _info(self):
-        features = datasets.Features({
-            'problem': datasets.Value('string'),
-            'level': datasets.Value('string'),
-            'type': datasets.Value('string'),
-            'solution': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            supervised_keys=None,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        download_dir = dl_manager.download_and_extract(_URL)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={'data_dir': dl_manager.iter_files(os.path.join(download_dir, 'MATH', 'train'))},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={'data_dir': dl_manager.iter_files(os.path.join(download_dir, 'MATH', 'test'))},
-            ),
-        ]
-
-    def _generate_examples(self, data_dir):
-        """Yields examples as (key, example) tuples."""
-        for id_, filepath in enumerate(data_dir):
-            with open(filepath, 'rb') as fin:
-                example = json.load(fin)
-                yield id_, example