evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/mmlu/mmlu.py

@@ -1,160 +0,0 @@
-# isort: skip_file
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-
-import datasets
-import os
-import pandas as pd
-"""The MMLU dataset on ModelScope hub. READ ONLY, DO NOT MODIFY."""
-
-_CITATION = """\
-@article{hendryckstest2021,
-  title={Measuring Massive Multitask Language Understanding},
-  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
-  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
-  year={2021}
-}
-"""
-
-_DESCRIPTION = """\
-Measuring Massive Multitask Language Understanding by Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas
-Mazeika, Dawn Song, and Jacob Steinhardt (ICLR 2021).
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/mmlu/summary'
-
-_LICENSE = 'MIT'
-
-# _URL = "https://people.eecs.berkeley.edu/~hendrycks/data.tar"
-_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/mmlu/repo?Revision=master&FilePath=data.tar'
-
-task_list = [
-    'high_school_european_history',
-    'business_ethics',
-    'clinical_knowledge',
-    'medical_genetics',
-    'high_school_us_history',
-    'high_school_physics',
-    'high_school_world_history',
-    'virology',
-    'high_school_microeconomics',
-    'econometrics',
-    'college_computer_science',
-    'high_school_biology',
-    'abstract_algebra',
-    'professional_accounting',
-    'philosophy',
-    'professional_medicine',
-    'nutrition',
-    'global_facts',
-    'machine_learning',
-    'security_studies',
-    'public_relations',
-    'professional_psychology',
-    'prehistory',
-    'anatomy',
-    'human_sexuality',
-    'college_medicine',
-    'high_school_government_and_politics',
-    'college_chemistry',
-    'logical_fallacies',
-    'high_school_geography',
-    'elementary_mathematics',
-    'human_aging',
-    'college_mathematics',
-    'high_school_psychology',
-    'formal_logic',
-    'high_school_statistics',
-    'international_law',
-    'high_school_mathematics',
-    'high_school_computer_science',
-    'conceptual_physics',
-    'miscellaneous',
-    'high_school_chemistry',
-    'marketing',
-    'professional_law',
-    'management',
-    'college_physics',
-    'jurisprudence',
-    'world_religions',
-    'sociology',
-    'us_foreign_policy',
-    'high_school_macroeconomics',
-    'computer_security',
-    'moral_scenarios',
-    'moral_disputes',
-    'electrical_engineering',
-    'astronomy',
-    'college_biology',
-]
-
-
-class MMLUConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class MMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [MMLUConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'input': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'target': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'test', f'{task_name}_test.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'val', f'{task_name}_val.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'dev', f'{task_name}_dev.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath)
-        df.columns = ['input', 'A', 'B', 'C', 'D', 'target']
-
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/mmlu/samples.jsonl

@@ -1,5 +0,0 @@
-{'input': 'A "dished face" profile is often associated with', 'A': 'a protruding mandible due to reactivation of the condylar cartilage by acromegaly.', 'B': 'a recessive maxilla due to failure of elongation of the cranial base.', 'C': 'an enlarged frontal bone due to hydrocephaly.', 'D': 'defective development of the maxillary air sinus.', 'target': 'B'}
-{'input': '___________ is based on the idea that customer expectations of the service they will receive shape their perception of the actual service encounter.', 'A': 'Service quality.', 'B': 'Service action.', 'C': 'Service recovery.', 'D': 'Service satisfaction.', 'target': 'A'}
-{'input': ' Information collected for the first time specifically for a marketing research study is called:', 'A': 'Secondary research.', 'B': 'Primary research.', 'C': 'Soft research.', 'D': 'Experimental research.', 'target': 'B'}
-{'input': "This includes advertisements that contain 'call-to-response' mechanisms such as telephone numbers, website addresses, email and postal addresses:", 'A': 'Direct response advertising.', 'B': 'Sales promotions.', 'C': 'Mass media advertising.', 'D': 'Public relations.', 'target': 'A'}
-{'input': 'Which of the following is not part of the external marketing environment?', 'A': 'Political.', 'B': 'Legal.', 'C': 'Product.', 'D': 'Socio-cultural.', 'target': 'C'}

evalscope/benchmarks/race/race.py

@@ -1,104 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import datasets
-import os
-import pandas as pd
-
-_CITATION = """\
-@inproceedings{lai-etal-2017-race,
-    title = "{RACE}: Large-scale {R}e{A}ding Comprehension Dataset From Examinations",
-    author = "Lai, Guokun  and
-      Xie, Qizhe  and
-      Liu, Hanxiao  and
-      Yang, Yiming  and
-      Hovy, Eduard",
-    booktitle = "Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing",
-    month = sep,
-    year = "2017",
-    address = "Copenhagen, Denmark",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/D17-1082",
-    doi = "10.18653/v1/D17-1082",
-    pages = "785--794",
-}
-"""
-
-_DESCRIPTION = """\
-RACE is a large-scale reading comprehension dataset with more than 28,000 passages and nearly 100,000 questions.
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/race/summary'
-
-_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/race/repo?Revision=master&FilePath=race.zip'
-
-task_list = [
-    'high',
-    'middle',
-]
-
-
-class RACEConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class RACE(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [RACEConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'example_id': datasets.Value('string'),
-            'article': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-            'question': datasets.Value('string'),
-            'options': [datasets.Value('string')],
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/test/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/val/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/train/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_parquet(filepath)
-        df.columns = ['example_id', 'article', 'answer', 'question', 'options']
-
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/race/samples.jsonl

@@ -1,5 +0,0 @@
-{'example_id': 'middle4227.txt', 'article': 'There are many kinds...ealthy.\n,.', 'answer': 'D', 'question': 'We may read this pas... in   _  .', 'options': ['a letter', 'a story', 'a newspaper', 'a health magazine']}
-{'example_id': 'middle3329.txt', 'article': 'Do you know why diff...ng at all.', 'answer': 'B', 'question': 'Those pests with dif...of danger.', 'options': ['change their colours', 'hide in the day time...r at night', 'move quietly', 'hide at night and ap...e day time']}
-{'example_id': 'middle3614.txt', 'article': 'The seahorse is a ve...o the sea.', 'answer': 'B', 'question': 'A seahorse eats   _  .', 'options': ['sea weed', 'small fish', 'water', 'nothing']}
-{'example_id': 'middle6632.txt', 'article': 'Kids have unbelievab...h at her."', 'answer': 'D', 'question': 'Which is NOT mention...e passage?', 'options': ['Robots keep secrets.', 'Robots give suggestions.', 'Robots do chores.', 'Robots make movies.']}
-{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}

evalscope/benchmarks/trivia_qa/trivia_qa.py

@@ -1,89 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import datasets
-import json
-import os
-import pandas as pd
-
-_CITATION = """\
-@article{2017arXivtriviaqa,
-       author = {{Joshi}, Mandar and {Choi}, Eunsol and {Weld},
-                 Daniel and {Zettlemoyer}, Luke},
-        title = "{triviaqa: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension}",
-      journal = {arXiv e-prints},
-         year = 2017,
-          eid = {arXiv:1705.03551},
-        pages = {arXiv:1705.03551},
-archivePrefix = {arXiv},
-       eprint = {1705.03551},
-}
-"""
-
-_DESCRIPTION = """\
-TriviaqQA is a reading comprehension dataset containing over 650K question-answer-evidence triples.
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/trivia_qa/summary'
-
-_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/trivia_qa/repo?Revision=master&FilePath=trivia_qa.zip'
-
-task_list = ['default']
-
-
-class TriviaQAConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class TriviaQA(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [TriviaQAConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'input': [{
-                'role': datasets.features.Value('string'),
-                'content': datasets.features.Value('string'),
-            }],
-            'ideal': [datasets.Value('string')],
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'trivia_qa/test.jsonl'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('dev'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'trivia_qa/dev.jsonl'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        with open(filepath, encoding='utf-8') as f:
-            contents = [json.loads(line) for line in f.readlines()]
-            for i, instance in enumerate(contents):
-                yield i, instance

evalscope/benchmarks/truthful_qa/truthful_qa.py

@@ -1,163 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TruthfulQA dataset."""
-# flake8: noqa
-
-import csv
-import datasets
-import json
-
-_CITATION = """\
-@misc{lin2021truthfulqa,
-    title={TruthfulQA: Measuring How Models Mimic Human Falsehoods},
-    author={Stephanie Lin and Jacob Hilton and Owain Evans},
-    year={2021},
-    eprint={2109.07958},
-    archivePrefix={arXiv},
-    primaryClass={cs.CL}
-}
-"""
-
-_DESCRIPTION = """\
-TruthfulQA is a benchmark to measure whether a language model is truthful in
-generating answers to questions. The benchmark comprises 817 questions that
-span 38 categories, including health, law, finance and politics. Questions are
-crafted so that some humans would answer falsely due to a false belief or
-misconception. To perform well, models must avoid generating false answers
-learned from imitating human texts.
-"""
-
-_HOMEPAGE = 'https://github.com/sylinrl/TruthfulQA'
-
-_LICENSE = 'Apache License 2.0'
-
-
-class TruthfulQaConfig(datasets.BuilderConfig):
-    """BuilderConfig for TruthfulQA."""
-
-    def __init__(self, url, features, **kwargs):
-        """BuilderConfig for TruthfulQA.
-        Args:
-          url: *string*, the url to the configuration's data.
-          features: *list[string]*, list of features that'll appear in the feature dict.
-          **kwargs: keyword arguments forwarded to super.
-        """
-        super().__init__(version=datasets.Version('1.1.0'), **kwargs)
-        self.url = url
-        self.features = features
-
-
-class TruthfulQa(datasets.GeneratorBasedBuilder):
-    """TruthfulQA is a benchmark to measure whether a language model is truthful in generating answers to questions."""
-
-    BUILDER_CONFIGS = [
-        TruthfulQaConfig(
-            name='generation',
-            # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/TruthfulQA.csv",
-            url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/TruthfulQA.csv',
-            features=datasets.Features({
-                'type': datasets.Value('string'),
-                'category': datasets.Value('string'),
-                'question': datasets.Value('string'),
-                'best_answer': datasets.Value('string'),
-                'correct_answers': datasets.features.Sequence(datasets.Value('string')),
-                'incorrect_answers': datasets.features.Sequence(datasets.Value('string')),
-                'source': datasets.Value('string'),
-            }),
-            description=
-            "The Generation TruthfulQA (main) task tests a model's ability to generate 1-2 sentence answers for a given question truthfully.",
-        ),
-        TruthfulQaConfig(
-            name='multiple_choice',
-            # url="https://raw.githubusercontent.com/sylinrl/TruthfulQA/013686a06be7a7bde5bf8223943e106c7250123c/data/mc_task.json",
-            url='https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/truthful_qa/mc_task.json',
-            features=datasets.Features({
-                'question': datasets.Value('string'),
-                'mc1_targets': {
-                    'choices': datasets.features.Sequence(datasets.Value('string')),
-                    'labels': datasets.features.Sequence(datasets.Value('int32')),
-                },
-                'mc2_targets': {
-                    'choices': datasets.features.Sequence(datasets.Value('string')),
-                    'labels': datasets.features.Sequence(datasets.Value('int32')),
-                },
-            }),
-            description=
-            "The Multiple-Choice TruthfulQA task provides a multiple-choice option to test a model's ability to identify true statements.",
-        ),
-    ]
-
-    def _info(self):
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=self.config.features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download(self.config.url)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    'filepath': data_dir,
-                },
-            ),
-        ]
-
-    def _split_csv_list(self, csv_list: str, delimiter: str = ';') -> str:
-        """
-        Splits a csv list field, delimited by `delimiter` (';'), into a list
-        of strings.
-        """
-        csv_list = csv_list.strip().split(delimiter)
-        return [item.strip() for item in csv_list]
-
-    def _generate_examples(self, filepath):
-        if self.config.name == 'multiple_choice':
-            # Multiple choice data is in a `JSON` file.
-            with open(filepath, encoding='utf-8') as f:
-                contents = json.load(f)
-                for key, row in enumerate(contents):
-                    yield key, {
-                        'question': row['question'],
-                        'mc1_targets': {
-                            'choices': list(row['mc1_targets'].keys()),
-                            'labels': list(row['mc1_targets'].values()),
-                        },
-                        'mc2_targets': {
-                            'choices': list(row['mc2_targets'].keys()),
-                            'labels': list(row['mc2_targets'].values()),
-                        },
-                    }
-        else:
-            # Generation data is in a `CSV` file.
-            with open(filepath, newline='', encoding='utf-8-sig') as f:
-                contents = csv.DictReader(f)
-                for key, row in enumerate(contents):
-                    # Ensure that references exist.
-                    if not row['Correct Answers'] or not row['Incorrect Answers']:
-                        continue
-                    yield key, {
-                        'type': row['Type'],
-                        'category': row['Category'],
-                        'question': row['Question'],
-                        'best_answer': row['Best Answer'],
-                        'correct_answers': self._split_csv_list(row['Correct Answers']),
-                        'incorrect_answers': self._split_csv_list(row['Incorrect Answers']),
-                        'source': row['Source'],
-                    }