evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/gsm8k/gsm8k.py

@@ -1,121 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-"""Grade School Math 8k dataset."""
-
-import datasets
-import json
-import textwrap
-
-_CITATION = """\
-@misc{cobbe2021training,
-      title={Training Verifiers to Solve Math Word Problems},
-      author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
-      year={2021},
-      eprint={2110.14168},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG}
-}
-"""
-
-_DESCRIPTION = """\
-GSM8K (Grade School Math 8K) is a dataset of 8.5K high quality
-linguistically diverse grade school math word problems. The
-dataset was created to support the task of question answering
-on basic mathematical problems that require multi-step reasoning.
-"""
-
-_HOMEPAGE = 'https://openai.com/blog/grade-school-math'
-_MODELSCOPE_PAGE = 'https://modelscope.cn/datasets/modelscope/gsm8k/summary'
-
-_LICENSE = 'MIT'
-
-# _BASE_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/"
-TRAIN_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/gsm8k/train.jsonl'
-TEST_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/gsm8k/test.jsonl'
-
-
-class Gsm8kConfig(datasets.BuilderConfig):
-    """BuilderConfig for GSM8K."""
-
-    def __init__(self, urls, **kwargs):
-        """BuilderConfig for GSM8K.
-        Args:
-        urls: *dict[string]*, the urls for each split of the GSM8k set.
-        """
-        super().__init__(version=datasets.Version('1.1.0'), **kwargs)
-        self.urls = urls
-
-
-class Gsm8k(datasets.GeneratorBasedBuilder):
-    """Grade School Math 8k (GSM8K)"""
-
-    BUILDER_CONFIGS = [
-        Gsm8kConfig(
-            name='main',
-            description=textwrap.dedent(
-                """
-                It is segmented into 7.5K training problems and 1K test problems.
-                These problems take between 2 and 8 steps to solve, and solutions
-                primarily involve performing a sequence of elementary calculations
-                using basic arithmetic operations (+ - / *) to reach the final
-                answer. A bright middle school student should be able to solve
-                every problem.
-                """, ),
-            urls={
-                'train': TRAIN_URL,
-                'test': TEST_URL,
-            },
-        ),
-    ]
-
-    def _info(self):
-        features = datasets.Features({
-            'question': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(self.config.urls)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': data_dir['train'],
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': data_dir['test'],
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        with open(filepath, encoding='utf-8') as f:
-            for key, row in enumerate(f):
-                data = json.loads(row)
-                yield key, {
-                    'question': data['question'],
-                    'answer': data['answer'],
-                }

evalscope/benchmarks/hellaswag/hellaswag.py

@@ -1,112 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-"""HellaSwag: Can a Machine Really Finish Your Sentence? is a new dataset for commonsense NLI.
-    A paper was published at ACL2019.
-"""
-"""DO NOT EDIT."""
-
-import datasets
-import json
-
-# flake8: noqa
-
-# HomePage: https://rowanzellers.com/hellaswag/
-# GitHub: https://github.com/rowanz/hellaswag
-
-_CITATION = """\
-@inproceedings{zellers2019hellaswag,
-    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
-    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
-    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
-    year={2019}
-}
-"""
-
-_DESCRIPTION = """
-HellaSwag: Can a Machine Really Finish Your Sentence? is a new dataset for commonsense NLI. A paper was published at ACL2019.
-"""
-_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/hellaswag/'
-_URLS = {
-    'train': _URL + 'hellaswag_train.jsonl',
-    'test': _URL + 'hellaswag_test.jsonl',
-    'dev': _URL + 'hellaswag_val.jsonl',
-}
-
-
-class Hellaswag(datasets.GeneratorBasedBuilder):
-    """TODO(hellaswag): Short description of my dataset."""
-
-    # TODO(hellaswag): Set up version.
-    VERSION = datasets.Version('0.1.0')
-
-    def _info(self):
-        # TODO(hellaswag): Specifies the datasets.DatasetInfo object
-        return datasets.DatasetInfo(
-            # This is the description that will appear on the datasets page.
-            description=_DESCRIPTION,
-            # datasets.features.FeatureConnectors
-            features=datasets.Features({
-                # These are the features of your dataset like images, labels ...
-                'ind': datasets.Value('int32'),
-                'activity_label': datasets.Value('string'),
-                'ctx_a': datasets.Value('string'),
-                'ctx_b': datasets.Value('string'),
-                'ctx': datasets.Value('string'),
-                'endings': datasets.features.Sequence(datasets.Value('string')),
-                'source_id': datasets.Value('string'),
-                'split': datasets.Value('string'),
-                'split_type': datasets.Value('string'),
-                'label': datasets.Value('string'),
-            }),
-            # If there's a common (input, target) tuple from the features,
-            # specify them here. They'll be used if as_supervised=True in
-            # builder.as_dataset.
-            supervised_keys=None,
-            # Homepage of the dataset for documentation
-            homepage='https://rowanzellers.com/hellaswag/',
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        # TODO(hellaswag): Downloads the data and defines the splits
-        # dl_manager is a datasets.download.DownloadManager that can be used to
-        # download and extract URLs
-        urls_to_download = _URLS
-        dl_dir = dl_manager.download_and_extract(urls_to_download)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['train']},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['test']},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['dev']},
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        # TODO(hellaswag): Yields (key, example) tuples from the dataset
-        with open(filepath, encoding='utf-8') as f:
-            for id_, row in enumerate(f):
-                data = json.loads(row)
-                yield id_, {
-                    'ind': int(data['ind']),
-                    'activity_label': data['activity_label'],
-                    'ctx_a': data.get('ctx_a', ''),
-                    'ctx_b': data.get('ctx_b', ''),
-                    'ctx': data['ctx'],
-                    'endings': data.get('endings', []),
-                    'source_id': data['source_id'],
-                    'split': data['split'],
-                    'split_type': data['split_type'],
-                    'label': str(data.get('label', '')),
-                }

evalscope/benchmarks/humaneval/humaneval.py

@@ -1,79 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import datasets
-import json
-
-# flake8: noqa
-
-# NOTE: AUTOGENERATED, DO NOT CHANGE.
-
-_DESCRIPTION = """\
-The HumanEval dataset released by OpenAI contains 164 handcrafted programming challenges together with unittests to very the viability of a proposed solution.
-"""
-
-# _URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz"
-_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/humaneval/HumanEval.jsonl.gz'
-
-_CITATION = """\
-@misc{chen2021evaluating,
-      title={Evaluating Large Language Models Trained on Code},
-      author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba},
-      year={2021},
-      eprint={2107.03374},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG}
-}"""
-
-_HOMEPAGE = 'https://github.com/openai/human-eval'
-
-_LICENSE = 'MIT'
-
-
-class OpenaiHumaneval(datasets.GeneratorBasedBuilder):
-    """HumanEval: A benchmark for code generation."""
-
-    VERSION = datasets.Version('1.0.0')
-
-    BUILDER_CONFIGS = [
-        datasets.BuilderConfig(
-            name='openai_humaneval',
-            version=datasets.Version('1.0.0'),
-            description=_DESCRIPTION,
-        )
-    ]
-
-    def _info(self):
-        features = datasets.Features({
-            'task_id': datasets.Value('string'),
-            'prompt': datasets.Value('string'),
-            'canonical_solution': datasets.Value('string'),
-            'test': datasets.Value('string'),
-            'entry_point': datasets.Value('string'),
-        })
-
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            supervised_keys=None,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        data_dir = dl_manager.download_and_extract(_URL)
-        return [datasets.SplitGenerator(
-            name=datasets.Split.TEST,
-            gen_kwargs={
-                'filepath': data_dir,
-            },
-        )]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        with open(filepath, encoding='utf-8') as file:
-            data = [json.loads(line) for line in file]
-            id_ = 0
-            for sample in data:
-                yield id_, sample
-                id_ += 1

evalscope/benchmarks/mmlu/mmlu.py

@@ -1,160 +0,0 @@
-# isort: skip_file
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-
-import datasets
-import os
-import pandas as pd
-"""The MMLU dataset on ModelScope hub. READ ONLY, DO NOT MODIFY."""
-
-_CITATION = """\
-@article{hendryckstest2021,
-  title={Measuring Massive Multitask Language Understanding},
-  author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
-  journal={Proceedings of the International Conference on Learning Representations (ICLR)},
-  year={2021}
-}
-"""
-
-_DESCRIPTION = """\
-Measuring Massive Multitask Language Understanding by Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas
-Mazeika, Dawn Song, and Jacob Steinhardt (ICLR 2021).
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/mmlu/summary'
-
-_LICENSE = 'MIT'
-
-# _URL = "https://people.eecs.berkeley.edu/~hendrycks/data.tar"
-_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/mmlu/repo?Revision=master&FilePath=data.tar'
-
-task_list = [
-    'high_school_european_history',
-    'business_ethics',
-    'clinical_knowledge',
-    'medical_genetics',
-    'high_school_us_history',
-    'high_school_physics',
-    'high_school_world_history',
-    'virology',
-    'high_school_microeconomics',
-    'econometrics',
-    'college_computer_science',
-    'high_school_biology',
-    'abstract_algebra',
-    'professional_accounting',
-    'philosophy',
-    'professional_medicine',
-    'nutrition',
-    'global_facts',
-    'machine_learning',
-    'security_studies',
-    'public_relations',
-    'professional_psychology',
-    'prehistory',
-    'anatomy',
-    'human_sexuality',
-    'college_medicine',
-    'high_school_government_and_politics',
-    'college_chemistry',
-    'logical_fallacies',
-    'high_school_geography',
-    'elementary_mathematics',
-    'human_aging',
-    'college_mathematics',
-    'high_school_psychology',
-    'formal_logic',
-    'high_school_statistics',
-    'international_law',
-    'high_school_mathematics',
-    'high_school_computer_science',
-    'conceptual_physics',
-    'miscellaneous',
-    'high_school_chemistry',
-    'marketing',
-    'professional_law',
-    'management',
-    'college_physics',
-    'jurisprudence',
-    'world_religions',
-    'sociology',
-    'us_foreign_policy',
-    'high_school_macroeconomics',
-    'computer_security',
-    'moral_scenarios',
-    'moral_disputes',
-    'electrical_engineering',
-    'astronomy',
-    'college_biology',
-]
-
-
-class MMLUConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class MMLU(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [MMLUConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'input': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'target': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'test', f'{task_name}_test.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'val', f'{task_name}_val.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'data', 'dev', f'{task_name}_dev.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath)
-        df.columns = ['input', 'A', 'B', 'C', 'D', 'target']
-
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/mmlu/samples.jsonl

@@ -1,5 +0,0 @@
-{'input': 'A "dished face" profile is often associated with', 'A': 'a protruding mandible due to reactivation of the condylar cartilage by acromegaly.', 'B': 'a recessive maxilla due to failure of elongation of the cranial base.', 'C': 'an enlarged frontal bone due to hydrocephaly.', 'D': 'defective development of the maxillary air sinus.', 'target': 'B'}
-{'input': '___________ is based on the idea that customer expectations of the service they will receive shape their perception of the actual service encounter.', 'A': 'Service quality.', 'B': 'Service action.', 'C': 'Service recovery.', 'D': 'Service satisfaction.', 'target': 'A'}
-{'input': ' Information collected for the first time specifically for a marketing research study is called:', 'A': 'Secondary research.', 'B': 'Primary research.', 'C': 'Soft research.', 'D': 'Experimental research.', 'target': 'B'}
-{'input': "This includes advertisements that contain 'call-to-response' mechanisms such as telephone numbers, website addresses, email and postal addresses:", 'A': 'Direct response advertising.', 'B': 'Sales promotions.', 'C': 'Mass media advertising.', 'D': 'Public relations.', 'target': 'A'}
-{'input': 'Which of the following is not part of the external marketing environment?', 'A': 'Political.', 'B': 'Legal.', 'C': 'Product.', 'D': 'Socio-cultural.', 'target': 'C'}

evalscope/benchmarks/process_bench/critique_template.txt

@@ -1,13 +0,0 @@
-The following is a math problem and a solution (split into paragraphs, enclosed with tags and indexed from 0):
-
-[Math Problem]
-
-{problem}
-
-[Solution]
-
-{tagged_response}
-
-Your task is to review and critique the solution paragraph by paragraph. Once you identify an error in a paragraph, return the index of the paragraph where the earliest error occurs. Otherwise, return the index of -1 (which typically denotes "not found").
-
-Please put your final answer (i.e., the index) in \boxed{{}}.

evalscope/benchmarks/race/race.py

@@ -1,104 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import datasets
-import os
-import pandas as pd
-
-_CITATION = """\
-@inproceedings{lai-etal-2017-race,
-    title = "{RACE}: Large-scale {R}e{A}ding Comprehension Dataset From Examinations",
-    author = "Lai, Guokun  and
-      Xie, Qizhe  and
-      Liu, Hanxiao  and
-      Yang, Yiming  and
-      Hovy, Eduard",
-    booktitle = "Proceedings of the 2017 Conference on Empirical Methods in Natural Language Processing",
-    month = sep,
-    year = "2017",
-    address = "Copenhagen, Denmark",
-    publisher = "Association for Computational Linguistics",
-    url = "https://aclanthology.org/D17-1082",
-    doi = "10.18653/v1/D17-1082",
-    pages = "785--794",
-}
-"""
-
-_DESCRIPTION = """\
-RACE is a large-scale reading comprehension dataset with more than 28,000 passages and nearly 100,000 questions.
-"""
-
-_HOMEPAGE = 'https://modelscope.cn/datasets/modelscope/race/summary'
-
-_URL = 'https://modelscope.cn/api/v1/datasets/modelscope/race/repo?Revision=master&FilePath=race.zip'
-
-task_list = [
-    'high',
-    'middle',
-]
-
-
-class RACEConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class RACE(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [RACEConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'example_id': datasets.Value('string'),
-            'article': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-            'question': datasets.Value('string'),
-            'options': [datasets.Value('string')],
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/test/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/val/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, f'race/train/{task_name}-00000-of-00001.parquet'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_parquet(filepath)
-        df.columns = ['example_id', 'article', 'answer', 'question', 'options']
-
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            yield i, instance

evalscope/benchmarks/race/samples.jsonl

@@ -1,5 +0,0 @@
-{'example_id': 'middle4227.txt', 'article': 'There are many kinds...ealthy.\n,.', 'answer': 'D', 'question': 'We may read this pas... in   _  .', 'options': ['a letter', 'a story', 'a newspaper', 'a health magazine']}
-{'example_id': 'middle3329.txt', 'article': 'Do you know why diff...ng at all.', 'answer': 'B', 'question': 'Those pests with dif...of danger.', 'options': ['change their colours', 'hide in the day time...r at night', 'move quietly', 'hide at night and ap...e day time']}
-{'example_id': 'middle3614.txt', 'article': 'The seahorse is a ve...o the sea.', 'answer': 'B', 'question': 'A seahorse eats   _  .', 'options': ['sea weed', 'small fish', 'water', 'nothing']}
-{'example_id': 'middle6632.txt', 'article': 'Kids have unbelievab...h at her."', 'answer': 'D', 'question': 'Which is NOT mention...e passage?', 'options': ['Robots keep secrets.', 'Robots give suggestions.', 'Robots do chores.', 'Robots make movies.']}
-{'example_id': 'middle3503.txt', 'article': 'Have you ever heard ...eir lives.', 'answer': 'B', 'question': 'Which of the followi...lue moon"?', 'options': ['Simon often tells jo...blue moon.', 'Tom rarely remembers...blue moon.', 'Mary likes to go sho...blue moon.', 'Cindy hates to stay ...blue moon.']}

evalscope/benchmarks/super_gpqa/zero_shot_prompt.txt

@@ -1,4 +0,0 @@
-Answer the following multiple choice question. There is only one correct answer. The last line of your response should be in the format 'Answer: $LETTER' (without quotes), where LETTER is one of A, B, C, D, E, F, G, H, I, or J.
-
-{query}
-{choices}