evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/data_adapter.py

@@ -1,291 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-import random
-from abc import ABC, abstractmethod
-from typing import Any, List, Optional
-
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
-from evalscope.metrics import Metric
-from evalscope.report import Report, ReportGenerator
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class DataAdapter(ABC):
-
-    def __init__(self,
-                 name: str,
-                 subset_list: list,
-                 metric_list: List[Metric],
-                 few_shot_num: Optional[int] = 0,
-                 train_split: Optional[str] = None,
-                 eval_split: Optional[str] = None,
-                 prompt_template: Optional[str] = None,
-                 **kwargs):
-        """
-        Data Adapter for the benchmark. You need to implement the following methods:
-            - gen_prompt
-            - get_gold_answer
-            - parse_pred_result
-            - match
-        Args:
-            name: str, the name of the benchmark.
-            subset_list: list of subset names for the dataset.
-            metric_list: list, the metric list to evaluate the model on specific benchmark.
-            few_shot_num: int, number of few-shot examples. Default: 0
-            train_split: str, usually for few-shot examples. e.g. 'train'
-            eval_split: str, the target eval split name. e.g. 'test'
-            prompt_template: str, the prompt template for the benchmark,
-                e.g. for ARC, it is `The following are multiple choice questions, please output correct answer in
-                    the form of A or B or C or D, do not output explanation:`
-        """
-        self.name = name
-        self.subset_list = subset_list
-        self.metric_list = metric_list
-        self.few_shot_num = few_shot_num
-        self.train_split = train_split
-        self.eval_split = eval_split
-        self.prompt_template = prompt_template
-        self.config_kwargs = kwargs
-        self.category_map = kwargs.get('category_map', {})
-
-    def load(self,
-             dataset_name_or_path: str,
-             subset_list: list = None,
-             work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
-             **kwargs) -> dict:
-        """
-        Load the dataset. Remote and local datasets are supported.
-        You can rewrite this method to support your own local dataset, just follow the format of the output.
-
-        Returns: {'subset_name': {'train': train_dataset, 'test': test_dataset}}
-            train_dataset, test_dataset: Iterable dataset, object each item of which is a dict.
-
-        """
-        dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
-        subset_list = subset_list or self.subset_list
-
-        # Try to load dataset from local disk
-        if os.path.exists(dataset_name_or_path):
-            logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                    subsets: {subset_list}')
-            data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
-            if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
-        else:
-            from modelscope.msdatasets import MsDataset
-
-            # Load dataset from remote
-            logger.info(
-                f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
-            data_dict = {}
-            split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-            if len(split_list) == 0:
-                logger.error(f'Got empty split list: {split_list}')
-
-            for sub_name in subset_list:
-                data_dict[sub_name] = {}
-                # e.g. train: few-shot, test: target dataset to evaluate
-                for split in split_list:
-                    dataset = MsDataset.load(
-                        dataset_name=dataset_name_or_path,
-                        subset_name=sub_name,
-                        split=split,
-                        cache_dir=work_dir,
-                        hub=datasets_hub,
-                        **kwargs)
-
-                    data_dict[sub_name].update({split: dataset})
-
-        return data_dict
-
-    def load_from_disk(self, *args, **kwargs) -> dict:
-        """
-        Load the dataset from local disk.
-        If you want to support local dataset, please rewrite this method in xxx_data_adapter.
-        """
-        return {}
-
-    def gen_prompts(self, data_dict: dict) -> dict:
-        """
-        Generate dataset prompts from raw input, unify the prompt format for different datasets.
-
-        Args:
-            data_dict:  Refer to the output of load method: evalscope.benchmarks.benchmark.Benchmark.load
-
-        Returns:
-            {'subset_name': [prompt_d_1, prompt_d_2, ...]}
-            prompt_d_i (dict): refer to the output of gen_prompt method.
-
-        e.g. train -- few-shot data, test -- target dataset to evaluate.
-        """
-        res_dict: dict = {}
-
-        if self.few_shot_num and self.few_shot_num < 0:
-            raise ValueError(f'Invalid shot_num: {self.few_shot_num} for few-shot evaluation.')
-
-        logger.info(f'Use default settings: '
-                    f'> few_shot_num: {self.few_shot_num}, '
-                    f'> few_shot_split: {self.train_split}, '
-                    f'> target_eval_split: {self.eval_split}')
-
-        for sub_name, sub_data_dict in data_dict.items():
-            few_shot_data = []
-            if self.few_shot_num and self.few_shot_num > 0:
-                few_shot_random: bool = self.config_kwargs.get('few_shot_random', True)
-                few_shot_data = self.get_fewshot_examples([item for item in sub_data_dict[self.train_split]],
-                                                          self.few_shot_num,
-                                                          few_shot_random=few_shot_random)
-
-            res_dict[sub_name] = []
-            for sample_d in sub_data_dict[self.eval_split]:
-                prompt_d = self.gen_prompt(input_d=sample_d, subset_name=sub_name, few_shot_list=few_shot_data)
-                prompt_d[AnswerKeys.RAW_INPUT] = sample_d
-                res_dict[sub_name].append(prompt_d)
-
-        return res_dict
-
-    def get_fewshot_examples(self, data_list: list, k: int, few_shot_random: bool = True):
-
-        if k > len(data_list):
-            k = len(data_list)
-        if few_shot_random:
-            return random.sample(data_list, k)
-        else:
-            return data_list[:k]
-
-    def compute_metric(self, review_res_list: list) -> List[dict]:
-        """
-        Compute evaluation result by specific metrics.
-
-        Args:
-            review_res_list: list, the review result list, each item of which is match result for gold and pred.
-
-        Returns:
-            Metric results. e.g. [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]
-        """
-        if len(self.metric_list) == 0:
-            raise ValueError('No metric list found for the benchmark.')
-
-        res_list = []
-        for metric in self.metric_list:
-            metric_name = metric.name
-            metric_func = metric.object
-            res_list.append({
-                'metric_name': metric_name,
-                'score': metric_func(review_res_list),
-                'num': len(review_res_list)
-            })
-        return res_list
-
-    def gen_report(self, subset_score_map: dict, report_name: str = None, **kwargs) -> Report:
-        """
-        Generate report for the evaluation results for all subsets.
-
-        Args:
-            subset_score_map: The subset-score map.
-                e.g. {subset_name: [{'metric_name': 'AverageAccuracy', 'score': 0.3389, 'num': 100}]}
-
-            report_name: str, the user-defined report name. Default: None
-
-        Returns: The evaluation report.
-
-        Here is a format example for gsm8k:
-        {
-            "name": "qwen2.5_gsm8k",
-            "metrics": [
-                {
-                    "name": "AverageAccuracy",
-                    "categories": [
-                        {
-                            "name": "default",
-                            "subsets": [
-                                {
-                                    "name": "main",
-                                    "score": 0.0,
-                                    "num": 2
-                                }
-                            ],
-                            "num": 2,
-                            "score": 0.0,
-                            "macro_score": 0.0
-                        }
-                    ],
-                    "num": 2,
-                    "score": 0.0,
-                    "macro_score": 0.0
-                }
-            ],
-            "dataset_name": "gsm8k",
-            "model_name": "qwen2.5"
-        }
-        """  # noqa: E501
-        kwargs['category_map'] = self.category_map
-        kwargs['metric_list'] = self.metric_list
-        return ReportGenerator.gen_report(subset_score_map, report_name, **kwargs)
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        """
-        Generate model prompt from raw input, unify the prompt format for different datasets.
-        The input format is compatible with OpenAI Chat Completions APIs.
-
-        Args:
-            input_d (Any): The raw input. Depending on the dataset.
-            subset_name (str): The subset name.
-            few_shot_list (list): The few-shot examples.
-
-        Returns:
-            For class ChatGenerationModelAdapter, the output format is:
-                {'data': [full_prompt], 'system_prompt': (str, optional)},  -- full_prompt: str, the constructed prompt for each sample from dataset.
-            For class MultiChoiceModelAdapter, the output format is:
-                {'data': [full_prompt], 'multi_choices': self.choices}  -- full_prompt: str, the constructed prompt for each sample from dataset.
-            For class ContinuationEvalModelAdapter, the output format is:
-                {'data': ctx_continuation_pair_list, 'multi_choices': self.choices} -- ctx_continuation_pair_list: list, the context-continuation pair list.
-        """  # noqa: E501
-        raise NotImplementedError
-
-    @abstractmethod
-    def get_gold_answer(self, input_d: Any) -> Any:
-        """
-        Parse the raw input labels (gold).
-
-        Args:
-            input_d: input raw data. Depending on the dataset.
-
-        Returns:
-            The parsed input. e.g. gold answer ... Depending on the dataset.
-        """
-        raise NotImplementedError
-
-    @abstractmethod
-    def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
-        """
-        Parse the predicted result and extract proper answer.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d: The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        raise NotImplementedError
-
-    @abstractmethod
-    def match(self, gold: Any, pred: Any) -> Any:
-        """
-        Match the gold answer and the predicted answer.
-
-        Args:
-            gold (Any): The golden answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'A', extracted from get_gold_answer method.
-            pred (Any): The predicted answer. Usually a string for chat/multiple-choice-questions.
-                        e.g. 'B', extracted from parse_pred_result method.
-
-        Returns:
-            The match result. Usually a score (float) for chat/multiple-choice-questions.
-        """
-        raise NotImplementedError

evalscope/benchmarks/gsm8k/gsm8k.py

@@ -1,121 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# flake8: noqa
-"""Grade School Math 8k dataset."""
-
-import datasets
-import json
-import textwrap
-
-_CITATION = """\
-@misc{cobbe2021training,
-      title={Training Verifiers to Solve Math Word Problems},
-      author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},
-      year={2021},
-      eprint={2110.14168},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG}
-}
-"""
-
-_DESCRIPTION = """\
-GSM8K (Grade School Math 8K) is a dataset of 8.5K high quality
-linguistically diverse grade school math word problems. The
-dataset was created to support the task of question answering
-on basic mathematical problems that require multi-step reasoning.
-"""
-
-_HOMEPAGE = 'https://openai.com/blog/grade-school-math'
-_MODELSCOPE_PAGE = 'https://modelscope.cn/datasets/modelscope/gsm8k/summary'
-
-_LICENSE = 'MIT'
-
-# _BASE_URL = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/"
-TRAIN_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/gsm8k/train.jsonl'
-TEST_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/gsm8k/test.jsonl'
-
-
-class Gsm8kConfig(datasets.BuilderConfig):
-    """BuilderConfig for GSM8K."""
-
-    def __init__(self, urls, **kwargs):
-        """BuilderConfig for GSM8K.
-        Args:
-        urls: *dict[string]*, the urls for each split of the GSM8k set.
-        """
-        super().__init__(version=datasets.Version('1.1.0'), **kwargs)
-        self.urls = urls
-
-
-class Gsm8k(datasets.GeneratorBasedBuilder):
-    """Grade School Math 8k (GSM8K)"""
-
-    BUILDER_CONFIGS = [
-        Gsm8kConfig(
-            name='main',
-            description=textwrap.dedent(
-                """
-                It is segmented into 7.5K training problems and 1K test problems.
-                These problems take between 2 and 8 steps to solve, and solutions
-                primarily involve performing a sequence of elementary calculations
-                using basic arithmetic operations (+ - / *) to reach the final
-                answer. A bright middle school student should be able to solve
-                every problem.
-                """, ),
-            urls={
-                'train': TRAIN_URL,
-                'test': TEST_URL,
-            },
-        ),
-    ]
-
-    def _info(self):
-        features = datasets.Features({
-            'question': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(self.config.urls)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    'filepath': data_dir['train'],
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': data_dir['test'],
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        with open(filepath, encoding='utf-8') as f:
-            for key, row in enumerate(f):
-                data = json.loads(row)
-                yield key, {
-                    'question': data['question'],
-                    'answer': data['answer'],
-                }

evalscope/benchmarks/hellaswag/hellaswag.py

@@ -1,112 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-"""HellaSwag: Can a Machine Really Finish Your Sentence? is a new dataset for commonsense NLI.
-    A paper was published at ACL2019.
-"""
-"""DO NOT EDIT."""
-
-import datasets
-import json
-
-# flake8: noqa
-
-# HomePage: https://rowanzellers.com/hellaswag/
-# GitHub: https://github.com/rowanz/hellaswag
-
-_CITATION = """\
-@inproceedings{zellers2019hellaswag,
-    title={HellaSwag: Can a Machine Really Finish Your Sentence?},
-    author={Zellers, Rowan and Holtzman, Ari and Bisk, Yonatan and Farhadi, Ali and Choi, Yejin},
-    booktitle ={Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics},
-    year={2019}
-}
-"""
-
-_DESCRIPTION = """
-HellaSwag: Can a Machine Really Finish Your Sentence? is a new dataset for commonsense NLI. A paper was published at ACL2019.
-"""
-_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/hellaswag/'
-_URLS = {
-    'train': _URL + 'hellaswag_train.jsonl',
-    'test': _URL + 'hellaswag_test.jsonl',
-    'dev': _URL + 'hellaswag_val.jsonl',
-}
-
-
-class Hellaswag(datasets.GeneratorBasedBuilder):
-    """TODO(hellaswag): Short description of my dataset."""
-
-    # TODO(hellaswag): Set up version.
-    VERSION = datasets.Version('0.1.0')
-
-    def _info(self):
-        # TODO(hellaswag): Specifies the datasets.DatasetInfo object
-        return datasets.DatasetInfo(
-            # This is the description that will appear on the datasets page.
-            description=_DESCRIPTION,
-            # datasets.features.FeatureConnectors
-            features=datasets.Features({
-                # These are the features of your dataset like images, labels ...
-                'ind': datasets.Value('int32'),
-                'activity_label': datasets.Value('string'),
-                'ctx_a': datasets.Value('string'),
-                'ctx_b': datasets.Value('string'),
-                'ctx': datasets.Value('string'),
-                'endings': datasets.features.Sequence(datasets.Value('string')),
-                'source_id': datasets.Value('string'),
-                'split': datasets.Value('string'),
-                'split_type': datasets.Value('string'),
-                'label': datasets.Value('string'),
-            }),
-            # If there's a common (input, target) tuple from the features,
-            # specify them here. They'll be used if as_supervised=True in
-            # builder.as_dataset.
-            supervised_keys=None,
-            # Homepage of the dataset for documentation
-            homepage='https://rowanzellers.com/hellaswag/',
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        # TODO(hellaswag): Downloads the data and defines the splits
-        # dl_manager is a datasets.download.DownloadManager that can be used to
-        # download and extract URLs
-        urls_to_download = _URLS
-        dl_dir = dl_manager.download_and_extract(urls_to_download)
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['train']},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['test']},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': dl_dir['dev']},
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        # TODO(hellaswag): Yields (key, example) tuples from the dataset
-        with open(filepath, encoding='utf-8') as f:
-            for id_, row in enumerate(f):
-                data = json.loads(row)
-                yield id_, {
-                    'ind': int(data['ind']),
-                    'activity_label': data['activity_label'],
-                    'ctx_a': data.get('ctx_a', ''),
-                    'ctx_b': data.get('ctx_b', ''),
-                    'ctx': data['ctx'],
-                    'endings': data.get('endings', []),
-                    'source_id': data['source_id'],
-                    'split': data['split'],
-                    'split_type': data['split_type'],
-                    'label': str(data.get('label', '')),
-                }

evalscope/benchmarks/humaneval/humaneval.py

@@ -1,79 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import datasets
-import json
-
-# flake8: noqa
-
-# NOTE: AUTOGENERATED, DO NOT CHANGE.
-
-_DESCRIPTION = """\
-The HumanEval dataset released by OpenAI contains 164 handcrafted programming challenges together with unittests to very the viability of a proposed solution.
-"""
-
-# _URL = "https://raw.githubusercontent.com/openai/human-eval/master/data/HumanEval.jsonl.gz"
-_URL = 'https://sail-moe.oss-cn-hangzhou.aliyuncs.com/open_data/humaneval/HumanEval.jsonl.gz'
-
-_CITATION = """\
-@misc{chen2021evaluating,
-      title={Evaluating Large Language Models Trained on Code},
-      author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba},
-      year={2021},
-      eprint={2107.03374},
-      archivePrefix={arXiv},
-      primaryClass={cs.LG}
-}"""
-
-_HOMEPAGE = 'https://github.com/openai/human-eval'
-
-_LICENSE = 'MIT'
-
-
-class OpenaiHumaneval(datasets.GeneratorBasedBuilder):
-    """HumanEval: A benchmark for code generation."""
-
-    VERSION = datasets.Version('1.0.0')
-
-    BUILDER_CONFIGS = [
-        datasets.BuilderConfig(
-            name='openai_humaneval',
-            version=datasets.Version('1.0.0'),
-            description=_DESCRIPTION,
-        )
-    ]
-
-    def _info(self):
-        features = datasets.Features({
-            'task_id': datasets.Value('string'),
-            'prompt': datasets.Value('string'),
-            'canonical_solution': datasets.Value('string'),
-            'test': datasets.Value('string'),
-            'entry_point': datasets.Value('string'),
-        })
-
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            supervised_keys=None,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        data_dir = dl_manager.download_and_extract(_URL)
-        return [datasets.SplitGenerator(
-            name=datasets.Split.TEST,
-            gen_kwargs={
-                'filepath': data_dir,
-            },
-        )]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        with open(filepath, encoding='utf-8') as file:
-            data = [json.loads(line) for line in file]
-            id_ = 0
-            for sample in data:
-                yield id_, sample
-                id_ += 1