evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/benchmarks/aigc/t2i/genai_bench_adapter.py

@@ -1,58 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
-
-logger = get_logger()
-
-
-@Benchmark.register(
-    name='genai_bench',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['GenAI-Bench-1600'],
-    metric_list=['VQAScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class GenAIBenchAdapter(T2IBaseAdapter):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
-
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
-
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-
-            res[metric_name] = score.cpu().item()
-
-            # fine-granular metrics
-            if gold['tags'].get('advanced'):
-                res[f'{metric_name}_advanced'] = score.cpu().item()
-            else:
-                res[f'{metric_name}_basic'] = score.cpu().item()
-
-        return res

evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py

@@ -1,58 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
-
-logger = get_logger()
-
-
-@Benchmark.register(
-    name='general_t2i',
-    dataset_id='general_t2i',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['default'],
-    metric_list=['PickScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class GeneralT2IAdapter(T2IBaseAdapter):
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
-
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict

evalscope/benchmarks/aigc/t2i/hpdv2_adapter.py

@@ -1,57 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
-
-logger = get_logger()
-
-
-@Benchmark.register(
-    name='hpdv2',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['HPDv2'],
-    metric_list=['HPSv2.1Score'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class HPDv2Adapter(T2IBaseAdapter):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
-
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
-
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-
-            res[metric_name] = score.cpu().item()
-
-            # fine-granular metrics
-            category = gold['tags'].get('category')
-            if category:
-                res[f'{metric_name}_{category}'] = score.cpu().item()
-
-        return res

evalscope/benchmarks/aigc/t2i/tifa_adapter.py

@@ -1,37 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
-
-logger = get_logger()
-
-
-@Benchmark.register(
-    name='tifa160',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['TIFA-160'],
-    metric_list=['PickScore'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class TIFA_Adapter(T2IBaseAdapter):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)

evalscope/benchmarks/arc/ai2_arc.py

@@ -1,151 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright (c) Allen Institute, and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-"""AI2 ARC (Abstraction and Reasoning Corpus) for General Artificial Intelligence Benchmark."""
-"""AUTO GENERATED, DO NOT EDIT"""
-
-import datasets
-import json
-import os
-
-# flake8: noqa
-
-_CITATION = """\
-@article{allenai:arc,
-      author    = {Peter Clark  and Isaac Cowhey and Oren Etzioni and Tushar Khot and
-                    Ashish Sabharwal and Carissa Schoenick and Oyvind Tafjord},
-      title     = {Think you have Solved Question Answering? Try ARC, the AI2 Reasoning Challenge},
-      journal   = {arXiv:1803.05457v1},
-      year      = {2018},
-}
-"""
-
-_DESCRIPTION = """\
-A new dataset of 7,787 genuine grade-school level, multiple-choice science questions, assembled to encourage research in
- advanced question-answering. The dataset is partitioned into a Challenge Set and an Easy Set, where the former contains
- only questions answered incorrectly by both a retrieval-based algorithm and a word co-occurrence algorithm. We are also
- including a corpus of over 14 million science sentences relevant to the task,
- and an implementation of three neural baseline models for this dataset. We pose ARC as a challenge to the community.
-
-ARC-Easy:
-    train: 2251
-    test: 2376
-    validation: 570
-
-ARC-Challenge:
-    train: 1119
-    test: 1172
-    validation: 299
-"""
-
-_URL = 'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/arc/ARC-V1-Feb2018.zip'
-
-# tasks: ['ARC-Easy', 'ARC-Challenge']
-
-
-class Ai2ArcConfig(datasets.BuilderConfig):
-    """BuilderConfig for Ai2ARC."""
-
-    def __init__(self, **kwargs):
-        """BuilderConfig for Ai2Arc.
-
-        Args:
-          **kwargs: keyword arguments forwarded to super.
-        """
-        super(Ai2ArcConfig, self).__init__(version=datasets.Version('1.0.0', ''), **kwargs)
-
-
-class Ai2Arc(datasets.GeneratorBasedBuilder):
-    """
-    The AI2 Reasoning Challenge (ARC) dataset.
-    Subset: ARC-Easy, ARC-Challenge.
-    """
-
-    VERSION = datasets.Version('1.0.0')
-    BUILDER_CONFIGS = [
-        Ai2ArcConfig(
-            name='ARC-Challenge',
-            description="""\
-          Challenge Set of 2590 “hard” questions (those that both a retrieval and a co-occurrence method fail to answer correctly)
-          """,
-        ),
-        Ai2ArcConfig(
-            name='ARC-Easy',
-            description="""\
-          Easy Set of 5197 questions
-          """,
-        ),
-    ]
-
-    def _info(self):
-        return datasets.DatasetInfo(
-            # This is the description that will appear on the datasets page.
-            description=_DESCRIPTION,
-            # datasets.features.FeatureConnectors
-            features=datasets.Features({
-                'id':
-                datasets.Value('string'),
-                'question':
-                datasets.Value('string'),
-                'choices':
-                datasets.features.Sequence({
-                    'text': datasets.Value('string'),
-                    'label': datasets.Value('string')
-                }),
-                'answerKey':
-                datasets.Value('string')
-                # These are the features of your dataset like images, labels ...
-            }),
-            # If there's a common (input, target) tuple from the features,
-            # specify them here. They'll be used if as_supervised=True in
-            # builder.as_dataset.
-            supervised_keys=None,
-            # Homepage of the dataset for documentation
-            homepage='https://allenai.org/data/arc',
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        """Returns SplitGenerators."""
-        # dl_manager is a datasets.download.DownloadManager that can be used to
-        # download and extract URLs
-        dl_dir = dl_manager.download_and_extract(_URL)
-        data_dir = os.path.join(dl_dir, 'ARC-V1-Feb2018-2')
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Train.jsonl')},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Test.jsonl')},
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                # These kwargs will be passed to _generate_examples
-                gen_kwargs={'filepath': os.path.join(data_dir, self.config.name, self.config.name + '-Dev.jsonl')},
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        """Yields examples."""
-        with open(filepath, encoding='utf-8') as f:
-            for row in f:
-                data = json.loads(row)
-                answerkey = data['answerKey']
-                id_ = data['id']
-                question = data['question']['stem']
-                choices = data['question']['choices']
-                text_choices = [choice['text'] for choice in choices]
-                label_choices = [choice['label'] for choice in choices]
-                yield id_, {
-                    'id': id_,
-                    'answerKey': answerkey,
-                    'question': question,
-                    'choices': {
-                        'text': text_choices,
-                        'label': label_choices
-                    },
-                }

evalscope/benchmarks/benchmark.py

@@ -1,81 +0,0 @@
-import copy
-from collections import OrderedDict
-from dataclasses import dataclass, field, fields
-from typing import TYPE_CHECKING, Dict, List, Optional
-
-from evalscope.constants import OutputType
-
-if TYPE_CHECKING:
-    from evalscope.benchmarks import DataAdapter
-
-BENCHMARK_MAPPINGS = {}
-
-
-@dataclass
-class BenchmarkMeta:
-    name: str
-    dataset_id: str
-    data_adapter: 'DataAdapter'
-    model_adapter: Optional[str] = OutputType.GENERATION
-    output_types: Optional[List[str]] = field(default_factory=lambda: [OutputType.GENERATION])
-    subset_list: List[str] = field(default_factory=lambda: ['default'])
-    metric_list: List[str] = field(default_factory=list)
-    few_shot_num: int = 0
-    few_shot_random: bool = False
-    train_split: Optional[str] = None
-    eval_split: Optional[str] = None
-    prompt_template: Optional[str] = None
-    system_prompt: Optional[str] = None
-    query_template: Optional[str] = None
-    pretty_name: Optional[str] = None
-    description: Optional[str] = None
-    tags: Optional[List[str]] = field(default_factory=list)
-    filters: Optional[OrderedDict] = None
-    extra_params: Optional[Dict] = field(default_factory=dict)
-
-    def _update(self, args: dict):
-        if args.get('local_path'):
-            self.dataset_id = args['local_path']
-            del args['local_path']
-        self.__dict__.update(args)
-
-    def to_dict(self) -> dict:
-        return self.__dict__
-
-    def to_string_dict(self) -> dict:
-        cur_dict = copy.deepcopy(self.to_dict())
-        # cur_dict['data_adapter'] = self.data_adapter.__name__
-        del cur_dict['data_adapter']
-        return cur_dict
-
-    def get_data_adapter(self, config: dict = {}) -> 'DataAdapter':
-        if config:
-            self._update(config)
-
-        data_adapter = self.data_adapter(**self.to_dict())
-        return data_adapter
-
-
-class Benchmark:
-
-    def __init__(self):
-        pass
-
-    @classmethod
-    def get(cls, name: str) -> 'BenchmarkMeta':
-        if name not in BENCHMARK_MAPPINGS:
-            raise Exception(f'Unknown benchmark: {name}. Available tasks: {list(BENCHMARK_MAPPINGS.keys())}')
-        benchmark = BENCHMARK_MAPPINGS[name]
-        return benchmark
-
-    @classmethod
-    def register(cls, name: str, dataset_id: str, **kwargs):
-
-        def register_wrapper(data_adapter):
-            if name in BENCHMARK_MAPPINGS:
-                raise Exception(f'Benchmark {name} already registered')
-            BENCHMARK_MAPPINGS[name] = BenchmarkMeta(
-                name=name, data_adapter=data_adapter, dataset_id=dataset_id, **kwargs)
-            return data_adapter
-
-        return register_wrapper

evalscope/benchmarks/ceval/ceval_exam.py

@@ -1,146 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License
-import datasets
-import os
-import pandas as pd
-
-# flake8: noqa
-"""DO NOT EDIT unless you are contributing a new dataset."""
-
-_CITATION = """\
-@article{huang2023ceval,
-    title={C-Eval: A Multi-Level Multi-Discipline Chinese Evaluation Suite for Foundation Models},
-    author={Huang, Yuzhen and Bai, Yuzhuo and Zhu, Zhihao and Zhang, Junlei and Zhang, Jinghan and Su, Tangjun and Liu, Junteng and Lv, Chuancheng and Zhang, Yikai and Lei, Jiayi and Fu, Yao and Sun, Maosong and He, Junxian},
-    journal={arXiv preprint arXiv:2305.08322},
-    year={2023}
-}
-"""
-
-_DESCRIPTION = """\
-C-Eval is a comprehensive Chinese evaluation suite for foundation models. It consists of 13948 multi-choice questions spanning 52 diverse disciplines and four difficulty levels.
-"""
-
-_HOMEPAGE = 'https://cevalbenchmark.com'
-
-_LICENSE = 'Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License'
-
-_URL = r'https://modelscope.oss-cn-beijing.aliyuncs.com/open_data/c-eval/ceval-exam.zip'
-
-task_list = [
-    'computer_network',
-    'operating_system',
-    'computer_architecture',
-    'college_programming',
-    'college_physics',
-    'college_chemistry',
-    'advanced_mathematics',
-    'probability_and_statistics',
-    'discrete_mathematics',
-    'electrical_engineer',
-    'metrology_engineer',
-    'high_school_mathematics',
-    'high_school_physics',
-    'high_school_chemistry',
-    'high_school_biology',
-    'middle_school_mathematics',
-    'middle_school_biology',
-    'middle_school_physics',
-    'middle_school_chemistry',
-    'veterinary_medicine',
-    'college_economics',
-    'business_administration',
-    'marxism',
-    'mao_zedong_thought',
-    'education_science',
-    'teacher_qualification',
-    'high_school_politics',
-    'high_school_geography',
-    'middle_school_politics',
-    'middle_school_geography',
-    'modern_chinese_history',
-    'ideological_and_moral_cultivation',
-    'logic',
-    'law',
-    'chinese_language_and_literature',
-    'art_studies',
-    'professional_tour_guide',
-    'legal_professional',
-    'high_school_chinese',
-    'high_school_history',
-    'middle_school_history',
-    'civil_servant',
-    'sports_science',
-    'plant_protection',
-    'basic_medicine',
-    'clinical_medicine',
-    'urban_and_rural_planner',
-    'accountant',
-    'fire_engineer',
-    'environmental_impact_assessment_engineer',
-    'tax_accountant',
-    'physician',
-]
-
-
-class CevalExamConfig(datasets.BuilderConfig):
-
-    def __init__(self, **kwargs):
-        super().__init__(version=datasets.Version('1.0.0'), **kwargs)
-
-
-class CevalExam(datasets.GeneratorBasedBuilder):
-    BUILDER_CONFIGS = [CevalExamConfig(name=task_name, ) for task_name in task_list]
-
-    def _info(self):
-        features = datasets.Features({
-            'id': datasets.Value('int32'),
-            'question': datasets.Value('string'),
-            'A': datasets.Value('string'),
-            'B': datasets.Value('string'),
-            'C': datasets.Value('string'),
-            'D': datasets.Value('string'),
-            'answer': datasets.Value('string'),
-            'explanation': datasets.Value('string'),
-        })
-        return datasets.DatasetInfo(
-            description=_DESCRIPTION,
-            features=features,
-            homepage=_HOMEPAGE,
-            license=_LICENSE,
-            citation=_CITATION,
-        )
-
-    def _split_generators(self, dl_manager):
-        data_dir = dl_manager.download_and_extract(_URL)
-        task_name = self.config.name
-        return [
-            datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'test', f'{task_name}_test.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('val'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'val', f'{task_name}_val.csv'),
-                },
-            ),
-            datasets.SplitGenerator(
-                name=datasets.Split('dev'),
-                gen_kwargs={
-                    'filepath': os.path.join(data_dir, 'dev', f'{task_name}_dev.csv'),
-                },
-            ),
-        ]
-
-    def _generate_examples(self, filepath):
-        df = pd.read_csv(filepath, encoding='utf-8')
-        for i, instance in enumerate(df.to_dict(orient='records')):
-            if 'answer' not in instance.keys():
-                instance['answer'] = ''
-            if 'explanation' not in instance.keys():
-                instance['explanation'] = ''
-            yield i, instance