evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

{evalscope-1.0.0.dist-info → evalscope-1.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: setuptools (79.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{evalscope-1.0.0.dist-info → evalscope-1.2.0.dist-info}/top_level.txt

@@ -1,2 +1 @@
 evalscope
-tests

evalscope/api/mixin/dataset_mixin.py

@@ -1,105 +0,0 @@
-from abc import ABC
-from collections import defaultdict
-from typing import Any, Callable, Dict
-
-from evalscope.api.dataset import Dataset, DatasetDict, RemoteDataLoader
-
-
-class DatasetLoaderMixin:
-    """
-    Mixin class providing dataset loading functionality for benchmarks.
-
-    This mixin provides common dataset loading methods that can be shared
-    across different data adapters, including support for:
-    - Loading multiple subsets
-    - Few-shot dataset loading
-    - Remote dataset loading with configuration
-    """
-
-    def load_subsets(self, load_func: Callable[[str], Dataset]) -> DatasetDict:
-        """
-        Load multiple subsets of the dataset using the provided loading function.
-
-        This method handles two loading strategies:
-        1. Reformat mode: Load only the default subset and reformat it
-        2. Multi-subset mode: Load all subsets specified in subset_list
-
-        Args:
-            load_func (Callable[[str], Dataset]): Function to load individual subsets
-
-        Returns:
-            DatasetDict: Dictionary containing all loaded subsets
-        """
-        if self.reformat_subset:
-            # Load only the default subset
-            subset_data = load_func(self.default_subset)
-            # Reformat the subset to create multiple subsets based on sample keys
-            # NOTE: subset_list and limit is applied here if specified
-            dataset_dict = DatasetDict.from_dataset(dataset=subset_data, subset_list=self.subset_list, limit=self.limit)
-        else:
-            # Load all specified subsets into separate entries
-            subset_dict = defaultdict()
-            for subset in self.subset_list:
-                subset_data = load_func(subset)
-                subset_dict[subset] = subset_data
-            dataset_dict = DatasetDict(subset_dict)
-        return dataset_dict
-
-    def load_subset(self, subset: str) -> Dataset:
-        """
-        Load a specific subset of the dataset for evaluation.
-
-        This method configures and executes the data loading for a single subset,
-        handling both split-as-subset and traditional subset configurations.
-
-        Args:
-            subset (str): The subset identifier to load
-
-        Returns:
-            Dataset: The loaded dataset subset with processed samples
-        """
-        # Determine the split and subset names based on configuration
-        split = subset if self.split_as_subset else self.eval_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create and configure the remote data loader
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,  # Custom sample conversion function
-            limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
-            repeats=self._task_config.repeats,  # Number of repetitions for each sample
-            data_source=self._task_config.dataset_hub,  # Data source configuration
-        )
-        return loader.load()
-
-    def load_fewshot_subset(self, subset: str) -> Dataset:
-        """
-        Load a subset specifically for few-shot examples.
-
-        This method loads training data to be used as demonstrations in few-shot prompting.
-        It typically loads from the training split with limited samples and optional shuffling.
-
-        Args:
-            subset (str): The subset identifier to load few-shot examples from
-
-        Returns:
-            Dataset: The loaded few-shot dataset with demonstration examples
-        """
-        # Use training split for few-shot examples
-        split = subset if self.split_as_subset else self.train_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create loader specifically configured for few-shot sampling
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,
-            limit=self.few_shot_num
-            if not self.reformat_subset else None,  # Limit to specified number of few-shot examples
-            shuffle=self.few_shot_random,  # Randomize selection if enabled
-            data_source=self._task_config.dataset_hub,
-        )
-        return loader.load()

evalscope/benchmarks/aigc/i2i/general_i2i_adapter.py

@@ -1,44 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class GeneralI2IAdapter:
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
-
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict

tests/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/aigc/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/aigc/test_t2i.py

@@ -1,142 +0,0 @@
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import os
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, ModelTask, OutputType
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-
-class TestRun(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            datasets=[
-                'general_t2i'
-            ],
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            dataset_args={
-                'general_t2i': {
-                    'metric_list': [
-                        'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'HPSv2.1Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        # 'FGA_BLIP2Score',
-                        # 'MPS'
-                    ],
-                    'dataset_id': 'custom_eval/multimodal/t2i/example.jsonl',
-                }
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    def test_run_local_evalmuse(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model_id='T2I-Model', # 只用于展示，实际运行时不需要指定模型ID
-            model_task=ModelTask.IMAGE_GENERATION,
-            datasets=[
-                'evalmuse',  # 使用 EvalMuse benchmark
-            ],
-            dataset_args={
-                'evalmuse': {
-                    'dataset_id': 'data/example.jsonl',  # 构建的jsonl路径
-                }
-            },
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_benchmark(self):
-
-        task_cfg = TaskConfig(
-            model='stabilityai/stable-diffusion-xl-base-1.0',  # model on modelscope
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            model_args={
-                'use_safetensors': True,
-                'variant': 'fp16',
-                'torch_dtype': 'torch.float16',
-            },
-            datasets=[
-                # 'tifa160',
-                # 'genai_bench',
-                'evalmuse',
-                # 'hpdv2',
-            ],
-            dataset_args={
-                'tifa160': {
-                    'metric_list': [
-                        # 'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        'FGA_BLIP2Score',
-                    ]
-                }
-            },
-            limit=5,
-            generation_config={
-                'num_inference_steps': 50,
-                'guidance_scale': 7.5
-            },
-            # use_cache='outputs/20250427_134122',
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_benchmark_flux(self):
-
-        task_cfg = TaskConfig(
-            model='black-forest-labs/FLUX.1-dev',  # model on modelscope
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            model_args={
-                'torch_dtype': 'torch.float16',
-            },
-            datasets=[
-                # 'tifa160',
-                # 'genai_bench',
-                'evalmuse',
-                # 'hpdv2',
-            ],
-            dataset_args={
-                'tifa160': {
-                    'metric_list': [
-                        'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        # 'FGA_BLIP2Score',
-                    ]
-                }
-            },
-            generation_config={
-                'num_inference_steps': 50,
-                'guidance_scale': 3.5
-            },
-            use_cache='outputs/20250520_112314'
-        )
-
-        run_task(task_cfg=task_cfg)

tests/benchmark/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.