evalscope 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

tests/cli/test_all.py

@@ -17,44 +17,44 @@ os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 logger = get_logger()
 
 datasets=[
-        'iquiz',
-        'ifeval',
-        'mmlu',
-        'mmlu_pro',
-        'musr',
-        'process_bench',
-        'race',
-        'trivia_qa',
-        'cmmlu',
-        'humaneval',
-        'gsm8k',
-        'bbh',
-        'competition_math',
-        'math_500',
-        'aime24',
-        'gpqa_diamond',
-        'arc',
-        'ceval',
-        'hellaswag',
-        'general_mcq',
-        'general_qa',
-        'super_gpqa',
-        # 'live_code_bench',
-        'mmlu_redux',
-        'simple_qa',
-        'chinese_simpleqa',
-        'alpaca_eval',
-        'arena_hard',
-        'maritime_bench',
-        'drop',
-        'winogrande',
-        'tool_bench',
-        'frames',
-        'docmath',
-        'needle_haystack',
-        'bfcl_v3',
-        'hle',
-        'tau_bench',
+    'iquiz',
+    'ifeval',
+    'mmlu',
+    'mmlu_pro',
+    'musr',
+    'process_bench',
+    'race',
+    'trivia_qa',
+    'cmmlu',
+    'humaneval',
+    'gsm8k',
+    'bbh',
+    'competition_math',
+    'math_500',
+    'aime24',
+    'gpqa_diamond',
+    'arc',
+    'ceval',
+    'hellaswag',
+    'general_mcq',
+    'general_qa',
+    'super_gpqa',
+    # 'live_code_bench',
+    'mmlu_redux',
+    'simple_qa',
+    'chinese_simpleqa',
+    'alpaca_eval',
+    'arena_hard',
+    'maritime_bench',
+    'drop',
+    'winogrande',
+    'tool_bench',
+    'frames',
+    'docmath',
+    'needle_haystack',
+    'bfcl_v3',
+    'hle',
+    'tau_bench',
 ]
 
 # Reverse the datasets list to ensure the order is from most recent to oldest
@@ -150,7 +150,6 @@ dataset_args={
 }
 
 class TestRun(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_benchmarks(self):
         from evalscope.config import TaskConfig
 
@@ -180,19 +179,60 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+    def test_vlm_benchmark(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-vl-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'mmmu',
+                # 'math_vista',
+            ],
+            dataset_args={
+                'mmmu': {
+                    'subset_list': ['Accounting']
+                },
+                'math_vista': {
+                    'subset_list': ['default']
+                }
+            },
+            eval_batch_size=1,
+            limit=1,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+                'image_height': 512,
+                'image_width': 512,
+                'image_num': 2,
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.AUTO,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+            }
+        )
+
+        run_task(task_cfg=task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_ci_lite(self):
         from evalscope.config import TaskConfig
 
+        api_key = env.get('DASHSCOPE_API_KEY')
+
         task_cfg = TaskConfig(
             model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
+            api_key=api_key,
+            eval_type=EvalType.SERVICE if api_key else EvalType.MOCK_LLM,
             datasets=[
                 'general_mcq',
-                'general_qa',
                 'iquiz',
             ],
             dataset_args={

tests/cli/test_collection.py

@@ -52,16 +52,19 @@ class TestCollection(unittest.TestCase):
             api_key=env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
+            dataset_args={
+                'data_collection': {
+                    # 'local_path': 'outputs/test_mix.jsonl'
+                    'local_path': 'outputs/mixed_data_test.jsonl',
+                    'shuffle': True,
+                }
+            },
             eval_batch_size=5,
             generation_config = {
                 'max_tokens': 10000,
                 'temperature': 0.0,
             },
-            limit=50,
+            limit=10,
             # use_cache='outputs/20250822_161804'
         )
         run_task(task_cfg=task_cfg)

tests/cli/test_reasoning.py

@@ -0,0 +1,81 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestReasoning(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'Qwen3-0.6B',
+            'api_url': 'http://0.0.0.0:8801/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True,
+                'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
+
+    # Math & Reasoning datasets
+    def test_gsm8k(self):
+        """Test GSM8K math reasoning dataset."""
+        self._run_dataset_test('gsm8k')
+
+
+if __name__ == '__main__':
+    # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k
+    # Run all tests: python -m unittest test_eval.TestBenchmark
+    unittest.main()

tests/common.py

@@ -0,0 +1,73 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestBenchmark(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'qwen-plus',
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if not env.get('DASHSCOPE_API_KEY'):
+            use_mock = True
+            logger.warning('DASHSCOPE_API_KEY is not set. Using mock evaluation.')
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)

tests/perf/test_perf.py

@@ -1,9 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 from dotenv import dotenv_values
 
 env = dotenv_values('.env')
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 import unittest
 
 from evalscope.perf.main import run_perf_benchmark
@@ -123,6 +121,10 @@ class TestPerf(unittest.TestCase):
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_multi_parallel(self):
+        if not env.get('DASHSCOPE_API_KEY'):
+            self.skipTest('DASHSCOPE_API_KEY is not set.')
+            return
+
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
             parallel=[1, 2],

tests/rag/test_clip_benchmark.py

@@ -1,8 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
 import os
-
-# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
 import subprocess
 import unittest
 

evalscope/api/mixin/dataset_mixin.py

@@ -1,105 +0,0 @@
-from abc import ABC
-from collections import defaultdict
-from typing import Any, Callable, Dict
-
-from evalscope.api.dataset import Dataset, DatasetDict, RemoteDataLoader
-
-
-class DatasetLoaderMixin:
-    """
-    Mixin class providing dataset loading functionality for benchmarks.
-
-    This mixin provides common dataset loading methods that can be shared
-    across different data adapters, including support for:
-    - Loading multiple subsets
-    - Few-shot dataset loading
-    - Remote dataset loading with configuration
-    """
-
-    def load_subsets(self, load_func: Callable[[str], Dataset]) -> DatasetDict:
-        """
-        Load multiple subsets of the dataset using the provided loading function.
-
-        This method handles two loading strategies:
-        1. Reformat mode: Load only the default subset and reformat it
-        2. Multi-subset mode: Load all subsets specified in subset_list
-
-        Args:
-            load_func (Callable[[str], Dataset]): Function to load individual subsets
-
-        Returns:
-            DatasetDict: Dictionary containing all loaded subsets
-        """
-        if self.reformat_subset:
-            # Load only the default subset
-            subset_data = load_func(self.default_subset)
-            # Reformat the subset to create multiple subsets based on sample keys
-            # NOTE: subset_list and limit is applied here if specified
-            dataset_dict = DatasetDict.from_dataset(dataset=subset_data, subset_list=self.subset_list, limit=self.limit)
-        else:
-            # Load all specified subsets into separate entries
-            subset_dict = defaultdict()
-            for subset in self.subset_list:
-                subset_data = load_func(subset)
-                subset_dict[subset] = subset_data
-            dataset_dict = DatasetDict(subset_dict)
-        return dataset_dict
-
-    def load_subset(self, subset: str) -> Dataset:
-        """
-        Load a specific subset of the dataset for evaluation.
-
-        This method configures and executes the data loading for a single subset,
-        handling both split-as-subset and traditional subset configurations.
-
-        Args:
-            subset (str): The subset identifier to load
-
-        Returns:
-            Dataset: The loaded dataset subset with processed samples
-        """
-        # Determine the split and subset names based on configuration
-        split = subset if self.split_as_subset else self.eval_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create and configure the remote data loader
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,  # Custom sample conversion function
-            limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
-            repeats=self._task_config.repeats,  # Number of repetitions for each sample
-            data_source=self._task_config.dataset_hub,  # Data source configuration
-        )
-        return loader.load()
-
-    def load_fewshot_subset(self, subset: str) -> Dataset:
-        """
-        Load a subset specifically for few-shot examples.
-
-        This method loads training data to be used as demonstrations in few-shot prompting.
-        It typically loads from the training split with limited samples and optional shuffling.
-
-        Args:
-            subset (str): The subset identifier to load few-shot examples from
-
-        Returns:
-            Dataset: The loaded few-shot dataset with demonstration examples
-        """
-        # Use training split for few-shot examples
-        split = subset if self.split_as_subset else self.train_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create loader specifically configured for few-shot sampling
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,
-            limit=self.few_shot_num
-            if not self.reformat_subset else None,  # Limit to specified number of few-shot examples
-            shuffle=self.few_shot_random,  # Randomize selection if enabled
-            data_source=self._task_config.dataset_hub,
-        )
-        return loader.load()

evalscope/benchmarks/aigc/i2i/general_i2i_adapter.py

@@ -1,44 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class GeneralI2IAdapter:
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
-
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict

tests/aigc/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.