evalscope 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

tests/cli/test_reasoning.py

@@ -0,0 +1,81 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestReasoning(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'Qwen3-0.6B',
+            'api_url': 'http://0.0.0.0:8801/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True,
+                'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
+
+    # Math & Reasoning datasets
+    def test_gsm8k(self):
+        """Test GSM8K math reasoning dataset."""
+        self._run_dataset_test('gsm8k')
+
+
+if __name__ == '__main__':
+    # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k
+    # Run all tests: python -m unittest test_eval.TestBenchmark
+    unittest.main()

tests/common.py

@@ -0,0 +1,73 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestBenchmark(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'qwen-plus',
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if not env.get('DASHSCOPE_API_KEY'):
+            use_mock = True
+            logger.warning('DASHSCOPE_API_KEY is not set. Using mock evaluation.')
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)

tests/perf/test_perf.py

@@ -1,9 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 from dotenv import dotenv_values
 
 env = dotenv_values('.env')
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 import unittest
 
 from evalscope.perf.main import run_perf_benchmark
@@ -18,7 +16,7 @@ class TestPerf(unittest.TestCase):
     def tearDown(self) -> None:
         pass
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf(self):
         task_cfg = {
             'url': 'http://127.0.0.1:8001/v1/chat/completions',
@@ -32,7 +30,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_stream(self):
         task_cfg = {
             'url': 'http://127.0.0.1:8801/v1/chat/completions',
@@ -46,7 +44,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_speed_benchmark(self):
         task_cfg = {
             'url': 'http://127.0.0.1:8001/v1/completions',
@@ -60,7 +58,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_local(self):
         task_cfg = {
             'parallel': 1,
@@ -72,7 +70,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_local_stream(self):
         task_cfg = {
             'parallel': 1,
@@ -85,7 +83,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_local_speed_benchmark(self):
         task_cfg = {
             'parallel': 1,
@@ -98,7 +96,7 @@ class TestPerf(unittest.TestCase):
         }
         run_perf_benchmark(task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_local_random(self):
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
@@ -121,13 +119,45 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_completion_endpoint(self):
+        if not env.get('DASHSCOPE_API_KEY'):
+            self.skipTest('DASHSCOPE_API_KEY is not set.')
+            return
+
+        from evalscope.perf.arguments import Arguments
+        task_cfg = Arguments(
+            parallel=[1, 2],
+            number=[2, 4],
+            model='qwen2.5-coder-7b-instruct',
+            url='https://dashscope.aliyuncs.com/compatible-mode/v1/completions',
+            api_key=env.get('DASHSCOPE_API_KEY'),
+            api='openai',
+            dataset='random',
+            min_tokens=100,
+            max_tokens=100,
+            prefix_length=0,
+            min_prompt_length=1024,
+            max_prompt_length=1024,
+            stream=False,
+            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            seed=None,
+            extra_args={'ignore_eos': True}
+        )
+        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
+        print(metrics_result)
+        print(percentile_result)
+
+
     def test_run_perf_multi_parallel(self):
+        if not env.get('DASHSCOPE_API_KEY'):
+            self.skipTest('DASHSCOPE_API_KEY is not set.')
+            return
+
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
             parallel=[1, 2],
             number=[2, 4],
-            model='qwen2.5-7b-instruct',
+            model='qwen-plus',
             url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
             api_key=env.get('DASHSCOPE_API_KEY'),
             api='openai',
@@ -145,7 +175,7 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+
     def test_run_perf_random_vl(self):
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
@@ -155,7 +185,7 @@ class TestPerf(unittest.TestCase):
             url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
             api_key=env.get('DASHSCOPE_API_KEY'),
             api='openai',
-            dataset='kontext_bench',
+            dataset='random_vl',
             min_tokens=100,
             max_tokens=100,
             prefix_length=0,
@@ -164,7 +194,7 @@ class TestPerf(unittest.TestCase):
             image_height=512,
             image_width=512,
             image_num=2,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            tokenizer_path='Qwen/Qwen2.5-VL-7B-Instruct',
             seed=None,
             extra_args={'ignore_eos': True}
         )

tests/rag/test_clip_benchmark.py

@@ -1,8 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
 import os
-
-# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
 import subprocess
 import unittest
 

evalscope/api/mixin/dataset_mixin.py

@@ -1,105 +0,0 @@
-from abc import ABC
-from collections import defaultdict
-from typing import Any, Callable, Dict
-
-from evalscope.api.dataset import Dataset, DatasetDict, RemoteDataLoader
-
-
-class DatasetLoaderMixin:
-    """
-    Mixin class providing dataset loading functionality for benchmarks.
-
-    This mixin provides common dataset loading methods that can be shared
-    across different data adapters, including support for:
-    - Loading multiple subsets
-    - Few-shot dataset loading
-    - Remote dataset loading with configuration
-    """
-
-    def load_subsets(self, load_func: Callable[[str], Dataset]) -> DatasetDict:
-        """
-        Load multiple subsets of the dataset using the provided loading function.
-
-        This method handles two loading strategies:
-        1. Reformat mode: Load only the default subset and reformat it
-        2. Multi-subset mode: Load all subsets specified in subset_list
-
-        Args:
-            load_func (Callable[[str], Dataset]): Function to load individual subsets
-
-        Returns:
-            DatasetDict: Dictionary containing all loaded subsets
-        """
-        if self.reformat_subset:
-            # Load only the default subset
-            subset_data = load_func(self.default_subset)
-            # Reformat the subset to create multiple subsets based on sample keys
-            # NOTE: subset_list and limit is applied here if specified
-            dataset_dict = DatasetDict.from_dataset(dataset=subset_data, subset_list=self.subset_list, limit=self.limit)
-        else:
-            # Load all specified subsets into separate entries
-            subset_dict = defaultdict()
-            for subset in self.subset_list:
-                subset_data = load_func(subset)
-                subset_dict[subset] = subset_data
-            dataset_dict = DatasetDict(subset_dict)
-        return dataset_dict
-
-    def load_subset(self, subset: str) -> Dataset:
-        """
-        Load a specific subset of the dataset for evaluation.
-
-        This method configures and executes the data loading for a single subset,
-        handling both split-as-subset and traditional subset configurations.
-
-        Args:
-            subset (str): The subset identifier to load
-
-        Returns:
-            Dataset: The loaded dataset subset with processed samples
-        """
-        # Determine the split and subset names based on configuration
-        split = subset if self.split_as_subset else self.eval_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create and configure the remote data loader
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,  # Custom sample conversion function
-            limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
-            repeats=self._task_config.repeats,  # Number of repetitions for each sample
-            data_source=self._task_config.dataset_hub,  # Data source configuration
-        )
-        return loader.load()
-
-    def load_fewshot_subset(self, subset: str) -> Dataset:
-        """
-        Load a subset specifically for few-shot examples.
-
-        This method loads training data to be used as demonstrations in few-shot prompting.
-        It typically loads from the training split with limited samples and optional shuffling.
-
-        Args:
-            subset (str): The subset identifier to load few-shot examples from
-
-        Returns:
-            Dataset: The loaded few-shot dataset with demonstration examples
-        """
-        # Use training split for few-shot examples
-        split = subset if self.split_as_subset else self.train_split
-        subset_name = self.default_subset if self.split_as_subset else subset
-
-        # Create loader specifically configured for few-shot sampling
-        loader = RemoteDataLoader(
-            data_id_or_path=self.dataset_id,
-            split=split,
-            subset=subset_name,
-            sample_fields=self.record_to_sample,
-            limit=self.few_shot_num
-            if not self.reformat_subset else None,  # Limit to specified number of few-shot examples
-            shuffle=self.few_shot_random,  # Randomize selection if enabled
-            data_source=self._task_config.dataset_hub,
-        )
-        return loader.load()

evalscope/benchmarks/aigc/i2i/general_i2i_adapter.py

@@ -1,44 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class GeneralI2IAdapter:
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
-
-    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
-        dataset_name_or_path = dataset_name_or_path or self.dataset_id
-        subset_list = subset_list or self.subset_list
-
-        data_file_dict = defaultdict(str)
-        data_item_dict = defaultdict(list)
-
-        # get data file path and subset name
-        if os.path.isdir(dataset_name_or_path):
-            for subset_name in subset_list:
-                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
-        elif os.path.isfile(dataset_name_or_path):
-            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
-            data_file_dict[cur_subset_name] = dataset_name_or_path
-        else:
-            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
-
-        # load data from local disk
-        try:
-            for subset_name, file_path in data_file_dict.items():
-                data_item_dict[subset_name] = jsonl_to_list(file_path)
-        except Exception as e:
-            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
-
-        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
-
-        return data_dict

tests/aigc/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.