evalscope 1.0.1__py3-none-any.whl → 1.1.0__py3-none-any.whl

tests/cli/test_reasoning.py

@@ -1,81 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import unittest
-from unittest import TestCase
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestReasoning(TestCase):
-    """Benchmark evaluation test cases."""
-
-    def setUp(self):
-        """Setup common test configuration."""
-        self.base_config = {
-            'model': 'Qwen3-0.6B',
-            'api_url': 'http://0.0.0.0:8801/v1',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'eval_type': EvalType.SERVICE,
-            'eval_batch_size': 5,
-            'limit': 5,
-            'generation_config': {
-                'max_tokens': 4096,
-                'temperature': 0.0,
-                'seed': 42,
-                'parallel_tool_calls': True,
-                'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式
-            },
-            'judge_strategy': JudgeStrategy.AUTO,
-            'judge_worker_num': 5,
-            'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096,
-                }
-            },
-            'debug': True,
-        }
-
-    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
-        """Helper method to run test for a specific dataset."""
-        config = self.base_config.copy()
-        config['datasets'] = [dataset_name]
-
-        if use_mock:
-            config['eval_type'] = EvalType.MOCK_LLM
-
-        # 应用配置覆盖
-        config.update(config_overrides)
-
-        if dataset_args:
-            config['dataset_args'] = {dataset_name: dataset_args}
-
-        task_cfg = TaskConfig(**config)
-        run_task(task_cfg=task_cfg)
-
-    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
-        """Helper method to test dataset loading."""
-
-        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
-
-    # Math & Reasoning datasets
-    def test_gsm8k(self):
-        """Test GSM8K math reasoning dataset."""
-        self._run_dataset_test('gsm8k')
-
-
-if __name__ == '__main__':
-    # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k
-    # Run all tests: python -m unittest test_eval.TestBenchmark
-    unittest.main()

tests/common.py

@@ -1,73 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import unittest
-from unittest import TestCase
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestBenchmark(TestCase):
-    """Benchmark evaluation test cases."""
-
-    def setUp(self):
-        """Setup common test configuration."""
-        self.base_config = {
-            'model': 'qwen-plus',
-            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'eval_type': EvalType.SERVICE,
-            'eval_batch_size': 5,
-            'limit': 5,
-            'generation_config': {
-                'max_tokens': 4096,
-                'temperature': 0.0,
-                'seed': 42,
-                'parallel_tool_calls': True
-            },
-            'judge_strategy': JudgeStrategy.AUTO,
-            'judge_worker_num': 5,
-            'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096,
-                }
-            },
-            'debug': True,
-        }
-
-    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
-        """Helper method to run test for a specific dataset."""
-        config = self.base_config.copy()
-        config['datasets'] = [dataset_name]
-
-        if not env.get('DASHSCOPE_API_KEY'):
-            use_mock = True
-            logger.warning('DASHSCOPE_API_KEY is not set. Using mock evaluation.')
-
-        if use_mock:
-            config['eval_type'] = EvalType.MOCK_LLM
-
-        # 应用配置覆盖
-        config.update(config_overrides)
-
-        if dataset_args:
-            config['dataset_args'] = {dataset_name: dataset_args}
-
-        task_cfg = TaskConfig(**config)
-        run_task(task_cfg=task_cfg)
-
-    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
-        """Helper method to test dataset loading."""
-
-        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)

tests/perf/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/perf/test_perf.py

@@ -1,178 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-import unittest
-
-from evalscope.perf.main import run_perf_benchmark
-from tests.utils import test_level_list
-
-
-class TestPerf(unittest.TestCase):
-
-    def setUp(self) -> None:
-        pass
-
-    def tearDown(self) -> None:
-        pass
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/chat/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            # 'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_stream(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8801/v1/chat/completions',
-            'parallel': 1,
-            'model': 'Qwen2.5-0.5B-Instruct',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_speed_benchmark(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'api': 'openai',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_stream(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_speed_benchmark(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'api': 'local_vllm',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_random(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=20,
-            model='Qwen3-1.7B',
-            url='http://127.0.0.1:8801/v1/completions',
-            api='openai',
-            dataset='random',
-            min_tokens=1024,
-            max_tokens=1024,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            number=20,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_multi_parallel(self):
-        if not env.get('DASHSCOPE_API_KEY'):
-            self.skipTest('DASHSCOPE_API_KEY is not set.')
-            return
-
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen2.5-7b-instruct',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='random',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_random_vl(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen-vl-max',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='kontext_bench',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=100,
-            max_prompt_length=100,
-            image_height=512,
-            image_width=512,
-            image_num=2,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/rag/test_clip_benchmark.py

@@ -1,87 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-import subprocess
-import unittest
-
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger()
-
-
-class TestCLIPBenchmark(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('webdataset')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': [
-                        'muge',
-                        'mnist',
-                        'flickr8k'
-                    ],
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'cache_dir': 'cache',
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': ['custom'],
-                    'data_dir': 'custom_eval/multimodal/text-image-retrieval',
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/rag/test_mteb.py

@@ -1,213 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import unittest
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestMTEB(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('mteb')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_one_stage_mteb(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'pooling_mode': None,  # load from model config
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 128,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': [
-                        'TNews',
-                        'CLSClusteringS2S',
-                        'T2Reranking',
-                        'T2Retrieval',
-                        'ATEC',
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 500,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_one_stage_api(self):
-        from evalscope import TaskConfig
-        task_cfg = TaskConfig(
-            eval_backend='RAGEval',
-            eval_config={
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': [
-                        'T2Retrieval',
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                },
-            },
-        )
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_two_stage_mteb(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'is_cross_encoder': False,
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 64,
-                        },
-                    },
-                    {
-                        'model_name_or_path': 'BAAI/bge-reranker-v2-m3',
-                        'is_cross_encoder': True,
-                        'max_seq_length': 512,
-                        'prompt': '为这个问题生成一个检索用的表示',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 32,
-                        },
-                    },
-                ],
-                'eval': {
-                    'tasks': [
-                        'MedicalRetrieval',
-                        'T2Retrieval'
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                    'top_k': 10,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_two_stage_api(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                    {
-                        'model_name': 'text-embedding-v3',
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
-                        'dimensions': 1024,
-                        'encode_kwargs': {
-                            'batch_size': 10,
-                        },
-                    },
-                ],
-                'eval': {
-                    'tasks': [
-                        'MedicalRetrieval',
-                        # 'T2Retrieval'
-                    ],
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 10,
-                    'top_k': 10,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'MTEB',
-                'model': [
-                    {
-                        'model_name_or_path': 'AI-ModelScope/m3e-base',
-                        'pooling_mode': None,  # load from model config
-                        'max_seq_length': 512,
-                        'prompt': '',
-                        'model_kwargs': {'torch_dtype': 'auto'},
-                        'encode_kwargs': {
-                            'batch_size': 128,
-                        },
-                    }
-                ],
-                'eval': {
-                    'tasks': ['CustomRetrieval'],
-                    'dataset_path': 'custom_eval/text/retrieval',
-                    'verbosity': 2,
-                    'overwrite_results': True,
-                    'limits': 500,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)