evalscope 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl

tests/cli/test_custom.py

@@ -1,268 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-
-import os
-import subprocess
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-
-class TestRunCustom(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom_task(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='Qwen/Qwen3-0.6B',
-            datasets=[
-                'general_mcq',
-                'general_qa'
-            ],
-            dataset_args={
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ]
-                }
-            },
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_local_dataset(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                # 'mmlu',
-                # 'race',
-                'trivia_qa',
-                # 'cmmlu',
-                # 'humaneval',
-                # 'gsm8k',
-                # 'bbh',
-                # 'competition_math',
-                # 'arc',
-                # 'ceval',
-            ],
-            dataset_args={
-                'mmlu': {
-                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
-                    'few_shot_num': 0,
-                    'dataset_id': 'data/data/mmlu',
-                },
-                'ceval': {
-                    'subset_list': [
-                        'computer_network', 'operating_system', 'computer_architecture'
-                    ],
-                    'few_shot_num': 0,
-                    'dataset_id': 'data/data/ceval',
-                },
-                'cmmlu': {
-                    'subset_list': ['elementary_chinese'],
-                    'dataset_id': 'data/data/cmmlu',
-                    'few_shot_num': 0
-                },
-                'bbh': {
-                    'subset_list': ['word_sorting', 'movie_recommendation'],
-                },
-                'humaneval': {
-                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
-                },
-                'trivia_qa': {
-                    'dataset_id': 'data/data/trivia_qa',
-                },
-            },
-            eval_batch_size=10,
-            limit=5,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_no_answer(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen2.5-7b-instruct',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                'general_qa',
-            ],
-            dataset_args={
-                'general_qa': {
-                    'dataset_id': 'custom_eval/text/qa',
-                    'subset_list': [
-                        'arena',
-                        # 'example'
-                    ],
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096
-                },
-                'score_type': 'numeric',
-                'prompt_template': """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response.
-Begin your evaluation by providing a short explanation. Be as objective as possible.
-After providing your explanation, you must rate the response on a scale of 0 (worst) to 100 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\"
-
-[Question]
-{question}
-
-[Response]
-{pred}
-"""
-            },
-            judge_worker_num=5,
-            judge_strategy=JudgeStrategy.LLM,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_with_answer(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                'general_qa',
-            ],
-            dataset_args={
-                'general_qa': {
-                    'dataset_id': 'custom_eval/text/qa',
-                    'subset_list': [
-                        'example'
-                    ],
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096
-                },
-                'score_type': 'pattern',
-            },
-            judge_worker_num=1,
-            judge_strategy=JudgeStrategy.LLM_RECALL,
-            use_cache='outputs/20250818_170420'
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_arena(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model_id='Arena',
-            datasets=[
-                'general_arena',
-            ],
-            dataset_args={
-                'general_arena': {
-                    'extra_params':{
-                        'models':[
-                            {
-                                'name': 'qwen2.5-7b',
-                                'report_path': 'outputs/20250819_165034/reports/qwen2.5-7b-instruct'
-                            },
-                            {
-                                'name': 'qwen2.5-72b',
-                                'report_path': 'outputs/20250819_164926/reports/qwen2.5-72b-instruct'
-                            }
-                        ],
-                        'baseline': 'qwen2.5-72b'
-                    }
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen-plus',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 8000
-                },
-            },
-            judge_worker_num=5,
-            # use_cache='outputs/20250819_173546'
-        )
-
-        run_task(task_cfg=task_cfg)

tests/cli/test_reasoning.py

@@ -1,81 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import unittest
-from unittest import TestCase
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestReasoning(TestCase):
-    """Benchmark evaluation test cases."""
-
-    def setUp(self):
-        """Setup common test configuration."""
-        self.base_config = {
-            'model': 'Qwen3-0.6B',
-            'api_url': 'http://0.0.0.0:8801/v1',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'eval_type': EvalType.SERVICE,
-            'eval_batch_size': 5,
-            'limit': 5,
-            'generation_config': {
-                'max_tokens': 4096,
-                'temperature': 0.0,
-                'seed': 42,
-                'parallel_tool_calls': True,
-                'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式
-            },
-            'judge_strategy': JudgeStrategy.AUTO,
-            'judge_worker_num': 5,
-            'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096,
-                }
-            },
-            'debug': True,
-        }
-
-    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
-        """Helper method to run test for a specific dataset."""
-        config = self.base_config.copy()
-        config['datasets'] = [dataset_name]
-
-        if use_mock:
-            config['eval_type'] = EvalType.MOCK_LLM
-
-        # 应用配置覆盖
-        config.update(config_overrides)
-
-        if dataset_args:
-            config['dataset_args'] = {dataset_name: dataset_args}
-
-        task_cfg = TaskConfig(**config)
-        run_task(task_cfg=task_cfg)
-
-    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
-        """Helper method to test dataset loading."""
-
-        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
-
-    # Math & Reasoning datasets
-    def test_gsm8k(self):
-        """Test GSM8K math reasoning dataset."""
-        self._run_dataset_test('gsm8k')
-
-
-if __name__ == '__main__':
-    # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k
-    # Run all tests: python -m unittest test_eval.TestBenchmark
-    unittest.main()

tests/common.py

@@ -1,73 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import unittest
-from unittest import TestCase
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class TestBenchmark(TestCase):
-    """Benchmark evaluation test cases."""
-
-    def setUp(self):
-        """Setup common test configuration."""
-        self.base_config = {
-            'model': 'qwen-plus',
-            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'eval_type': EvalType.SERVICE,
-            'eval_batch_size': 5,
-            'limit': 5,
-            'generation_config': {
-                'max_tokens': 4096,
-                'temperature': 0.0,
-                'seed': 42,
-                'parallel_tool_calls': True
-            },
-            'judge_strategy': JudgeStrategy.AUTO,
-            'judge_worker_num': 5,
-            'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096,
-                }
-            },
-            'debug': True,
-        }
-
-    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
-        """Helper method to run test for a specific dataset."""
-        config = self.base_config.copy()
-        config['datasets'] = [dataset_name]
-
-        if not env.get('DASHSCOPE_API_KEY'):
-            use_mock = True
-            logger.warning('DASHSCOPE_API_KEY is not set. Using mock evaluation.')
-
-        if use_mock:
-            config['eval_type'] = EvalType.MOCK_LLM
-
-        # 应用配置覆盖
-        config.update(config_overrides)
-
-        if dataset_args:
-            config['dataset_args'] = {dataset_name: dataset_args}
-
-        task_cfg = TaskConfig(**config)
-        run_task(task_cfg=task_cfg)
-
-    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
-        """Helper method to test dataset loading."""
-
-        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)

tests/perf/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/perf/test_perf.py

@@ -1,206 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-import unittest
-
-from evalscope.perf.main import run_perf_benchmark
-from tests.utils import test_level_list
-
-
-class TestPerf(unittest.TestCase):
-
-    def setUp(self) -> None:
-        pass
-
-    def tearDown(self) -> None:
-        pass
-
-
-    def test_run_perf(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/chat/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            # 'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_stream(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8801/v1/chat/completions',
-            'parallel': 1,
-            'model': 'Qwen2.5-0.5B-Instruct',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_speed_benchmark(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'api': 'openai',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_local(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_local_stream(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_local_speed_benchmark(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'api': 'local_vllm',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-
-    def test_run_perf_local_random(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=20,
-            model='Qwen3-1.7B',
-            url='http://127.0.0.1:8801/v1/completions',
-            api='openai',
-            dataset='random',
-            min_tokens=1024,
-            max_tokens=1024,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            number=20,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-    def test_run_completion_endpoint(self):
-        if not env.get('DASHSCOPE_API_KEY'):
-            self.skipTest('DASHSCOPE_API_KEY is not set.')
-            return
-
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen2.5-coder-7b-instruct',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='random',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            stream=False,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-
-    def test_run_perf_multi_parallel(self):
-        if not env.get('DASHSCOPE_API_KEY'):
-            self.skipTest('DASHSCOPE_API_KEY is not set.')
-            return
-
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen-plus',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='random',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-
-    def test_run_perf_random_vl(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen-vl-max',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='random_vl',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=100,
-            max_prompt_length=100,
-            image_height=512,
-            image_width=512,
-            image_num=2,
-            tokenizer_path='Qwen/Qwen2.5-VL-7B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)