evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

tests/cli/test_collection.py

@@ -1,96 +0,0 @@
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-import json
-import os
-import unittest
-
-from evalscope.collections import CollectionSchema, DatasetInfo, WeightedSampler
-from evalscope.constants import EvalType, JudgeStrategy
-from evalscope.utils.io_utils import dump_jsonl_data
-from tests.utils import test_level_list
-
-
-class TestCollection(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_create_collection(self):
-        schema = CollectionSchema(name='math&reasoning', datasets=[
-                    CollectionSchema(name='math', datasets=[
-                        CollectionSchema(name='generation', datasets=[
-                            DatasetInfo(name='gsm8k', weight=1, task_type='math', tags=['en', 'math']),
-                        ]),
-                        CollectionSchema(name='multiple_choice', datasets=[
-                            DatasetInfo(name='cmmlu', weight=2, task_type='math', tags=['zh', 'math'], args={'subset_list': ['college_mathematics', 'high_school_mathematics']}),
-                            DatasetInfo(name='ceval', weight=3, task_type='math', tags=['zh', 'math'], args={'subset_list': ['advanced_mathematics', 'high_school_mathematics', 'discrete_mathematics', 'middle_school_mathematics']}),
-                        ]),
-                    ]),
-                    CollectionSchema(name='reasoning', datasets=[
-                        DatasetInfo(name='arc', weight=1, task_type='reasoning', tags=['en', 'reasoning']),
-                        DatasetInfo(name='ceval', weight=1, task_type='reasoning', tags=['zh', 'reasoning'], args={'subset_list': ['logic']}),
-                        DatasetInfo(name='race', weight=1, task_type='reasoning', tags=['en', 'reasoning']),
-                    ]),
-                ])
-        print(schema.to_dict())
-        print(schema.flatten())
-        schema.dump_json('outputs/schema_test.json')
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_generate_data(self):
-        schema = CollectionSchema.from_dict(json.load(open('outputs/schema_test.json', 'r')))
-        print(schema.to_dict())
-        mixed_data = WeightedSampler(schema).sample(100)
-        dump_jsonl_data(mixed_data, 'outputs/mixed_data_test.jsonl')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_evaluate_collection(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
-            eval_batch_size=5,
-            generation_config = {
-                'max_tokens': 10000,
-                'temperature': 0.0,
-            },
-            limit=50,
-            # use_cache='outputs/20250822_161804'
-        )
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_evaluate_collection_with_judge(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model='qwen2.5-7b-instruct',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= os.getenv('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
-            limit=5,
-            judge_strategy=JudgeStrategy.AUTO,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': os.getenv('DASHSCOPE_API_KEY'),
-            },
-            analysis_report=True,
-            ignore_errors=True,
-            # use_cache='outputs/20250522_204520'
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)

tests/cli/test_custom.py

@@ -1,268 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-
-import os
-import subprocess
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-
-class TestRunCustom(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom_task(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='Qwen/Qwen3-0.6B',
-            datasets=[
-                'general_mcq',
-                'general_qa'
-            ],
-            dataset_args={
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ]
-                }
-            },
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_local_dataset(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                # 'mmlu',
-                # 'race',
-                'trivia_qa',
-                # 'cmmlu',
-                # 'humaneval',
-                # 'gsm8k',
-                # 'bbh',
-                # 'competition_math',
-                # 'arc',
-                # 'ceval',
-            ],
-            dataset_args={
-                'mmlu': {
-                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
-                    'few_shot_num': 0,
-                    'dataset_id': 'data/data/mmlu',
-                },
-                'ceval': {
-                    'subset_list': [
-                        'computer_network', 'operating_system', 'computer_architecture'
-                    ],
-                    'few_shot_num': 0,
-                    'dataset_id': 'data/data/ceval',
-                },
-                'cmmlu': {
-                    'subset_list': ['elementary_chinese'],
-                    'dataset_id': 'data/data/cmmlu',
-                    'few_shot_num': 0
-                },
-                'bbh': {
-                    'subset_list': ['word_sorting', 'movie_recommendation'],
-                },
-                'humaneval': {
-                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
-                },
-                'trivia_qa': {
-                    'dataset_id': 'data/data/trivia_qa',
-                },
-            },
-            eval_batch_size=10,
-            limit=5,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_no_answer(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen2.5-7b-instruct',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                'general_qa',
-            ],
-            dataset_args={
-                'general_qa': {
-                    'dataset_id': 'custom_eval/text/qa',
-                    'subset_list': [
-                        'arena',
-                        # 'example'
-                    ],
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096
-                },
-                'score_type': 'numeric',
-                'prompt_template': """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response.
-Begin your evaluation by providing a short explanation. Be as objective as possible.
-After providing your explanation, you must rate the response on a scale of 0 (worst) to 100 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\"
-
-[Question]
-{question}
-
-[Response]
-{pred}
-"""
-            },
-            judge_worker_num=5,
-            judge_strategy=JudgeStrategy.LLM,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_with_answer(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                'general_qa',
-            ],
-            dataset_args={
-                'general_qa': {
-                    'dataset_id': 'custom_eval/text/qa',
-                    'subset_list': [
-                        'example'
-                    ],
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096
-                },
-                'score_type': 'pattern',
-            },
-            judge_worker_num=1,
-            judge_strategy=JudgeStrategy.LLM_RECALL,
-            use_cache='outputs/20250818_170420'
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general_arena(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model_id='Arena',
-            datasets=[
-                'general_arena',
-            ],
-            dataset_args={
-                'general_arena': {
-                    'extra_params':{
-                        'models':[
-                            {
-                                'name': 'qwen2.5-7b',
-                                'report_path': 'outputs/20250819_165034/reports/qwen2.5-7b-instruct'
-                            },
-                            {
-                                'name': 'qwen2.5-72b',
-                                'report_path': 'outputs/20250819_164926/reports/qwen2.5-72b-instruct'
-                            }
-                        ],
-                        'baseline': 'qwen2.5-72b'
-                    }
-                }
-            },
-            eval_batch_size=10,
-            limit=10,
-            debug=True,
-            stream=True,
-            ignore_errors=False,
-            judge_model_args={
-                'model_id': 'qwen-plus',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 8000
-                },
-            },
-            judge_worker_num=5,
-            # use_cache='outputs/20250819_173546'
-        )
-
-        run_task(task_cfg=task_cfg)

tests/perf/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/perf/test_perf.py

@@ -1,176 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
-import unittest
-
-from evalscope.perf.main import run_perf_benchmark
-from tests.utils import test_level_list
-
-
-class TestPerf(unittest.TestCase):
-
-    def setUp(self) -> None:
-        pass
-
-    def tearDown(self) -> None:
-        pass
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/chat/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            # 'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_stream(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8801/v1/chat/completions',
-            'parallel': 1,
-            'model': 'Qwen2.5-0.5B-Instruct',
-            'number': 15,
-            'api': 'openai',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_speed_benchmark(self):
-        task_cfg = {
-            'url': 'http://127.0.0.1:8001/v1/completions',
-            'parallel': 1,
-            'model': 'qwen2.5',
-            'api': 'openai',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_stream(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'number': 5,
-            'api': 'local',
-            'dataset': 'openqa',
-            'stream': True,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_speed_benchmark(self):
-        task_cfg = {
-            'parallel': 1,
-            'model': 'Qwen/Qwen2.5-0.5B-Instruct',
-            'api': 'local_vllm',
-            'dataset': 'speed_benchmark',
-            'min_tokens': 2048,
-            'max_tokens': 2048,
-            'debug': True,
-        }
-        run_perf_benchmark(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_local_random(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=20,
-            model='Qwen3-1.7B',
-            url='http://127.0.0.1:8801/v1/completions',
-            api='openai',
-            dataset='random',
-            min_tokens=1024,
-            max_tokens=1024,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            number=20,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_multi_parallel(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen2.5-7b-instruct',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='random',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=1024,
-            max_prompt_length=1024,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_perf_random_vl(self):
-        from evalscope.perf.arguments import Arguments
-        task_cfg = Arguments(
-            parallel=[1, 2],
-            number=[2, 4],
-            model='qwen-vl-max',
-            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            api='openai',
-            dataset='kontext_bench',
-            min_tokens=100,
-            max_tokens=100,
-            prefix_length=0,
-            min_prompt_length=100,
-            max_prompt_length=100,
-            image_height=512,
-            image_width=512,
-            image_num=2,
-            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
-            seed=None,
-            extra_args={'ignore_eos': True}
-        )
-        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
-        print(metrics_result)
-        print(percentile_result)
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)

tests/rag/test_clip_benchmark.py

@@ -1,90 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import os
-
-# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
-import subprocess
-import unittest
-
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-logger = get_logger()
-
-
-class TestCLIPBenchmark(unittest.TestCase):
-
-    def setUp(self) -> None:
-        self._check_env('webdataset')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': [
-                        'muge',
-                        'mnist',
-                        'flickr8k'
-                    ],
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'cache_dir': 'cache',
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom(self):
-        task_cfg = {
-            'eval_backend': 'RAGEval',
-            'eval_config': {
-                'tool': 'clip_benchmark',
-                'eval': {
-                    'models': [
-                        {
-                            'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
-                        }
-                    ],
-                    'dataset_name': ['custom'],
-                    'data_dir': 'custom_eval/multimodal/text-image-retrieval',
-                    'split': 'test',
-                    'batch_size': 128,
-                    'num_workers': 1,
-                    'verbose': True,
-                    'skip_existing': False,
-                    'limit': 1000,
-                },
-            },
-        }
-
-        run_task(task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main(buffer=False)