evalscope 1.0.2__py3-none-any.whl → 1.1.0__py3-none-any.whl

tests/benchmark/test_t2i.py

@@ -1,142 +0,0 @@
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import os
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, ModelTask, OutputType
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-
-class TestRun(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_general(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            datasets=[
-                'general_t2i'
-            ],
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            dataset_args={
-                'general_t2i': {
-                    'metric_list': [
-                        'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'HPSv2.1Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        # 'FGA_BLIP2Score',
-                        # 'MPS'
-                    ],
-                    'dataset_id': 'custom_eval/multimodal/t2i/example.jsonl',
-                }
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    def test_run_local_evalmuse(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model_id='T2I-Model', # 只用于展示，实际运行时不需要指定模型ID
-            model_task=ModelTask.IMAGE_GENERATION,
-            datasets=[
-                'evalmuse',  # 使用 EvalMuse benchmark
-            ],
-            dataset_args={
-                'evalmuse': {
-                    'dataset_id': 'data/example.jsonl',  # 构建的jsonl路径
-                }
-            },
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_benchmark(self):
-
-        task_cfg = TaskConfig(
-            model='stabilityai/stable-diffusion-xl-base-1.0',  # model on modelscope
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            model_args={
-                'use_safetensors': True,
-                'variant': 'fp16',
-                'torch_dtype': 'torch.float16',
-            },
-            datasets=[
-                # 'tifa160',
-                # 'genai_bench',
-                'evalmuse',
-                # 'hpdv2',
-            ],
-            dataset_args={
-                'tifa160': {
-                    'metric_list': [
-                        # 'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        'FGA_BLIP2Score',
-                    ]
-                }
-            },
-            limit=5,
-            generation_config={
-                'num_inference_steps': 50,
-                'guidance_scale': 7.5
-            },
-            # use_cache='outputs/20250427_134122',
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_benchmark_flux(self):
-
-        task_cfg = TaskConfig(
-            model='black-forest-labs/FLUX.1-dev',  # model on modelscope
-            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
-            model_args={
-                'torch_dtype': 'torch.float16',
-            },
-            datasets=[
-                # 'tifa160',
-                # 'genai_bench',
-                'evalmuse',
-                # 'hpdv2',
-            ],
-            dataset_args={
-                'tifa160': {
-                    'metric_list': [
-                        'PickScore',
-                        # 'CLIPScore',
-                        # 'HPSv2Score',
-                        # 'BLIPv2Score',
-                        # 'ImageRewardScore',
-                        # 'VQAScore',
-                        # 'FGA_BLIP2Score',
-                    ]
-                }
-            },
-            generation_config={
-                'num_inference_steps': 50,
-                'guidance_scale': 3.5
-            },
-            use_cache='outputs/20250520_112314'
-        )
-
-        run_task(task_cfg=task_cfg)

tests/benchmark/test_vlm.py

@@ -1,137 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import unittest
-
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.utils.logger import get_logger
-from tests.common import TestBenchmark
-
-logger = get_logger()
-
-
-class TestVLMBenchmark(TestBenchmark):
-    """Benchmark evaluation test cases."""
-
-    def setUp(self):
-        """Setup common test configuration."""
-        self.base_config = {
-            'model': 'qwen-vl-plus',
-            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'eval_type': EvalType.SERVICE,
-            'eval_batch_size': 5,
-            'limit': 5,
-            'generation_config': {
-                'max_tokens': 2048,
-                'temperature': 0.0,
-                'seed': 42,
-                'parallel_tool_calls': True
-            },
-            'judge_strategy': JudgeStrategy.AUTO,
-            'judge_worker_num': 5,
-            'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096,
-                }
-            },
-            'debug': True,
-        }
-
-    def test_mmmu(self):
-        dataset_args = {
-            'subset_list':[
-                'Accounting',
-                'Agriculture',
-                # 'Architecture_and_Engineering'
-            ]
-        }
-        self._run_dataset_test('mmmu', dataset_args=dataset_args)
-
-    def test_math_vista(self):
-        dataset_args = {
-            'subset_list': ['default']
-        }
-        self._run_dataset_test('math_vista', dataset_args=dataset_args)
-
-    def test_mmmu_pro(self):
-        dataset_args = {
-            'subset_list':[
-                'Accounting',
-                # 'Agriculture',
-            ],
-            'extra_params': {
-                'dataset_format': 'standard (4 options)',  # 'standard (4 options)', 'standard (10 options)', 'vision'
-            },
-        }
-        self._run_dataset_test('mmmu_pro', dataset_args=dataset_args, limit=10)
-
-    def test_qwen3_vl_collection(self):
-        dataset_args = {
-            'dataset_id': 'outputs/qwen3_vl_test.jsonl',
-            'shuffle': True,
-        }
-        self._run_dataset_test('data_collection', dataset_args, limit=100)
-
-    def test_real_world_qa(self):
-        dataset_args = {
-            'subset_list': ['default']
-        }
-        self._run_dataset_test('real_world_qa', dataset_args=dataset_args, limit=10)
-
-    def test_ai2d(self):
-        dataset_args = {
-            'subset_list': ['default']
-        }
-        self._run_dataset_test('ai2d', dataset_args=dataset_args)
-
-    def test_cc_bench(self):
-        dataset_args = {
-            'subset_list': ['cc']
-        }
-        self._run_dataset_test('cc_bench', dataset_args=dataset_args)
-
-    def test_mm_bench(self):
-        dataset_args = {
-            'subset_list': ['cn', 'en']
-        }
-        self._run_dataset_test('mm_bench', dataset_args=dataset_args)
-
-    def test_mm_star(self):
-        dataset_args = {
-            # 'subset_list': ['val']
-        }
-        self._run_dataset_test('mm_star', dataset_args=dataset_args)
-
-    def test_omni_bench(self):
-        dataset_args = {
-            'extra_params': {
-                'use_image': True, # Whether to use image input, if False, use text alternative image content.
-                'use_audio': True, # Whether to use audio input, if False, use text alternative audio content.
-            }
-        }
-        self._run_dataset_test('omni_bench', dataset_args=dataset_args, model='qwen-omni-turbo')
-
-    def test_olympiad_bench(self):
-        dataset_args = {
-            'subset_list': [
-                # 'OE_MM_maths_en_COMP',
-                # 'OE_MM_maths_zh_CEE',
-                # 'OE_MM_maths_zh_COMP',
-                # 'OE_MM_physics_en_COMP',
-                # 'OE_MM_physics_zh_CEE',
-                # 'OE_TO_maths_en_COMP',
-                # 'OE_TO_maths_zh_CEE',
-                # 'OE_TO_maths_zh_COMP',
-                # 'OE_TO_physics_en_COMP',
-                # 'OE_TO_physics_zh_CEE',
-                'TP_TO_maths_zh_CEE',
-            ]
-        }
-        self._run_dataset_test('olympiad_bench', dataset_args=dataset_args)

tests/cli/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.

tests/cli/test_all.py

@@ -1,269 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-
-import os
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.logger import get_logger
-from tests.utils import test_level_list
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-datasets=[
-    'iquiz',
-    'ifeval',
-    'mmlu',
-    'mmlu_pro',
-    'musr',
-    'process_bench',
-    'race',
-    'trivia_qa',
-    'cmmlu',
-    'humaneval',
-    'gsm8k',
-    'bbh',
-    'competition_math',
-    'math_500',
-    'aime24',
-    'gpqa_diamond',
-    'arc',
-    'ceval',
-    'hellaswag',
-    'general_mcq',
-    'general_qa',
-    'super_gpqa',
-    # 'live_code_bench',
-    'mmlu_redux',
-    'simple_qa',
-    'chinese_simpleqa',
-    'alpaca_eval',
-    'arena_hard',
-    'maritime_bench',
-    'drop',
-    'winogrande',
-    'tool_bench',
-    'frames',
-    'docmath',
-    'needle_haystack',
-    'bfcl_v3',
-    'hle',
-    'tau_bench',
-]
-
-# Reverse the datasets list to ensure the order is from most recent to oldest
-datasets.reverse()
-
-dataset_args={
-    'mmlu': {
-        'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
-        'few_shot_num': 0
-    },
-    'mmlu_pro': {
-        'subset_list': ['math', 'health'],
-        'few_shot_num': 4
-    },
-    'ceval': {
-        'subset_list': [
-            'computer_network', 'operating_system', 'computer_architecture'
-        ],
-        'few_shot_num': 0
-    },
-    'cmmlu': {
-        'subset_list': ['elementary_chinese'],
-        'few_shot_num': 0
-    },
-    'bbh': {
-        'subset_list': ['word_sorting', 'movie_recommendation'],
-    },
-    'gpqa_diamond': {
-        'few_shot_num': 0,
-    },
-    'humaneval': {
-        'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
-    },
-    'competition_math': {
-        'subset_list': ['Level 1']
-    },
-    'math_500': {
-        'subset_list': ['Level 1']
-    },
-    'process_bench': {
-        'subset_list': ['gsm8k'],
-    },
-    'musr': {
-        'subset_list': ['murder_mysteries']
-    },
-    'general_mcq': {
-        'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-        'subset_list': [
-            'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-        ],
-    },
-    'general_qa': {
-        'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-        'subset_list': [
-            'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
-            # 'test'
-        ]
-    },
-    'super_gpqa': {
-        'subset_list': ['Philosophy', 'Education'],
-        'few_shot_num': 0
-    },
-    'live_code_bench': {
-        'subset_list': ['v4_v5'],
-        'extra_params': {
-            'start_date': '2024-12-01',
-            'end_date': '2025-01-01'
-        },
-    },
-    'chinese_simpleqa': {
-        'subset_list': ['中华文化']
-    },
-    'mmlu_redux':{
-        'subset_list': ['abstract_algebra']
-    },
-    'docmath':{
-        'subset_list': ['simpshort_testmini']
-    },
-    'bfcl_v3':{
-        'subset_list': ['simple', 'multiple']
-    },
-    'hle': {
-        'subset_list': ['Math', 'Other'],
-    },
-    'tau_bench': {
-        'extra_params': {
-            'user_model': 'qwen-plus',
-            'api_key': env.get('DASHSCOPE_API_KEY'),
-            'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-        },
-        'subset_list': ['airline'],
-    },
-}
-
-class TestRun(unittest.TestCase):
-    def test_benchmarks(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=datasets,
-            dataset_args=dataset_args,
-            eval_batch_size=1,
-            limit=1,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            judge_worker_num=5,
-            judge_strategy=JudgeStrategy.AUTO,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    def test_vlm_benchmark(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-vl-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                'mmmu',
-                # 'math_vista',
-            ],
-            dataset_args={
-                'mmmu': {
-                    'subset_list': ['Accounting']
-                },
-                'math_vista': {
-                    'subset_list': ['default']
-                }
-            },
-            eval_batch_size=1,
-            limit=1,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-                'image_height': 512,
-                'image_width': 512,
-                'image_num': 2,
-            },
-            judge_worker_num=5,
-            judge_strategy=JudgeStrategy.AUTO,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
-
-    def test_ci_lite(self):
-        from evalscope.config import TaskConfig
-
-        api_key = env.get('DASHSCOPE_API_KEY')
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key=api_key,
-            eval_type=EvalType.SERVICE if api_key else EvalType.MOCK_LLM,
-            datasets=[
-                'general_mcq',
-                'iquiz',
-            ],
-            dataset_args={
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',
-                    'subset_list': [
-                        'example'
-                    ],
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',
-                    'subset_list': [
-                        'example'
-                    ]
-                }
-            },
-            eval_batch_size=1,
-            limit=1,
-            stream=True,
-            generation_config={
-                'temperature': 0,
-                'n': 1,
-                'max_tokens': 4096,
-            },
-            judge_worker_num=1,
-            judge_strategy=JudgeStrategy.AUTO,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-            }
-        )
-
-        run_task(task_cfg=task_cfg)

tests/cli/test_collection.py

@@ -1,99 +0,0 @@
-from dotenv import dotenv_values
-
-env = dotenv_values('.env')
-import json
-import os
-import unittest
-
-from evalscope.collections import CollectionSchema, DatasetInfo, WeightedSampler
-from evalscope.constants import EvalType, JudgeStrategy
-from evalscope.utils.io_utils import dump_jsonl_data
-from tests.utils import test_level_list
-
-
-class TestCollection(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_create_collection(self):
-        schema = CollectionSchema(name='math&reasoning', datasets=[
-                    CollectionSchema(name='math', datasets=[
-                        CollectionSchema(name='generation', datasets=[
-                            DatasetInfo(name='gsm8k', weight=1, task_type='math', tags=['en', 'math']),
-                        ]),
-                        CollectionSchema(name='multiple_choice', datasets=[
-                            DatasetInfo(name='cmmlu', weight=2, task_type='math', tags=['zh', 'math'], args={'subset_list': ['college_mathematics', 'high_school_mathematics']}),
-                            DatasetInfo(name='ceval', weight=3, task_type='math', tags=['zh', 'math'], args={'subset_list': ['advanced_mathematics', 'high_school_mathematics', 'discrete_mathematics', 'middle_school_mathematics']}),
-                        ]),
-                    ]),
-                    CollectionSchema(name='reasoning', datasets=[
-                        DatasetInfo(name='arc', weight=1, task_type='reasoning', tags=['en', 'reasoning']),
-                        DatasetInfo(name='ceval', weight=1, task_type='reasoning', tags=['zh', 'reasoning'], args={'subset_list': ['logic']}),
-                        DatasetInfo(name='race', weight=1, task_type='reasoning', tags=['en', 'reasoning']),
-                    ]),
-                ])
-        print(schema.to_dict())
-        print(schema.flatten())
-        schema.dump_json('outputs/schema_test.json')
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_generate_data(self):
-        schema = CollectionSchema.from_dict(json.load(open('outputs/schema_test.json', 'r')))
-        print(schema.to_dict())
-        mixed_data = WeightedSampler(schema).sample(100)
-        dump_jsonl_data(mixed_data, 'outputs/mixed_data_test.jsonl')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_evaluate_collection(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key=env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=['data_collection'],
-            dataset_args={
-                'data_collection': {
-                    # 'local_path': 'outputs/test_mix.jsonl'
-                    'local_path': 'outputs/mixed_data_test.jsonl',
-                    'shuffle': True,
-                }
-            },
-            eval_batch_size=5,
-            generation_config = {
-                'max_tokens': 10000,
-                'temperature': 0.0,
-            },
-            limit=10,
-            # use_cache='outputs/20250822_161804'
-        )
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_evaluate_collection_with_judge(self):
-        from evalscope import TaskConfig, run_task
-
-        task_cfg = TaskConfig(
-            model='qwen2.5-7b-instruct',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= os.getenv('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
-            limit=5,
-            judge_strategy=JudgeStrategy.AUTO,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': os.getenv('DASHSCOPE_API_KEY'),
-            },
-            analysis_report=True,
-            ignore_errors=True,
-            # use_cache='outputs/20250522_204520'
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)