evalscope 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

tests/benchmark/test_eval.py

@@ -4,17 +4,15 @@ from dotenv import dotenv_values
 env = dotenv_values('.env')
 
 import unittest
-from unittest import TestCase
 
-from evalscope.config import TaskConfig
 from evalscope.constants import EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
 from evalscope.utils.logger import get_logger
+from tests.common import TestBenchmark
 
 logger = get_logger()
 
 
-class TestBenchmark(TestCase):
+class TestNativeBenchmark(TestBenchmark):
     """Benchmark evaluation test cases."""
 
     def setUp(self):
@@ -35,38 +33,18 @@ class TestBenchmark(TestCase):
             'judge_strategy': JudgeStrategy.AUTO,
             'judge_worker_num': 5,
             'judge_model_args': {
-                'model_id': 'qwen2.5-72b-instruct',
+                'model_id': 'qwen3-235b-a22b',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'generation_config': {
                     'temperature': 0.0,
                     'max_tokens': 4096,
+                    'extra_body': {'enable_thinking': False}
                 }
             },
             'debug': True,
         }
 
-    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
-        """Helper method to run test for a specific dataset."""
-        config = self.base_config.copy()
-        config['datasets'] = [dataset_name]
-
-        if use_mock:
-            config['eval_type'] = EvalType.MOCK_LLM
-
-        # 应用配置覆盖
-        config.update(config_overrides)
-
-        if dataset_args:
-            config['dataset_args'] = {dataset_name: dataset_args}
-
-        task_cfg = TaskConfig(**config)
-        run_task(task_cfg=task_cfg)
-
-    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
-        """Helper method to test dataset loading."""
-
-        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
 
     # Math & Reasoning datasets
     def test_gsm8k(self):
@@ -84,10 +62,18 @@ class TestBenchmark(TestCase):
         """Test MMLU reasoning dataset."""
         dataset_args = {
             'few_shot_num': 0,
-            # 'subset_list': ['abstract_algebra', 'computer_security']
+            'subset_list': ['abstract_algebra', 'computer_security']
         }
         self._run_dataset_test('mmlu', use_mock=True, dataset_args=dataset_args)
 
+    def test_mmlu_reasoning(self):
+        """Test MMLU reasoning dataset."""
+        dataset_args = {
+            'few_shot_num': 0,
+            'subset_list': ['abstract_algebra', 'computer_security']
+        }
+        self._run_dataset_test('mmlu', dataset_args=dataset_args, model='qwen3-0.6b', stream=True)
+
     def test_mmlu_pro(self):
         """Test MMLU-Pro reasoning dataset."""
         dataset_args = {
@@ -116,7 +102,11 @@ class TestBenchmark(TestCase):
     def test_math_500(self):
         """Test MATH 500 dataset."""
         # self._run_dataset_load_test('math_500')
-        self._run_dataset_test('math_500')
+        dataset_args = {
+            'subset_list': ['Level 1', 'Level 2'],
+            'few_shot_num': 0,
+        }
+        self._run_dataset_test('math_500', dataset_args=dataset_args)
 
     def test_aime24(self):
         """Test AIME 2024 dataset."""
@@ -222,6 +212,7 @@ class TestBenchmark(TestCase):
     def test_bbh(self):
         dataset_args = {
             'subset_list': ['temporal_sequences', 'navigate'],
+            'few_shot_num': 0,
         }
         self._run_dataset_test('bbh', dataset_args=dataset_args)
 
@@ -336,20 +327,21 @@ class TestBenchmark(TestCase):
     def test_humaneval(self):
         """Test HumanEval dataset."""
         dataset_args = {
-            'metric_list': ['Pass@1', 'Pass@2', 'Pass@5']
+            'metric_list': ['Pass@1']
         }
-        self._run_dataset_test('humaneval', dataset_args, repeats=5)
+        self._run_dataset_test('humaneval', dataset_args)
 
     def test_live_code_bench(self):
         """Test LiveCodeBench dataset."""
         dataset_args = {
-            'subset_list': ['v6'],
+            'subset_list': ['v5'],
+            'review_timeout': 6,
             'extra_params': {
                 'start_date': '2024-08-01',
                 'end_date': '2025-02-28'
             },
         }
-        self._run_dataset_test('live_code_bench', dataset_args, judge_worker_num=1)
+        self._run_dataset_test('live_code_bench', dataset_args, limit=20, use_cache='outputs/20250918_200232', rerun_review=True)
 
     def test_tool_bench(self):
         """Test ToolBench dataset."""
@@ -358,27 +350,78 @@ class TestBenchmark(TestCase):
     def test_bfcl(self):
         """Test BFCL dataset."""
         dataset_args = {
-            'subset_list': ['simple', 'live_multiple', 'multi_turn_base'],
+            'subset_list': [
+                # 'simple',
+                # 'live_multiple',
+                # 'multi_turn_base',
+                'multi_turn_miss_func'
+            ],
             'extra_params': {
                 'is_fc_model': True,
                 'underscore_to_dot': True
             }
         }
-        self._run_dataset_test('bfcl_v3', dataset_args)
+        self._run_dataset_test('bfcl_v3', dataset_args, model='qwen-plus', limit=30, eval_batch_size=5)
 
     def test_tau_bench(self):
         dataset_args = {
+            'subset_list': [
+                'airline',
+                'retail'
+            ],
             'extra_params': {
                 'user_model': 'qwen-plus',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'generation_config': {
-                    'temperature': 0.7,
-                    'max_new_tokens': 1024
+                    'temperature': 0.0,
+                    'max_tokens': 12000,
+                    'stream': True
                 }
             }
         }
-        self._run_dataset_test('tau_bench', dataset_args, limit=1)
+        self._run_dataset_test('tau_bench', dataset_args, limit=5, model='qwq-plus', stream=True)
+
+    def test_r1_collection(self):
+        dataset_args = {
+            'dataset_id': 'evalscope/R1-Distill-Math-Test-v2'
+        }
+        self._run_dataset_test('data_collection', dataset_args)
+
+    def test_qwen3_collection(self):
+        dataset_args = {
+            'dataset_id': 'evalscope/Qwen3-Test-Collection'
+        }
+        self._run_dataset_test('data_collection', dataset_args)
+
+    def test_multi_if(self):
+        dataset_args = {
+            'subset_list': ['English', 'Chinese'],
+            'few_shot_num': 0,
+        }
+        self._run_dataset_test('multi_if', dataset_args, limit=5)
+
+    def test_healthbench(self):
+        dataset_args = {
+            'subset_list': ['health_data_tasks'],
+            'extra_params': {
+                'version': 'Hard'
+            }
+        }
+        self._run_dataset_test('health_bench', dataset_args, limit=5)
+
+
+    def test_amc(self):
+        dataset_args = {
+            'subset_list': ['amc22'],
+        }
+        self._run_dataset_test('amc', dataset_args)
+
+    def test_minerva_math(self):
+        dataset_args = {
+            'subset_list': ['default'],
+        }
+        self._run_dataset_test('minerva_math', dataset_args)
 
 if __name__ == '__main__':
     # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k

tests/benchmark/test_image_edit.py

@@ -0,0 +1,65 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+
+from evalscope.constants import EvalType, JudgeStrategy, ModelTask
+from evalscope.utils.logger import get_logger
+from tests.common import TestBenchmark
+
+logger = get_logger()
+
+
+class TestImageEditBenchmark(TestBenchmark):
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'Qwen/Qwen-Image-Edit',
+            'model_args':{
+                'precision': 'bfloat16',
+                'device_map': 'cuda:2'
+            },
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'model_task': ModelTask.IMAGE_GENERATION,
+            'eval_type': EvalType.IMAGE_EDITING,
+            'eval_batch_size': 1,
+            'limit': 5,
+            'generation_config': {
+                'true_cfg_scale': 4.0,
+                'num_inference_steps': 50,
+                'negative_prompt': ' ',
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-vl-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def test_gedit(self):
+        """Test GEdit dataset."""
+        dataset_args = {
+            'extra_params':{
+                'language': 'cn',
+            }
+        }
+        self._run_dataset_test('gedit', dataset_args=dataset_args, use_cache='outputs/20250829_150058')
+
+    def test_gedit_local(self):
+        dataset_args = {
+            'extra_params':{
+                'language': 'cn',
+                'local_file': 'outputs/example_edit.jsonl',
+            }
+        }
+        self._run_dataset_test('gedit', dataset_args=dataset_args, model=None, model_id='offline_model')

tests/benchmark/test_sandbox.py

@@ -0,0 +1,81 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.utils.logger import get_logger
+from tests.common import TestBenchmark
+
+logger = get_logger()
+
+
+class TestCodeBenchmark(TestBenchmark):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'qwen-plus',
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'use_sandbox': True,
+            'sandbox_type': 'docker',
+            'debug': True,
+        }
+
+    def test_humaneval(self):
+        """Test Humaneval dataset."""
+        self._run_dataset_test('humaneval', limit=5)
+
+    def test_humaneval_remote_sandbox(self):
+        """Test Humaneval dataset with remote sandbox manager."""
+        sandbox_manager_config = {'base_url': 'http://localhost:8000'}
+        self._run_dataset_test('humaneval', limit=5, sandbox_manager_config=sandbox_manager_config)
+
+    def test_live_code_bench(self):
+        """Test Live Code Bench dataset."""
+        dataset_args = {
+            'subset_list': ['v5'],
+            'review_timeout': 6,
+            'extra_params': {
+                'start_date': '2024-08-01',
+                'end_date': '2025-02-28'
+            },
+        }
+        self._run_dataset_test('live_code_bench', limit=5, dataset_args=dataset_args, use_cache='outputs/20250918_200232', rerun_review=True)
+
+    def test_live_code_bench_remote_sandbox(self):
+        """Test Live Code Bench dataset."""
+        dataset_args = {
+            'subset_list': ['v5'],
+            'review_timeout': 6,
+            'extra_params': {
+                'start_date': '2024-08-01',
+                'end_date': '2025-02-28'
+            },
+        }
+        sandbox_manager_config = {'base_url': 'http://localhost:8000'}
+        self._run_dataset_test('live_code_bench', limit=20, dataset_args=dataset_args, sandbox_manager_config=sandbox_manager_config, use_cache='outputs/20250918_200232_2', rerun_review=True)

tests/benchmark/test_vlm.py

@@ -0,0 +1,137 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.utils.logger import get_logger
+from tests.common import TestBenchmark
+
+logger = get_logger()
+
+
+class TestVLMBenchmark(TestBenchmark):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'qwen-vl-plus',
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 2048,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def test_mmmu(self):
+        dataset_args = {
+            'subset_list':[
+                'Accounting',
+                'Agriculture',
+                # 'Architecture_and_Engineering'
+            ]
+        }
+        self._run_dataset_test('mmmu', dataset_args=dataset_args)
+
+    def test_math_vista(self):
+        dataset_args = {
+            'subset_list': ['default']
+        }
+        self._run_dataset_test('math_vista', dataset_args=dataset_args)
+
+    def test_mmmu_pro(self):
+        dataset_args = {
+            'subset_list':[
+                'Accounting',
+                # 'Agriculture',
+            ],
+            'extra_params': {
+                'dataset_format': 'standard (4 options)',  # 'standard (4 options)', 'standard (10 options)', 'vision'
+            },
+        }
+        self._run_dataset_test('mmmu_pro', dataset_args=dataset_args, limit=10)
+
+    def test_qwen3_vl_collection(self):
+        dataset_args = {
+            'dataset_id': 'outputs/qwen3_vl_test.jsonl',
+            'shuffle': True,
+        }
+        self._run_dataset_test('data_collection', dataset_args, limit=100)
+
+    def test_real_world_qa(self):
+        dataset_args = {
+            'subset_list': ['default']
+        }
+        self._run_dataset_test('real_world_qa', dataset_args=dataset_args, limit=10)
+
+    def test_ai2d(self):
+        dataset_args = {
+            'subset_list': ['default']
+        }
+        self._run_dataset_test('ai2d', dataset_args=dataset_args)
+
+    def test_cc_bench(self):
+        dataset_args = {
+            'subset_list': ['cc']
+        }
+        self._run_dataset_test('cc_bench', dataset_args=dataset_args)
+
+    def test_mm_bench(self):
+        dataset_args = {
+            'subset_list': ['cn', 'en']
+        }
+        self._run_dataset_test('mm_bench', dataset_args=dataset_args)
+
+    def test_mm_star(self):
+        dataset_args = {
+            # 'subset_list': ['val']
+        }
+        self._run_dataset_test('mm_star', dataset_args=dataset_args)
+
+    def test_omni_bench(self):
+        dataset_args = {
+            'extra_params': {
+                'use_image': True, # Whether to use image input, if False, use text alternative image content.
+                'use_audio': True, # Whether to use audio input, if False, use text alternative audio content.
+            }
+        }
+        self._run_dataset_test('omni_bench', dataset_args=dataset_args, model='qwen-omni-turbo')
+
+    def test_olympiad_bench(self):
+        dataset_args = {
+            'subset_list': [
+                # 'OE_MM_maths_en_COMP',
+                # 'OE_MM_maths_zh_CEE',
+                # 'OE_MM_maths_zh_COMP',
+                # 'OE_MM_physics_en_COMP',
+                # 'OE_MM_physics_zh_CEE',
+                # 'OE_TO_maths_en_COMP',
+                # 'OE_TO_maths_zh_CEE',
+                # 'OE_TO_maths_zh_COMP',
+                # 'OE_TO_physics_en_COMP',
+                # 'OE_TO_physics_zh_CEE',
+                'TP_TO_maths_zh_CEE',
+            ]
+        }
+        self._run_dataset_test('olympiad_bench', dataset_args=dataset_args)

tests/cli/test_all.py

@@ -17,44 +17,44 @@ os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 logger = get_logger()
 
 datasets=[
-        'iquiz',
-        'ifeval',
-        'mmlu',
-        'mmlu_pro',
-        'musr',
-        'process_bench',
-        'race',
-        'trivia_qa',
-        'cmmlu',
-        'humaneval',
-        'gsm8k',
-        'bbh',
-        'competition_math',
-        'math_500',
-        'aime24',
-        'gpqa_diamond',
-        'arc',
-        'ceval',
-        'hellaswag',
-        'general_mcq',
-        'general_qa',
-        'super_gpqa',
-        # 'live_code_bench',
-        'mmlu_redux',
-        'simple_qa',
-        'chinese_simpleqa',
-        'alpaca_eval',
-        'arena_hard',
-        'maritime_bench',
-        'drop',
-        'winogrande',
-        'tool_bench',
-        'frames',
-        'docmath',
-        'needle_haystack',
-        'bfcl_v3',
-        'hle',
-        'tau_bench',
+    'iquiz',
+    'ifeval',
+    'mmlu',
+    'mmlu_pro',
+    'musr',
+    'process_bench',
+    'race',
+    'trivia_qa',
+    'cmmlu',
+    'humaneval',
+    'gsm8k',
+    'bbh',
+    'competition_math',
+    'math_500',
+    'aime24',
+    'gpqa_diamond',
+    'arc',
+    'ceval',
+    'hellaswag',
+    'general_mcq',
+    'general_qa',
+    'super_gpqa',
+    # 'live_code_bench',
+    'mmlu_redux',
+    'simple_qa',
+    'chinese_simpleqa',
+    'alpaca_eval',
+    'arena_hard',
+    'maritime_bench',
+    'drop',
+    'winogrande',
+    'tool_bench',
+    'frames',
+    'docmath',
+    'needle_haystack',
+    'bfcl_v3',
+    'hle',
+    'tau_bench',
 ]
 
 # Reverse the datasets list to ensure the order is from most recent to oldest
@@ -150,7 +150,6 @@ dataset_args={
 }
 
 class TestRun(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_benchmarks(self):
         from evalscope.config import TaskConfig
 
@@ -180,19 +179,60 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+    def test_vlm_benchmark(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-vl-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'mmmu',
+                # 'math_vista',
+            ],
+            dataset_args={
+                'mmmu': {
+                    'subset_list': ['Accounting']
+                },
+                'math_vista': {
+                    'subset_list': ['default']
+                }
+            },
+            eval_batch_size=1,
+            limit=1,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+                'image_height': 512,
+                'image_width': 512,
+                'image_num': 2,
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.AUTO,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+            }
+        )
+
+        run_task(task_cfg=task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_ci_lite(self):
         from evalscope.config import TaskConfig
 
+        api_key = env.get('DASHSCOPE_API_KEY')
+
         task_cfg = TaskConfig(
             model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
+            api_key=api_key,
+            eval_type=EvalType.SERVICE if api_key else EvalType.MOCK_LLM,
             datasets=[
                 'general_mcq',
-                'general_qa',
                 'iquiz',
             ],
             dataset_args={

tests/cli/test_collection.py

@@ -52,16 +52,19 @@ class TestCollection(unittest.TestCase):
             api_key=env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
+            dataset_args={
+                'data_collection': {
+                    # 'local_path': 'outputs/test_mix.jsonl'
+                    'local_path': 'outputs/mixed_data_test.jsonl',
+                    'shuffle': True,
+                }
+            },
             eval_batch_size=5,
             generation_config = {
                 'max_tokens': 10000,
                 'temperature': 0.0,
             },
-            limit=50,
+            limit=10,
             # use_cache='outputs/20250822_161804'
         )
         run_task(task_cfg=task_cfg)