evalscope 0.16.3__py3-none-any.whl → 0.17.1__py3-none-any.whl

tests/cli/test_custom.py

@@ -0,0 +1,261 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+from tests.utils import test_level_list
+
+env = dotenv_values('.env')
+
+import os
+import subprocess
+import unittest
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
+from evalscope.run import run_task
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.logger import get_logger
+
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
+
+logger = get_logger()
+
+
+class TestRunCustom(unittest.TestCase):
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_custom_task(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='Qwen/Qwen3-0.6B',
+            datasets=[
+                'general_mcq',
+                'general_qa'
+            ],
+            dataset_args={
+                'general_mcq': {
+                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ],
+                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ]
+                }
+            },
+        )
+        res = run_task(task_cfg=task_cfg)
+        print(res)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_local_dataset(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                # 'mmlu',
+                # 'race',
+                'trivia_qa',
+                # 'cmmlu',
+                # 'humaneval',
+                # 'gsm8k',
+                # 'bbh',
+                # 'competition_math',
+                # 'arc',
+                # 'ceval',
+            ],
+            dataset_args={
+                'mmlu': {
+                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
+                    'few_shot_num': 0,
+                    'dataset_id': 'data/data/mmlu',
+                },
+                'ceval': {
+                    'subset_list': [
+                        'computer_network', 'operating_system', 'computer_architecture'
+                    ],
+                    'few_shot_num': 0,
+                    'dataset_id': 'data/data/ceval',
+                },
+                'cmmlu': {
+                    'subset_list': ['elementary_chinese'],
+                    'dataset_id': 'data/data/cmmlu',
+                    'few_shot_num': 0
+                },
+                'bbh': {
+                    'subset_list': ['word_sorting', 'movie_recommendation'],
+                },
+                'humaneval': {
+                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
+                },
+                'trivia_qa': {
+                    'dataset_id': 'data/data/trivia_qa',
+                },
+            },
+            eval_batch_size=10,
+            limit=5,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_no_answer(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen2.5-72b-instruct',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'general_qa',
+            ],
+            dataset_args={
+                'general_qa': {
+                    'dataset_id': 'custom_eval/text/qa',
+                    'subset_list': [
+                        'arena',
+                        'example'
+                    ],
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096
+                },
+                'score_type': 'numeric',
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.AUTO,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_with_answer(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'general_qa',
+            ],
+            dataset_args={
+                'general_qa': {
+                    'dataset_id': 'custom_eval/text/qa',
+                    'subset_list': [
+                        'example'
+                    ],
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096
+                },
+                'score_type': 'pattern',
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.LLM,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_arena(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model_id='Arena',
+            datasets=[
+                'general_arena',
+            ],
+            dataset_args={
+                'general_arena': {
+                    'extra_params':{
+                        'models':[
+                            {
+                                'name': 'qwen2.5-0.5b',
+                                'report_path': 'outputs/20250702_140354/reports/qwen2.5-0.5b-instruct'
+                            },
+                            {
+                                'name': 'qwen2.5-7b',
+                                'report_path': 'outputs/20250702_140702/reports/qwen2.5-7b-instruct'
+                            },
+                            {
+                                'name': 'qwen2.5-72b',
+                                'report_path': 'outputs/20250702_140802/reports/qwen2.5-72b-instruct'
+                            }
+                        ],
+                        'baseline': 'qwen2.5-7b'
+                    }
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen-plus',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 8000
+                },
+            },
+            judge_worker_num=5,
+            use_cache='outputs/20250702_165727'
+        )
+
+        run_task(task_cfg=task_cfg)

tests/cli/test_run.py

@@ -1,6 +1,8 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 
 import os
@@ -8,9 +10,9 @@ import subprocess
 import unittest
 
 from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
@@ -182,35 +184,6 @@ class TestRun(unittest.TestCase):
         run_task(task_cfg=task_cfg)
 
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom_task(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='Qwen/Qwen3-0.6B',
-            datasets=[
-                'general_mcq',
-                'general_qa'
-            ],
-            dataset_args={
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ]
-                }
-            },
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)
-
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_one_task(self):
         from evalscope.config import TaskConfig
@@ -293,7 +266,7 @@ class TestRun(unittest.TestCase):
                 # 'musr',
                 # 'process_bench',
                 # 'race',
-                # 'trivia_qa',
+                'trivia_qa',
                 # 'cmmlu',
                 # 'humaneval',
                 # 'gsm8k',
@@ -306,7 +279,7 @@ class TestRun(unittest.TestCase):
                 # 'ceval',
                 # 'hellaswag',
                 # 'general_mcq',
-                'general_qa',
+                # 'general_qa',
                 # 'super_gpqa',
                 # 'mmlu_redux',
                 # 'maritime_bench',
@@ -315,6 +288,9 @@ class TestRun(unittest.TestCase):
                 # 'tool_bench',
                 # 'frames',
                 # 'bfcl_v3',
+                # 'truthful_qa',
+                # 'tau_bench',
+                # 'hle'
             ],
             dataset_args={
                 'mmlu': {
@@ -323,7 +299,7 @@ class TestRun(unittest.TestCase):
                 },
                 'mmlu_pro': {
                     'subset_list': ['math', 'health'],
-                    'few_shot_num': 4
+                    'few_shot_num': 0
                 },
                 'ceval': {
                     'subset_list': [
@@ -354,7 +330,6 @@ class TestRun(unittest.TestCase):
                 },
                 'musr': {
                     'subset_list': ['murder_mysteries'],
-                    'local_path': '/root/.cache/modelscope/hub/datasets/AI-ModelScope/MuSR'
                 },
                 'general_mcq': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
@@ -378,59 +353,42 @@ class TestRun(unittest.TestCase):
                 'mmlu_redux':{
                     'subset_list': ['abstract_algebra']
                 },
+                'frames':{
+                    'local_path': 'data/iic/frames',
+                },
                 'bfcl_v3': {
                     'subset_list': ['parallel'],
                     'extra_params': {
                         # 'is_fc_model': False,
                     }
                 },
+                'tau_bench': {
+                    'extra_params': {
+                        'user_model': 'qwen-plus',
+                        'api_key': env.get('DASHSCOPE_API_KEY'),
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                    }
+                },
+                'hle': {
+                    'subset_list': ['Math', 'Other'],
+                },
             },
             eval_batch_size=10,
-            limit=5,
-            debug=True,
+            limit=10,
+            # debug=True,
             stream=True,
             generation_config={
-                'temperature': 0,
+                'temperature': 0.6,
                 'n': 1,
                 'max_tokens': 4096,
                 # 'extra_headers':{'key': 'value'},
             },
             ignore_errors=False,
-            use_cache='outputs/test_2'
         )
 
         run_task(task_cfg=task_cfg)
 
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_batch_eval(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='LLM-Research/Llama-3.2-1B-Instruct',
-            datasets=[
-                # 'math_500',
-                # 'aime24',
-                # 'competition_math'
-                # 'arc',
-                'gsm8k'
-                # 'truthful_qa'
-            ],
-            dataset_args={
-                'competition_math': {
-                    'subset_list': ['Level 4', 'Level 5']
-                }
-            },
-            eval_batch_size=2,
-            limit=5,
-            generation_config={
-                'max_new_tokens': 2048,
-                'temperature': 0.7,
-                'num_return_sequences': 2,
-            }
-        )
-
-        run_task(task_cfg=task_cfg)
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_judge_model(self):
@@ -442,7 +400,7 @@ class TestRun(unittest.TestCase):
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                'math_500',
+                # 'math_500',
                 # 'aime24',
                 # 'competition_math',
                 # 'arc',
@@ -459,6 +417,7 @@ class TestRun(unittest.TestCase):
                 # 'docmath',
                 # 'needle_haystack',
                 # 'ifeval',
+                'hle'
             ],
             dataset_args={
                 'needle_haystack': {
@@ -491,7 +450,10 @@ class TestRun(unittest.TestCase):
                 },
                 'frames': {
                     'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
-                }
+                },
+                'hle': {
+                    'subset_list': ['Math', 'Other'],
+                },
             },
             eval_batch_size=10,
             limit=3,
@@ -514,6 +476,7 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
+            use_cache='outputs/20250714_150626'
             # analysis_report=True,
             # debug=True,
             # use_cache='outputs/20250616_161931'
@@ -521,5 +484,6 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+
 if __name__ == '__main__':
     unittest.main()

tests/perf/test_perf.py

@@ -7,7 +7,7 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 import unittest
 
 from evalscope.perf.main import run_perf_benchmark
-from evalscope.utils import test_level_list
+from tests.utils import test_level_list
 
 
 class TestPerf(unittest.TestCase):
@@ -35,9 +35,9 @@ class TestPerf(unittest.TestCase):
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_stream(self):
         task_cfg = {
-            'url': 'http://127.0.0.1:8000/v1/chat/completions',
+            'url': 'http://127.0.0.1:8801/v1/chat/completions',
             'parallel': 1,
-            'model': 'qwen2.5',
+            'model': 'Qwen2.5-0.5B-Instruct',
             'number': 15,
             'api': 'openai',
             'dataset': 'openqa',
@@ -126,7 +126,7 @@ class TestPerf(unittest.TestCase):
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
             parallel=[1, 2],
-            number=[2, 5],
+            number=[2, 4],
             model='qwen2.5-7b-instruct',
             url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
             api_key=env.get('DASHSCOPE_API_KEY'),
@@ -145,5 +145,32 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_random_vl(self):
+        from evalscope.perf.arguments import Arguments
+        task_cfg = Arguments(
+            parallel=[1, 2],
+            number=[2, 4],
+            model='qwen-vl-max',
+            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+            api_key=env.get('DASHSCOPE_API_KEY'),
+            api='openai',
+            dataset='kontext_bench',
+            min_tokens=100,
+            max_tokens=100,
+            prefix_length=0,
+            min_prompt_length=100,
+            max_prompt_length=100,
+            image_height=512,
+            image_width=512,
+            image_num=2,
+            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            seed=None,
+            extra_args={'ignore_eos': True}
+        )
+        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
+        print(metrics_result)
+        print(percentile_result)
+
 if __name__ == '__main__':
     unittest.main(buffer=False)

tests/rag/test_clip_benchmark.py

@@ -6,8 +6,9 @@ import subprocess
 import unittest
 
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger()
 

tests/rag/test_mteb.py

@@ -3,9 +3,11 @@
 import unittest
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

tests/rag/test_ragas.py

@@ -2,11 +2,13 @@
 import os
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 import unittest
 
 from evalscope import TaskConfig, run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

tests/swift/test_run_swift_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.opencompass import OpenCompassBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/swift/test_run_swift_vlm_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/swift/test_run_swift_vlm_jugde_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/utils.py

@@ -0,0 +1,13 @@
+import os
+
+TEST_LEVEL_LIST = [0, 1]
+# Example: export TEST_LEVEL_LIST=0,1
+TEST_LEVEL_LIST_STR = 'TEST_LEVEL_LIST'
+
+
+def test_level_list():
+    global TEST_LEVEL_LIST
+    if TEST_LEVEL_LIST_STR in os.environ:
+        TEST_LEVEL_LIST = [int(x) for x in os.environ[TEST_LEVEL_LIST_STR].split(',')]
+
+    return TEST_LEVEL_LIST

tests/vlm/test_vlmeval.py

@@ -1,12 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 import unittest
 
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -62,7 +64,11 @@ class TestVLMEval(unittest.TestCase):
         task_cfg = {
             'eval_backend': 'VLMEvalKit',
             'eval_config': {
-                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'data': [
+                    # 'SEEDBench_IMG',
+                    # 'ChartQA_TEST',
+                    'MMDU'
+                    ],
                 'limit': 5,
                 'mode': 'all',
                 'model': [