evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

tests/cli/test_all.py

@@ -12,40 +12,46 @@ from evalscope.run import run_task
 from evalscope.utils import test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
 datasets=[
-        'iquiz',
-        'ifeval',
-        'mmlu',
-        'mmlu_pro',
-        'musr',
-        'process_bench',
-        'race',
-        'trivia_qa',
-        'cmmlu',
-        'humaneval',
-        'gsm8k',
-        'bbh',
-        'competition_math',
-        'math_500',
-        'aime24',
-        'gpqa',
-        'arc',
-        'ceval',
-        'hellaswag',
-        'general_mcq',
-        'general_qa',
-        'super_gpqa',
-        'live_code_bench',
-        'mmlu_redux',
-        'simple_qa',
-        'chinese_simpleqa',
-        'alpaca_eval',
-        'arena_hard',
-        'maritime_bench',
+        # 'iquiz',
+        # 'ifeval',
+        # 'mmlu',
+        # 'mmlu_pro',
+        # 'musr',
+        # 'process_bench',
+        # 'race',
+        # 'trivia_qa',
+        # 'cmmlu',
+        # 'humaneval',
+        # 'gsm8k',
+        # 'bbh',
+        # 'competition_math',
+        # 'math_500',
+        # 'aime24',
+        # 'gpqa',
+        # 'arc',
+        # 'ceval',
+        # 'hellaswag',
+        # 'general_mcq',
+        # 'general_qa',
+        # 'super_gpqa',
+        # 'live_code_bench',
+        # 'mmlu_redux',
+        # 'simple_qa',
+        # 'chinese_simpleqa',
+        # 'alpaca_eval',
+        # 'arena_hard',
+        # 'maritime_bench',
+        # 'drop',
+        # 'winogrande',
+        # 'tool_bench',
+        'frames',
+        'docmath',
+        'needle_haystack'
 ]
 
 dataset_args={
@@ -128,7 +134,7 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-0.5b-instruct',
+            model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
@@ -142,9 +148,10 @@ class TestRun(unittest.TestCase):
                 'n': 1,
                 'max_tokens': 4096,
             },
+            judge_worker_num=5,
             judge_strategy=JudgeStrategy.AUTO,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
             }

tests/cli/test_collection.py

@@ -72,13 +72,15 @@ class TestCollection(unittest.TestCase):
                 'local_path': 'outputs/mixed_data_test.jsonl'
                 # 'local_path': 'outputs/weighted_mixed_data.jsonl'
             }},
-            limit=10,
-            judge_strategy=JudgeStrategy.LLM_RECALL,
+            limit=5,
+            judge_strategy=JudgeStrategy.AUTO,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': os.getenv('DASHSCOPE_API_KEY'),
-            }
+                # 'model_id': 'qwen2.5-72b-instruct',
+                # 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                # 'api_key': os.getenv('DASHSCOPE_API_KEY'),
+            },
+            analysis_report=True,
+            # use_cache='outputs/20250522_204520'
         )
         res = run_task(task_cfg=task_cfg)
         print(res)

tests/cli/test_run.py

@@ -13,7 +13,7 @@ from evalscope.run import run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
@@ -73,6 +73,12 @@ class TestRun(unittest.TestCase):
         logger.info(f'>>test_run_eval_with_args stdout: {run_res.stdout}')
         logger.error(f'>>test_run_eval_with_args stderr: {run_res.stderr}')
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_yaml_config(self):
+        from evalscope import run_task
+
+        run_task(task_cfg='examples/tasks/eval_native.yaml')
+
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_task(self):
         task_cfg = TaskConfig(
@@ -209,12 +215,16 @@ class TestRun(unittest.TestCase):
         task_cfg = TaskConfig(
             model='Qwen/Qwen3-1.7B',
             datasets=[
-                # 'iquiz',
+                'iquiz',
                 # 'math_500',
-                'aime24',
+                # 'aime24',
                 # 'competition_math',
                 # 'mmlu',
+                # 'simple_qa',
             ],
+            model_args={
+                'device_map': 'auto',
+            },
             dataset_args={
                 'competition_math': {
                     'subset_list': ['Level 4', 'Level 5']
@@ -232,7 +242,8 @@ class TestRun(unittest.TestCase):
                 'top_p': 0.8,  # top-p采样 (qwen 报告推荐值)
                 'top_k': 20,  # top-k采样 (qwen 报告推荐值)
                 'chat_template_kwargs': {'enable_thinking': False}  # 关闭思考模式
-            }
+            },
+            judge_strategy=JudgeStrategy.AUTO,
         )
 
         run_task(task_cfg=task_cfg)
@@ -276,7 +287,7 @@ class TestRun(unittest.TestCase):
             datasets=[
                 # 'iquiz',
                 # 'ifeval',
-                'mmlu',
+                # 'mmlu',
                 # 'mmlu_pro',
                 # 'musr',
                 # 'process_bench',
@@ -294,10 +305,14 @@ class TestRun(unittest.TestCase):
                 # 'ceval',
                 # 'hellaswag',
                 # 'general_mcq',
-                # 'general_qa'
+                # 'general_qa',
                 # 'super_gpqa',
                 # 'mmlu_redux',
-                # 'maritime_bench'
+                # 'maritime_bench',
+                # 'drop',
+                # 'winogrande',
+                # 'tool_bench',
+                'frames',
             ],
             dataset_args={
                 'mmlu': {
@@ -363,14 +378,17 @@ class TestRun(unittest.TestCase):
                 },
             },
             eval_batch_size=32,
-            limit=15,
+            limit=10,
             debug=True,
             stream=False,
             generation_config={
                 'temperature': 0,
                 'n': 1,
                 'max_tokens': 4096,
-            }
+                # 'extra_headers':{'key': 'value'},
+            },
+            # ignore_errors=True,
+            # use_cache='outputs/20250519_142106'
         )
 
         run_task(task_cfg=task_cfg)
@@ -411,16 +429,16 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-0.5b-instruct',
+            model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
                 # 'math_500',
-                # 'aime24',
+                'aime24',
                 # 'competition_math',
                 # 'arc',
-                # 'gsm8k'
+                # 'gsm8k',
                 # 'truthful_qa',
                 # 'simple_qa',
                 # 'chinese_simpleqa',
@@ -428,7 +446,10 @@ class TestRun(unittest.TestCase):
                 # 'humaneval',
                 # 'general_qa',
                 # 'alpaca_eval',
-                'arena_hard'
+                # 'arena_hard',
+                # 'frames',
+                # 'docmath',
+                # 'needle_haystack',
             ],
             dataset_args={
                 'competition_math': {
@@ -453,13 +474,16 @@ class TestRun(unittest.TestCase):
                         '中华文化'
                     ]
                 },
+                'frames': {
+                    'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
+                }
             },
-            eval_batch_size=5,
-            limit=10,
+            eval_batch_size=10,
+            limit=1,
             judge_strategy=JudgeStrategy.AUTO,
             judge_worker_num=5,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'generation_config': {
@@ -475,7 +499,9 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
-            # use_cache='outputs/20250320_143658'
+            analysis_report=True,
+            # debug=True,
+            # use_cache='outputs/20250602_135859'
         )
 
         run_task(task_cfg=task_cfg)

tests/perf/test_perf.py

@@ -121,6 +121,29 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_multi_parallel(self):
+        from evalscope.perf.arguments import Arguments
+        task_cfg = Arguments(
+            parallel=[1, 2],
+            number=[2, 5],
+            model='qwen2.5-7b-instruct',
+            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+            api_key=env.get('DASHSCOPE_API_KEY'),
+            api='openai',
+            dataset='random',
+            min_tokens=100,
+            max_tokens=100,
+            prefix_length=0,
+            min_prompt_length=1024,
+            max_prompt_length=1024,
+            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            seed=None,
+            extra_args={'ignore_eos': True}
+        )
+        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
+        print(metrics_result)
+        print(percentile_result)
 
 if __name__ == '__main__':
     unittest.main(buffer=False)

tests/rag/test_mteb.py

@@ -46,11 +46,11 @@ class TestMTEB(unittest.TestCase):
                 ],
                 'eval': {
                     'tasks': [
-                        # 'TNews',
-                        # 'CLSClusteringS2S',
+                        'TNews',
+                        'CLSClusteringS2S',
                         'T2Reranking',
-                        # 'T2Retrieval',
-                        # 'ATEC',
+                        'T2Retrieval',
+                        'ATEC',
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
@@ -85,7 +85,7 @@ class TestMTEB(unittest.TestCase):
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
-                    'limits': 30,
+                    'limits': 10,
                 },
             },
         )