evalscope 0.15.1__py3-none-any.whl → 0.16.0__py3-none-any.whl

tests/cli/test_run.py

@@ -73,6 +73,12 @@ class TestRun(unittest.TestCase):
         logger.info(f'>>test_run_eval_with_args stdout: {run_res.stdout}')
         logger.error(f'>>test_run_eval_with_args stderr: {run_res.stderr}')
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_yaml_config(self):
+        from evalscope import run_task
+
+        run_task(task_cfg='examples/tasks/eval_native.yaml')
+
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_task(self):
         task_cfg = TaskConfig(
@@ -209,12 +215,16 @@ class TestRun(unittest.TestCase):
         task_cfg = TaskConfig(
             model='Qwen/Qwen3-1.7B',
             datasets=[
-                # 'iquiz',
+                'iquiz',
                 # 'math_500',
-                'aime24',
+                # 'aime24',
                 # 'competition_math',
                 # 'mmlu',
+                # 'simple_qa',
             ],
+            model_args={
+                'device_map': 'auto',
+            },
             dataset_args={
                 'competition_math': {
                     'subset_list': ['Level 4', 'Level 5']
@@ -232,7 +242,8 @@ class TestRun(unittest.TestCase):
                 'top_p': 0.8,  # top-p采样 (qwen 报告推荐值)
                 'top_k': 20,  # top-k采样 (qwen 报告推荐值)
                 'chat_template_kwargs': {'enable_thinking': False}  # 关闭思考模式
-            }
+            },
+            judge_strategy=JudgeStrategy.AUTO,
         )
 
         run_task(task_cfg=task_cfg)
@@ -276,7 +287,7 @@ class TestRun(unittest.TestCase):
             datasets=[
                 # 'iquiz',
                 # 'ifeval',
-                'mmlu',
+                # 'mmlu',
                 # 'mmlu_pro',
                 # 'musr',
                 # 'process_bench',
@@ -294,10 +305,13 @@ class TestRun(unittest.TestCase):
                 # 'ceval',
                 # 'hellaswag',
                 # 'general_mcq',
-                # 'general_qa'
+                # 'general_qa',
                 # 'super_gpqa',
                 # 'mmlu_redux',
-                # 'maritime_bench'
+                # 'maritime_bench',
+                # 'drop',
+                # 'winogrande',
+                'tool_bench',
             ],
             dataset_args={
                 'mmlu': {
@@ -363,14 +377,16 @@ class TestRun(unittest.TestCase):
                 },
             },
             eval_batch_size=32,
-            limit=15,
+            limit=10,
             debug=True,
             stream=False,
             generation_config={
                 'temperature': 0,
                 'n': 1,
                 'max_tokens': 4096,
-            }
+            },
+            # ignore_errors=True,
+            use_cache='outputs/20250519_142106'
         )
 
         run_task(task_cfg=task_cfg)
@@ -423,12 +439,12 @@ class TestRun(unittest.TestCase):
                 # 'gsm8k'
                 # 'truthful_qa',
                 # 'simple_qa',
-                # 'chinese_simpleqa',
+                'chinese_simpleqa',
                 # 'live_code_bench',
                 # 'humaneval',
                 # 'general_qa',
                 # 'alpaca_eval',
-                'arena_hard'
+                # 'arena_hard'
             ],
             dataset_args={
                 'competition_math': {
@@ -454,7 +470,7 @@ class TestRun(unittest.TestCase):
                     ]
                 },
             },
-            eval_batch_size=5,
+            eval_batch_size=10,
             limit=10,
             judge_strategy=JudgeStrategy.AUTO,
             judge_worker_num=5,
@@ -475,7 +491,7 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
-            # use_cache='outputs/20250320_143658'
+            use_cache='outputs/20250519_142551'
         )
 
         run_task(task_cfg=task_cfg)

tests/perf/test_perf.py

@@ -121,6 +121,29 @@ class TestPerf(unittest.TestCase):
         print(metrics_result)
         print(percentile_result)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_perf_multi_parallel(self):
+        from evalscope.perf.arguments import Arguments
+        task_cfg = Arguments(
+            parallel=[1, 2],
+            number=[2, 5],
+            model='qwen2.5-7b-instruct',
+            url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+            api_key=env.get('DASHSCOPE_API_KEY'),
+            api='openai',
+            dataset='random',
+            min_tokens=100,
+            max_tokens=100,
+            prefix_length=0,
+            min_prompt_length=1024,
+            max_prompt_length=1024,
+            tokenizer_path='Qwen/Qwen2.5-0.5B-Instruct',
+            seed=None,
+            extra_args={'ignore_eos': True}
+        )
+        metrics_result, percentile_result = run_perf_benchmark(task_cfg)
+        print(metrics_result)
+        print(percentile_result)
 
 if __name__ == '__main__':
     unittest.main(buffer=False)