evalscope 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

tests/cli/test_run.py

@@ -1,10 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
 import os
 import subprocess
-import torch
 import unittest
 
-from evalscope.constants import EvalType
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
 from evalscope.run import run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
@@ -71,21 +75,104 @@ class TestRun(unittest.TestCase):
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_task(self):
-        task_cfg = {'model': 'qwen/Qwen2-0.5B-Instruct',
-                    'datasets': [
-                        # 'mmlu_pro',
-                        # 'bbh',
-                        # 'hellaswag',
-                        'gsm8k',
-                        # 'arc',
-                        # 'race',
-                        # 'ifeval',
-                        # 'truthful_qa',
-                        # 'trivia_qa',
-                        ],
-                    'limit': 2,
-                    'eval_batch_size': 2,
-                    'debug': True}
+        task_cfg = TaskConfig(
+            model='qwen/Qwen2.5-0.5B-Instruct',
+            datasets=[
+                'iquiz',
+                # 'ifeval',
+                # 'mmlu',
+                # 'mmlu_pro',
+                # 'musr',
+                # 'process_bench',
+                # 'race',
+                # 'trivia_qa',
+                # 'cmmlu',
+                # 'humaneval',
+                # 'super_gpqa',
+                # 'gsm8k',
+                # 'bbh',
+                # 'competition_math',
+                # 'math_500',
+                'aime24',
+                'gpqa',
+                # 'arc',
+                # 'ceval',
+                # 'hellaswag',
+                # 'general_mcq',
+                # 'general_qa'
+            ],
+            dataset_args={
+                'mmlu': {
+                    'subset_list': ['elementary_mathematics'],
+                    'few_shot_num': 0
+                },
+                'mmlu_pro': {
+                    'subset_list': ['math', 'health'],
+                    'few_shot_num': 4
+                },
+                'ceval': {
+                    'subset_list': [
+                        'computer_network', 'operating_system', 'computer_architecture'
+                    ],
+                    'few_shot_num': 0
+                },
+                'cmmlu': {
+                    'subset_list': ['elementary_chinese'],
+                    'few_shot_num': 0
+                },
+                'bbh': {
+                    'subset_list': ['word_sorting', 'movie_recommendation'],
+                },
+                'gpqa': {
+                    'subset_list': ['gpqa_diamond'],
+                    'few_shot_num': 0
+                },
+                'humaneval': {
+                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
+                },
+                'competition_math': {
+                    'subset_list': ['Level 1']
+                },
+                'process_bench': {
+                    'subset_list': ['gsm8k'],
+                },
+                'musr': {
+                    'subset_list': ['murder_mysteries']
+                },
+                'general_mcq': {
+                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ],
+                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
+                        # 'test'
+                    ],
+                    'metric_list': ['AverageBLEU']
+                },
+                'super_gpqa': {
+                    'subset_list': ['Philosophy', 'Education'],
+                    'few_shot_num': 0
+                },
+                'ifeval': {
+                    'filters': {
+                        'remove_until': '</think>'
+                    }
+                }
+            },
+            limit=2,
+            eval_batch_size=2,
+            generation_config={
+                'max_new_tokens': 2048,
+                'temperature': 0.7,
+                'num_return_sequences': 1,
+            },
+            # debug=True
+        )
         run_task(task_cfg=task_cfg)
 
 
@@ -141,12 +228,12 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='Qwen2.5-0.5B-Instruct',
-            api_url='http://127.0.0.1:8801/v1',
-            api_key='EMPTY',
+            model='qwen2.5-7b-instruct',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                'iquiz',
+                # 'iquiz',
                 # 'ifeval',
                 # 'mmlu',
                 # 'mmlu_pro',
@@ -164,11 +251,14 @@ class TestRun(unittest.TestCase):
                 # 'gpqa',
                 # 'arc',
                 # 'ceval',
-                # 'hellaswag',
+                'hellaswag',
+                # 'general_mcq',
+                # 'general_qa'
+                # 'super_gpqa',
             ],
             dataset_args={
                 'mmlu': {
-                    'subset_list': ['elementary_mathematics'],
+                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
                     'few_shot_num': 0
                 },
                 'mmlu_pro': {
@@ -189,8 +279,9 @@ class TestRun(unittest.TestCase):
                     'subset_list': ['word_sorting', 'movie_recommendation'],
                 },
                 'gpqa': {
-                    'subset_list': ['gpqa_diamond'],
-                    'few_shot_num': 0
+                    # 'subset_list': ['gpqa_diamond'],
+                    'few_shot_num': 0,
+                    'local_path': './data/data/gpqa',
                 },
                 'humaneval': {
                     'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
@@ -204,17 +295,36 @@ class TestRun(unittest.TestCase):
                 'musr': {
                     'subset_list': ['murder_mysteries']
                 },
+                'general_mcq': {
+                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ],
+                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
+                        # 'test'
+                    ],
+                    'metric_list': ['AverageBLEU']
+                },
+                'super_gpqa': {
+                    # 'subset_list': ['Philosophy', 'Education'],
+                    'few_shot_num': 0
+                }
             },
-            eval_batch_size=5,
-            limit=5,
-            debug=True,
-            stream=True,
+            eval_batch_size=32,
+            limit=15,
+            # debug=True,
+            stream=False,
             generation_config={
-                'temperature': 0.7,
+                'temperature': 0,
                 'n': 1,
-                'max_tokens': 512,
+                'max_tokens': 4096,
             },
-            # use_cache='/mnt/data/data/user/maoyunlin.myl/eval-scope/outputs/20250212_150525',
+            # use_cache='./outputs/20250212_150525',
         )
 
         run_task(task_cfg=task_cfg)
@@ -250,5 +360,64 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_judge_model(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen2.5-7b-instruct',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                # 'math_500',
+                'aime24',
+                # 'competition_math',
+                # 'arc',
+                # 'gsm8k'
+                # 'truthful_qa',
+                # 'simple_qa',
+                # # 'chinese_simpleqa',
+                # 'live_code_bench',
+                # 'humaneval'
+                # 'general_qa'
+            ],
+            dataset_args={
+                'competition_math': {
+                    'subset_list': ['Level 4']
+                },
+                'live_code_bench': {
+                    'subset_list': ['v4_v5'],
+                    'extra_params': {
+                        'start_date': '2024-12-01',
+                        'end_date': '2025-01-01'
+                    },
+                    'local_path': '/root/.cache/modelscope/hub/datasets/AI-ModelScope/code_generation_lite'
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
+                        # 'test'
+                    ]
+                },
+            },
+            eval_batch_size=5,
+            limit=5,
+            judge_strategy=JudgeStrategy.AUTO,
+            judge_model_args={
+                'model_id': 'qwen2.5-7b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+            },
+            generation_config={
+                'max_new_tokens': 2048,
+                'temperature': 0.0,
+                'seed': 42,
+            }
+        )
+
+        run_task(task_cfg=task_cfg)
+
 if __name__ == '__main__':
     unittest.main()