evalscope 0.16.0__py3-none-any.whl → 0.16.2__py3-none-any.whl

tests/aigc/test_t2i.py

@@ -11,7 +11,7 @@ from evalscope.run import run_task
 from evalscope.utils import test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
@@ -28,15 +28,15 @@ class TestRun(unittest.TestCase):
             dataset_args={
                 'general_t2i': {
                     'metric_list': [
-                        'PickScore',
+                        # 'PickScore',
                         'CLIPScore',
-                        'HPSv2Score',
-                        'HPSv2.1Score',
-                        'BLIPv2Score',
-                        'ImageRewardScore',
-                        'VQAScore',
-                        'FGA_BLIP2Score',
-                        'MPS'
+                        # 'HPSv2Score',
+                        # 'HPSv2.1Score',
+                        # 'BLIPv2Score',
+                        # 'ImageRewardScore',
+                        # 'VQAScore',
+                        # 'FGA_BLIP2Score',
+                        # 'MPS'
                     ],
                     'dataset_id': 'custom_eval/multimodal/t2i/example.jsonl',
                 }
@@ -58,9 +58,9 @@ class TestRun(unittest.TestCase):
                 'torch_dtype': 'torch.float16',
             },
             datasets=[
-                'tifa160',
+                # 'tifa160',
                 # 'genai_bench',
-                # 'evalmuse',
+                'evalmuse',
                 # 'hpdv2',
             ],
             dataset_args={
@@ -85,3 +85,40 @@ class TestRun(unittest.TestCase):
         )
 
         run_task(task_cfg=task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_benchmark_flux(self):
+
+        task_cfg = TaskConfig(
+            model='black-forest-labs/FLUX.1-dev',  # model on modelscope
+            model_task=ModelTask.IMAGE_GENERATION,  # must be IMAGE_GENERATION
+            model_args={
+                'torch_dtype': 'torch.float16',
+            },
+            datasets=[
+                # 'tifa160',
+                # 'genai_bench',
+                'evalmuse',
+                # 'hpdv2',
+            ],
+            dataset_args={
+                'tifa160': {
+                    'metric_list': [
+                        'PickScore',
+                        # 'CLIPScore',
+                        # 'HPSv2Score',
+                        # 'BLIPv2Score',
+                        # 'ImageRewardScore',
+                        # 'VQAScore',
+                        # 'FGA_BLIP2Score',
+                    ]
+                }
+            },
+            generation_config={
+                'num_inference_steps': 50,
+                'guidance_scale': 3.5
+            },
+            use_cache='outputs/20250520_112314'
+        )
+
+        run_task(task_cfg=task_cfg)

tests/cli/test_all.py

@@ -12,7 +12,7 @@ from evalscope.run import run_task
 from evalscope.utils import test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
@@ -49,6 +49,10 @@ datasets=[
         'drop',
         'winogrande',
         'tool_bench',
+        'frames',
+        'docmath',
+        'needle_haystack',
+        'bfcl_v3',
 ]
 
 dataset_args={
@@ -123,6 +127,12 @@ dataset_args={
     'mmlu_redux':{
         'subset_list': ['abstract_algebra']
     },
+    'frames':{
+        'subset_list': ['simpshort_testmini']
+    },
+    'bfcl_v3':{
+        'subset_list': ['simple', 'multiple']
+    }
 }
 
 class TestRun(unittest.TestCase):
@@ -131,7 +141,7 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-0.5b-instruct',
+            model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
@@ -145,9 +155,10 @@ class TestRun(unittest.TestCase):
                 'n': 1,
                 'max_tokens': 4096,
             },
+            judge_worker_num=5,
             judge_strategy=JudgeStrategy.AUTO,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
             }

tests/cli/test_collection.py

@@ -72,14 +72,16 @@ class TestCollection(unittest.TestCase):
                 'local_path': 'outputs/mixed_data_test.jsonl'
                 # 'local_path': 'outputs/weighted_mixed_data.jsonl'
             }},
-            limit=10,
-            judge_strategy=JudgeStrategy.LLM_RECALL,
+            limit=5,
+            judge_strategy=JudgeStrategy.AUTO,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': os.getenv('DASHSCOPE_API_KEY'),
             },
-            use_cache='outputs/20250519_114427'
+            analysis_report=True,
+            ignore_errors=True,
+            # use_cache='outputs/20250522_204520'
         )
         res = run_task(task_cfg=task_cfg)
         print(res)

tests/cli/test_run.py

@@ -13,7 +13,7 @@ from evalscope.run import run_task
 from evalscope.utils import is_module_installed, test_level_list
 from evalscope.utils.logger import get_logger
 
-os.environ['LOG_LEVEL'] = 'DEBUG'
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 
 logger = get_logger()
 
@@ -63,7 +63,7 @@ class TestRun(unittest.TestCase):
                         f'--model {model} ' \
                         f'--datasets {datasets} ' \
                         f'--limit {limit} ' \
-                        f'--generation-config do_sample=false,temperature=0.0 ' \
+                        f'--generation-config do_sample=true,temperature=0.6,max_length=65535,max_new_tokens=65535,max_tokens=65535,n=1,top_p=0.95,top_k=20 ' \
                         f"""--dataset-args \'{dataset_args}\' """
 
         logger.info(f'Start to run command: {cmd_with_args}')
@@ -187,8 +187,11 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen/Qwen2-0.5B-Instruct',
-            datasets=['general_mcq', 'general_qa'],  # 数据格式，选择题格式固定为 'ceval'
+            model='Qwen/Qwen3-0.6B',
+            datasets=[
+                'general_mcq',
+                'general_qa'
+            ],
             dataset_args={
                 'general_mcq': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
@@ -215,16 +218,14 @@ class TestRun(unittest.TestCase):
         task_cfg = TaskConfig(
             model='Qwen/Qwen3-1.7B',
             datasets=[
-                'iquiz',
+                # 'iquiz',
                 # 'math_500',
                 # 'aime24',
                 # 'competition_math',
                 # 'mmlu',
                 # 'simple_qa',
+                'truthful_qa',
             ],
-            model_args={
-                'device_map': 'auto',
-            },
             dataset_args={
                 'competition_math': {
                     'subset_list': ['Level 4', 'Level 5']
@@ -304,14 +305,16 @@ class TestRun(unittest.TestCase):
                 # 'arc',
                 # 'ceval',
                 # 'hellaswag',
-                # 'general_mcq',
+                'general_mcq',
                 # 'general_qa',
                 # 'super_gpqa',
                 # 'mmlu_redux',
                 # 'maritime_bench',
                 # 'drop',
                 # 'winogrande',
-                'tool_bench',
+                # 'tool_bench',
+                # 'frames',
+                # 'bfcl_v3',
             ],
             dataset_args={
                 'mmlu': {
@@ -369,24 +372,31 @@ class TestRun(unittest.TestCase):
                     'metric_list': ['AverageRouge']
                 },
                 'super_gpqa': {
-                    # 'subset_list': ['Philosophy', 'Education'],
+                    'subset_list': ['Philosophy', 'Education'],
                     'few_shot_num': 0
                 },
                 'mmlu_redux':{
                     'subset_list': ['abstract_algebra']
                 },
+                'bfcl_v3': {
+                    'subset_list': ['parallel'],
+                    'extra_params': {
+                        # 'is_fc_model': False,
+                    }
+                },
             },
-            eval_batch_size=32,
-            limit=10,
+            eval_batch_size=10,
+            limit=5,
             debug=True,
-            stream=False,
+            stream=True,
             generation_config={
                 'temperature': 0,
                 'n': 1,
                 'max_tokens': 4096,
+                # 'extra_headers':{'key': 'value'},
             },
-            # ignore_errors=True,
-            use_cache='outputs/20250519_142106'
+            ignore_errors=False,
+            # use_cache='outputs/20250616_153756'
         )
 
         run_task(task_cfg=task_cfg)
@@ -427,26 +437,36 @@ class TestRun(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-0.5b-instruct',
+            model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                # 'math_500',
+                'math_500',
                 # 'aime24',
                 # 'competition_math',
                 # 'arc',
-                # 'gsm8k'
+                # 'gsm8k',
                 # 'truthful_qa',
                 # 'simple_qa',
-                'chinese_simpleqa',
+                # 'chinese_simpleqa',
                 # 'live_code_bench',
                 # 'humaneval',
                 # 'general_qa',
                 # 'alpaca_eval',
-                # 'arena_hard'
+                # 'arena_hard',
+                # 'frames',
+                # 'docmath',
+                # 'needle_haystack',
+                # 'ifeval',
             ],
             dataset_args={
+                'needle_haystack': {
+                    'subset_list': ['english'],
+                    'extra_params': {
+                        'show_score': True,
+                    }
+                },
                 'competition_math': {
                     'subset_list': ['Level 4']
                 },
@@ -469,13 +489,16 @@ class TestRun(unittest.TestCase):
                         '中华文化'
                     ]
                 },
+                'frames': {
+                    'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
+                }
             },
             eval_batch_size=10,
-            limit=10,
-            judge_strategy=JudgeStrategy.AUTO,
+            limit=3,
+            judge_strategy=JudgeStrategy.LLM,
             judge_worker_num=5,
             judge_model_args={
-                'model_id': 'qwen2.5-7b-instruct',
+                'model_id': 'qwen2.5-72b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'generation_config': {
@@ -491,7 +514,9 @@ class TestRun(unittest.TestCase):
             },
             timeout=60000,
             stream=True,
-            use_cache='outputs/20250519_142551'
+            # analysis_report=True,
+            # debug=True,
+            # use_cache='outputs/20250616_161931'
         )
 
         run_task(task_cfg=task_cfg)

tests/rag/test_clip_benchmark.py

@@ -39,7 +39,11 @@ class TestCLIPBenchmark(unittest.TestCase):
                             'model_name': 'AI-ModelScope/chinese-clip-vit-large-patch14-336px',
                         }
                     ],
-                    'dataset_name': ['muge', 'mnist'],
+                    'dataset_name': [
+                        'muge',
+                        'mnist',
+                        'flickr8k'
+                    ],
                     'split': 'test',
                     'batch_size': 128,
                     'num_workers': 1,

tests/rag/test_mteb.py

@@ -46,11 +46,11 @@ class TestMTEB(unittest.TestCase):
                 ],
                 'eval': {
                     'tasks': [
-                        # 'TNews',
-                        # 'CLSClusteringS2S',
+                        'TNews',
+                        'CLSClusteringS2S',
                         'T2Reranking',
-                        # 'T2Retrieval',
-                        # 'ATEC',
+                        'T2Retrieval',
+                        'ATEC',
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
@@ -85,7 +85,7 @@ class TestMTEB(unittest.TestCase):
                     ],
                     'verbosity': 2,
                     'overwrite_results': True,
-                    'limits': 30,
+                    'limits': 10,
                 },
             },
         )
@@ -121,10 +121,54 @@ class TestMTEB(unittest.TestCase):
                     },
                 ],
                 'eval': {
-                    'tasks': ['MedicalRetrieval', 'T2Retrieval'],
+                    'tasks': [
+                        'MedicalRetrieval',
+                        'T2Retrieval'
+                    ],
+                    'verbosity': 2,
+                    'overwrite_results': True,
+                    'limits': 10,
+                    'top_k': 10,
+                },
+            },
+        }
+
+        run_task(task_cfg)
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_two_stage_api(self):
+        task_cfg = {
+            'eval_backend': 'RAGEval',
+            'eval_config': {
+                'tool': 'MTEB',
+                'model': [
+                    {
+                        'model_name': 'text-embedding-v3',
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
+                        'dimensions': 1024,
+                        'encode_kwargs': {
+                            'batch_size': 10,
+                        },
+                    },
+                    {
+                        'model_name': 'text-embedding-v3',
+                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                        'api_key': env.get('DASHSCOPE_API_KEY', 'EMPTY'),
+                        'dimensions': 1024,
+                        'encode_kwargs': {
+                            'batch_size': 10,
+                        },
+                    },
+                ],
+                'eval': {
+                    'tasks': [
+                        'MedicalRetrieval',
+                        # 'T2Retrieval'
+                    ],
                     'verbosity': 2,
                     'overwrite_results': True,
-                    # 'limits': 10,
+                    'limits': 10,
                     'top_k': 10,
                 },
             },