evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

tests/cli/test_all.py

@@ -17,44 +17,44 @@ os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
 logger = get_logger()
 
 datasets=[
-        'iquiz',
-        'ifeval',
-        'mmlu',
-        'mmlu_pro',
-        'musr',
-        'process_bench',
-        'race',
-        'trivia_qa',
-        'cmmlu',
-        'humaneval',
-        'gsm8k',
-        'bbh',
-        'competition_math',
-        'math_500',
-        'aime24',
-        'gpqa',
-        'arc',
-        'ceval',
-        'hellaswag',
-        'general_mcq',
-        'general_qa',
-        'super_gpqa',
-        # 'live_code_bench',
-        'mmlu_redux',
-        'simple_qa',
-        'chinese_simpleqa',
-        'alpaca_eval',
-        'arena_hard',
-        'maritime_bench',
-        'drop',
-        'winogrande',
-        'tool_bench',
-        'frames',
-        'docmath',
-        'needle_haystack',
-        'bfcl_v3',
-        'hle',
-        'tau_bench',
+    'iquiz',
+    'ifeval',
+    'mmlu',
+    'mmlu_pro',
+    'musr',
+    'process_bench',
+    'race',
+    'trivia_qa',
+    'cmmlu',
+    'humaneval',
+    'gsm8k',
+    'bbh',
+    'competition_math',
+    'math_500',
+    'aime24',
+    'gpqa_diamond',
+    'arc',
+    'ceval',
+    'hellaswag',
+    'general_mcq',
+    'general_qa',
+    'super_gpqa',
+    # 'live_code_bench',
+    'mmlu_redux',
+    'simple_qa',
+    'chinese_simpleqa',
+    'alpaca_eval',
+    'arena_hard',
+    'maritime_bench',
+    'drop',
+    'winogrande',
+    'tool_bench',
+    'frames',
+    'docmath',
+    'needle_haystack',
+    'bfcl_v3',
+    'hle',
+    'tau_bench',
 ]
 
 # Reverse the datasets list to ensure the order is from most recent to oldest
@@ -82,8 +82,7 @@ dataset_args={
     'bbh': {
         'subset_list': ['word_sorting', 'movie_recommendation'],
     },
-    'gpqa': {
-        'subset_list': ['gpqa_diamond'],
+    'gpqa_diamond': {
         'few_shot_num': 0,
     },
     'humaneval': {
@@ -112,8 +111,7 @@ dataset_args={
         'subset_list': [
             'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
             # 'test'
-        ],
-        'metric_list': ['AverageBLEU']
+        ]
     },
     'super_gpqa': {
         'subset_list': ['Philosophy', 'Education'],
@@ -152,7 +150,6 @@ dataset_args={
 }
 
 class TestRun(unittest.TestCase):
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_benchmarks(self):
         from evalscope.config import TaskConfig
 
@@ -182,19 +179,60 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+    def test_vlm_benchmark(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-vl-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'mmmu',
+                # 'math_vista',
+            ],
+            dataset_args={
+                'mmmu': {
+                    'subset_list': ['Accounting']
+                },
+                'math_vista': {
+                    'subset_list': ['default']
+                }
+            },
+            eval_batch_size=1,
+            limit=1,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+                'image_height': 512,
+                'image_width': 512,
+                'image_num': 2,
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.AUTO,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+            }
+        )
+
+        run_task(task_cfg=task_cfg)
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_ci_lite(self):
         from evalscope.config import TaskConfig
 
+        api_key = env.get('DASHSCOPE_API_KEY')
+
         task_cfg = TaskConfig(
             model='qwen-plus',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
+            api_key=api_key,
+            eval_type=EvalType.SERVICE if api_key else EvalType.MOCK_LLM,
             datasets=[
                 'general_mcq',
-                'general_qa',
                 'iquiz',
             ],
             dataset_args={

tests/cli/test_collection.py

@@ -1,3 +1,6 @@
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
 import json
 import os
 import unittest
@@ -15,7 +18,6 @@ class TestCollection(unittest.TestCase):
                     CollectionSchema(name='math', datasets=[
                         CollectionSchema(name='generation', datasets=[
                             DatasetInfo(name='gsm8k', weight=1, task_type='math', tags=['en', 'math']),
-                            DatasetInfo(name='competition_math', weight=1, task_type='math', tags=['en', 'math']),
                         ]),
                         CollectionSchema(name='multiple_choice', datasets=[
                             DatasetInfo(name='cmmlu', weight=2, task_type='math', tags=['zh', 'math'], args={'subset_list': ['college_mathematics', 'high_school_mathematics']}),
@@ -45,15 +47,25 @@ class TestCollection(unittest.TestCase):
         from evalscope import TaskConfig, run_task
 
         task_cfg = TaskConfig(
-            model='Qwen2.5-0.5B-Instruct',
-            api_url='http://127.0.0.1:8801/v1/chat/completions',
-            api_key='EMPTY',
+            model='qwen-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key=env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=['data_collection'],
-            dataset_args={'data_collection': {
-                'local_path': 'outputs/mixed_data_test.jsonl'
-                # 'local_path': 'outputs/weighted_mixed_data.jsonl'
-            }},
+            dataset_args={
+                'data_collection': {
+                    # 'local_path': 'outputs/test_mix.jsonl'
+                    'local_path': 'outputs/mixed_data_test.jsonl',
+                    'shuffle': True,
+                }
+            },
+            eval_batch_size=5,
+            generation_config = {
+                'max_tokens': 10000,
+                'temperature': 0.0,
+            },
+            limit=10,
+            # use_cache='outputs/20250822_161804'
         )
         run_task(task_cfg=task_cfg)
 

tests/cli/test_custom.py

@@ -10,7 +10,7 @@ import subprocess
 import unittest
 
 from evalscope.config import TaskConfig
-from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
 from evalscope.run import run_task
 from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
@@ -120,7 +120,7 @@ class TestRunCustom(unittest.TestCase):
         from evalscope.config import TaskConfig
 
         task_cfg = TaskConfig(
-            model='qwen2.5-72b-instruct',
+            model='qwen2.5-7b-instruct',
             api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
@@ -132,7 +132,7 @@ class TestRunCustom(unittest.TestCase):
                     'dataset_id': 'custom_eval/text/qa',
                     'subset_list': [
                         'arena',
-                        'example'
+                        # 'example'
                     ],
                 }
             },
@@ -147,7 +147,7 @@ class TestRunCustom(unittest.TestCase):
             },
             ignore_errors=False,
             judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
+                'model_id': 'qwen2.5-7b-instruct',
                 'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
                 'api_key': env.get('DASHSCOPE_API_KEY'),
                 'generation_config': {
@@ -155,9 +155,19 @@ class TestRunCustom(unittest.TestCase):
                     'max_tokens': 4096
                 },
                 'score_type': 'numeric',
+                'prompt_template': """Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response.
+Begin your evaluation by providing a short explanation. Be as objective as possible.
+After providing your explanation, you must rate the response on a scale of 0 (worst) to 100 (best) by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\"
+
+[Question]
+{question}
+
+[Response]
+{pred}
+"""
             },
             judge_worker_num=5,
-            judge_strategy=JudgeStrategy.AUTO,
+            judge_strategy=JudgeStrategy.LLM,
         )
 
         run_task(task_cfg=task_cfg)
@@ -203,8 +213,9 @@ class TestRunCustom(unittest.TestCase):
                 },
                 'score_type': 'pattern',
             },
-            judge_worker_num=5,
-            judge_strategy=JudgeStrategy.LLM,
+            judge_worker_num=1,
+            judge_strategy=JudgeStrategy.LLM_RECALL,
+            use_cache='outputs/20250818_170420'
         )
 
         run_task(task_cfg=task_cfg)
@@ -223,20 +234,16 @@ class TestRunCustom(unittest.TestCase):
                 'general_arena': {
                     'extra_params':{
                         'models':[
-                            {
-                                'name': 'qwen2.5-0.5b',
-                                'report_path': 'outputs/20250702_140354/reports/qwen2.5-0.5b-instruct'
-                            },
                             {
                                 'name': 'qwen2.5-7b',
-                                'report_path': 'outputs/20250702_140702/reports/qwen2.5-7b-instruct'
+                                'report_path': 'outputs/20250819_165034/reports/qwen2.5-7b-instruct'
                             },
                             {
                                 'name': 'qwen2.5-72b',
-                                'report_path': 'outputs/20250702_140802/reports/qwen2.5-72b-instruct'
+                                'report_path': 'outputs/20250819_164926/reports/qwen2.5-72b-instruct'
                             }
                         ],
-                        'baseline': 'qwen2.5-7b'
+                        'baseline': 'qwen2.5-72b'
                     }
                 }
             },
@@ -255,7 +262,7 @@ class TestRunCustom(unittest.TestCase):
                 },
             },
             judge_worker_num=5,
-            use_cache='outputs/20250702_165727'
+            # use_cache='outputs/20250819_173546'
         )
 
         run_task(task_cfg=task_cfg)

tests/cli/test_reasoning.py

@@ -0,0 +1,81 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestReasoning(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'Qwen3-0.6B',
+            'api_url': 'http://0.0.0.0:8801/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True,
+                'extra_body':{'chat_template_kwargs': {'enable_thinking': False}} # 关闭思考模式
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)
+
+    # Math & Reasoning datasets
+    def test_gsm8k(self):
+        """Test GSM8K math reasoning dataset."""
+        self._run_dataset_test('gsm8k')
+
+
+if __name__ == '__main__':
+    # Run specific test: python -m unittest test_eval.TestBenchmark.test_gsm8k
+    # Run all tests: python -m unittest test_eval.TestBenchmark
+    unittest.main()

tests/common.py

@@ -0,0 +1,73 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+env = dotenv_values('.env')
+
+import unittest
+from unittest import TestCase
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalType, JudgeStrategy
+from evalscope.run import run_task
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestBenchmark(TestCase):
+    """Benchmark evaluation test cases."""
+
+    def setUp(self):
+        """Setup common test configuration."""
+        self.base_config = {
+            'model': 'qwen-plus',
+            'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+            'api_key': env.get('DASHSCOPE_API_KEY'),
+            'eval_type': EvalType.SERVICE,
+            'eval_batch_size': 5,
+            'limit': 5,
+            'generation_config': {
+                'max_tokens': 4096,
+                'temperature': 0.0,
+                'seed': 42,
+                'parallel_tool_calls': True
+            },
+            'judge_strategy': JudgeStrategy.AUTO,
+            'judge_worker_num': 5,
+            'judge_model_args': {
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096,
+                }
+            },
+            'debug': True,
+        }
+
+    def _run_dataset_test(self, dataset_name, dataset_args=None, use_mock=False, **config_overrides):
+        """Helper method to run test for a specific dataset."""
+        config = self.base_config.copy()
+        config['datasets'] = [dataset_name]
+
+        if not env.get('DASHSCOPE_API_KEY'):
+            use_mock = True
+            logger.warning('DASHSCOPE_API_KEY is not set. Using mock evaluation.')
+
+        if use_mock:
+            config['eval_type'] = EvalType.MOCK_LLM
+
+        # 应用配置覆盖
+        config.update(config_overrides)
+
+        if dataset_args:
+            config['dataset_args'] = {dataset_name: dataset_args}
+
+        task_cfg = TaskConfig(**config)
+        run_task(task_cfg=task_cfg)
+
+    def _run_dataset_load_test(self, dataset_name, dataset_args=None):
+        """Helper method to test dataset loading."""
+
+        self._run_dataset_test(dataset_name, dataset_args, use_mock=True, limit=None)

tests/perf/test_perf.py

@@ -1,9 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 from dotenv import dotenv_values
 
 env = dotenv_values('.env')
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 import unittest
 
 from evalscope.perf.main import run_perf_benchmark
@@ -123,6 +121,10 @@ class TestPerf(unittest.TestCase):
 
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_perf_multi_parallel(self):
+        if not env.get('DASHSCOPE_API_KEY'):
+            self.skipTest('DASHSCOPE_API_KEY is not set.')
+            return
+
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
             parallel=[1, 2],

tests/rag/test_clip_benchmark.py

@@ -1,7 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
 import os
-# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
 import subprocess
 import unittest
 

evalscope/benchmarks/aigc/t2i/base.py

@@ -1,56 +0,0 @@
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import DataAdapter
-from evalscope.metrics import mean, metric_registry
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-class T2IBaseAdapter(DataAdapter):
-
-    def __init__(self, **kwargs):
-
-        super().__init__(**kwargs)
-
-        logger.info(f'Initializing metrics: {self.metric_list}')
-        self.metrics = {m: metric_registry.get(m).object() for m in self.metric_list}
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        # dummy prompt for general t2i
-        return self.gen_prompt_data(prompt=input_d.get('prompt', ''), id=input_d.get('id', 0))
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # dummy gold answer for general t2i
-        return input_d.get('prompt', '')
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        # dummy parse pred result for general t2i
-        return result or raw_input_d.get('image_path', '')
-
-    def match(self, gold: str, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            score = metric_func(images=[pred], texts=[gold])[0][0]
-            if isinstance(score, dict):
-                for k, v in score.items():
-                    res[f'{metric_name}_{k}'] = v.cpu().item()
-            else:
-                res[metric_name] = score.cpu().item()  # Updated to use score.cpu().item()
-        return res
-
-    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [score1, score2, ...]
-
-        Returns:
-            avg_res: List[dict]
-
-        """
-        items = super().compute_dict_metric(review_res_list, **kwargs)
-        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]

evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py

@@ -1,78 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os.path
-from collections import defaultdict
-from typing import List, Optional, Union
-
-from evalscope.benchmarks import Benchmark
-from evalscope.constants import OutputType
-from evalscope.metrics import mean
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-from .base import T2IBaseAdapter
-
-logger = get_logger()
-
-
-@Benchmark.register(
-    name='evalmuse',
-    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
-    model_adapter=OutputType.IMAGE_GENERATION,
-    output_types=[OutputType.IMAGE_GENERATION],
-    subset_list=['EvalMuse'],
-    metric_list=['FGA_BLIP2Score'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-)
-class EvalMuseAdapter(T2IBaseAdapter):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load(self, **kwargs) -> dict:
-        if os.path.isfile(self.dataset_id):
-            data_list = jsonl_to_list(self.dataset_id)
-            data_dict = {self.subset_list[0]: {'test': data_list}}
-            return data_dict
-        else:
-            return super().load(**kwargs)
-
-    def get_gold_answer(self, input_d: dict) -> dict:
-        # return prompt and elements dict
-        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
-
-    def match(self, gold: dict, pred: str) -> dict:
-        # dummy match for general t2i
-        # pred is the image path, gold is the prompt
-        res = {}
-        for metric_name, metric_func in self.metrics.items():
-            if metric_name == 'FGA_BLIP2Score':
-                # For FGA_BLIP2Score, we need to pass the dictionary
-                score = metric_func(images=[pred], texts=[gold])[0][0]
-            else:
-                score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
-            if isinstance(score, dict):
-                for k, v in score.items():
-                    res[f'{metric_name}:{k}'] = v.cpu().item()
-            else:
-                res[metric_name] = score.cpu().item()
-        return res
-
-    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-        """
-        items = super().compute_dict_metric(review_res_list, **kwargs)
-        # add statistics for each metric
-        new_items = defaultdict(list)
-        for metric_name, value_list in items.items():
-            if 'FGA_BLIP2Score' in metric_name and '(' in metric_name:  # FGA_BLIP2Score element score
-                metrics_prefix = metric_name.split(':')[0]
-                category = metric_name.rpartition('(')[-1].split(')')[0]
-                category = category.split('-')[0].lower()  # remove the suffix if exists
-                new_items[f'{metrics_prefix}:{category}'].extend(value_list)
-            else:
-                new_items[metric_name].extend(value_list)
-
-        # calculate mean for each metric
-        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in new_items.items()]