evalscope 0.16.2__py3-none-any.whl → 0.17.0__py3-none-any.whl

tests/cli/test_custom.py

@@ -0,0 +1,261 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dotenv import dotenv_values
+
+from tests.utils import test_level_list
+
+env = dotenv_values('.env')
+
+import os
+import subprocess
+import unittest
+
+from evalscope.config import TaskConfig
+from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
+from evalscope.run import run_task
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.logger import get_logger
+
+os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
+
+logger = get_logger()
+
+
+class TestRunCustom(unittest.TestCase):
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_custom_task(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='Qwen/Qwen3-0.6B',
+            datasets=[
+                'general_mcq',
+                'general_qa'
+            ],
+            dataset_args={
+                'general_mcq': {
+                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ],
+                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
+                },
+                'general_qa': {
+                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
+                    'subset_list': [
+                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
+                    ]
+                }
+            },
+        )
+        res = run_task(task_cfg=task_cfg)
+        print(res)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_local_dataset(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                # 'mmlu',
+                # 'race',
+                'trivia_qa',
+                # 'cmmlu',
+                # 'humaneval',
+                # 'gsm8k',
+                # 'bbh',
+                # 'competition_math',
+                # 'arc',
+                # 'ceval',
+            ],
+            dataset_args={
+                'mmlu': {
+                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
+                    'few_shot_num': 0,
+                    'dataset_id': 'data/data/mmlu',
+                },
+                'ceval': {
+                    'subset_list': [
+                        'computer_network', 'operating_system', 'computer_architecture'
+                    ],
+                    'few_shot_num': 0,
+                    'dataset_id': 'data/data/ceval',
+                },
+                'cmmlu': {
+                    'subset_list': ['elementary_chinese'],
+                    'dataset_id': 'data/data/cmmlu',
+                    'few_shot_num': 0
+                },
+                'bbh': {
+                    'subset_list': ['word_sorting', 'movie_recommendation'],
+                },
+                'humaneval': {
+                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
+                },
+                'trivia_qa': {
+                    'dataset_id': 'data/data/trivia_qa',
+                },
+            },
+            eval_batch_size=10,
+            limit=5,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_no_answer(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen2.5-72b-instruct',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'general_qa',
+            ],
+            dataset_args={
+                'general_qa': {
+                    'dataset_id': 'custom_eval/text/qa',
+                    'subset_list': [
+                        'arena',
+                        'example'
+                    ],
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096
+                },
+                'score_type': 'numeric',
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.AUTO,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_with_answer(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model='qwen-plus',
+            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
+            api_key= env.get('DASHSCOPE_API_KEY'),
+            eval_type=EvalType.SERVICE,
+            datasets=[
+                'general_qa',
+            ],
+            dataset_args={
+                'general_qa': {
+                    'dataset_id': 'custom_eval/text/qa',
+                    'subset_list': [
+                        'example'
+                    ],
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            generation_config={
+                'temperature': 0,
+                'n': 1,
+                'max_tokens': 4096,
+            },
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen2.5-72b-instruct',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 4096
+                },
+                'score_type': 'pattern',
+            },
+            judge_worker_num=5,
+            judge_strategy=JudgeStrategy.LLM,
+        )
+
+        run_task(task_cfg=task_cfg)
+
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_general_arena(self):
+        from evalscope.config import TaskConfig
+
+        task_cfg = TaskConfig(
+            model_id='Arena',
+            datasets=[
+                'general_arena',
+            ],
+            dataset_args={
+                'general_arena': {
+                    'extra_params':{
+                        'models':[
+                            {
+                                'name': 'qwen2.5-0.5b',
+                                'report_path': 'outputs/20250702_140354/reports/qwen2.5-0.5b-instruct'
+                            },
+                            {
+                                'name': 'qwen2.5-7b',
+                                'report_path': 'outputs/20250702_140702/reports/qwen2.5-7b-instruct'
+                            },
+                            {
+                                'name': 'qwen2.5-72b',
+                                'report_path': 'outputs/20250702_140802/reports/qwen2.5-72b-instruct'
+                            }
+                        ],
+                        'baseline': 'qwen2.5-7b'
+                    }
+                }
+            },
+            eval_batch_size=10,
+            limit=10,
+            debug=True,
+            stream=True,
+            ignore_errors=False,
+            judge_model_args={
+                'model_id': 'qwen-plus',
+                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+                'api_key': env.get('DASHSCOPE_API_KEY'),
+                'generation_config': {
+                    'temperature': 0.0,
+                    'max_tokens': 8000
+                },
+            },
+            judge_worker_num=5,
+            use_cache='outputs/20250702_165727'
+        )
+
+        run_task(task_cfg=task_cfg)

tests/cli/test_run.py

@@ -1,6 +1,8 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 
 import os
@@ -8,9 +10,9 @@ import subprocess
 import unittest
 
 from evalscope.config import TaskConfig
-from evalscope.constants import EvalType, JudgeStrategy, OutputType
+from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
@@ -182,35 +184,6 @@ class TestRun(unittest.TestCase):
         run_task(task_cfg=task_cfg)
 
 
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_custom_task(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='Qwen/Qwen3-0.6B',
-            datasets=[
-                'general_mcq',
-                'general_qa'
-            ],
-            dataset_args={
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ]
-                }
-            },
-        )
-        res = run_task(task_cfg=task_cfg)
-        print(res)
-
     @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
     def test_run_one_task(self):
         from evalscope.config import TaskConfig
@@ -286,7 +259,7 @@ class TestRun(unittest.TestCase):
             api_key= env.get('DASHSCOPE_API_KEY'),
             eval_type=EvalType.SERVICE,
             datasets=[
-                # 'iquiz',
+                'iquiz',
                 # 'ifeval',
                 # 'mmlu',
                 # 'mmlu_pro',
@@ -305,7 +278,7 @@ class TestRun(unittest.TestCase):
                 # 'arc',
                 # 'ceval',
                 # 'hellaswag',
-                'general_mcq',
+                # 'general_mcq',
                 # 'general_qa',
                 # 'super_gpqa',
                 # 'mmlu_redux',
@@ -315,6 +288,7 @@ class TestRun(unittest.TestCase):
                 # 'tool_bench',
                 # 'frames',
                 # 'bfcl_v3',
+                # 'truthful_qa',
             ],
             dataset_args={
                 'mmlu': {
@@ -354,7 +328,6 @@ class TestRun(unittest.TestCase):
                 },
                 'musr': {
                     'subset_list': ['murder_mysteries'],
-                    'local_path': '/root/.cache/modelscope/hub/datasets/AI-ModelScope/MuSR'
                 },
                 'general_mcq': {
                     'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
@@ -378,6 +351,9 @@ class TestRun(unittest.TestCase):
                 'mmlu_redux':{
                     'subset_list': ['abstract_algebra']
                 },
+                'frames':{
+                    'local_path': 'data/iic/frames',
+                },
                 'bfcl_v3': {
                     'subset_list': ['parallel'],
                     'extra_params': {
@@ -385,9 +361,9 @@ class TestRun(unittest.TestCase):
                     }
                 },
             },
-            eval_batch_size=10,
+            eval_batch_size=1,
             limit=5,
-            debug=True,
+            # debug=True,
             stream=True,
             generation_config={
                 'temperature': 0,
@@ -396,7 +372,6 @@ class TestRun(unittest.TestCase):
                 # 'extra_headers':{'key': 'value'},
             },
             ignore_errors=False,
-            # use_cache='outputs/20250616_153756'
         )
 
         run_task(task_cfg=task_cfg)
@@ -521,5 +496,6 @@ class TestRun(unittest.TestCase):
 
         run_task(task_cfg=task_cfg)
 
+
 if __name__ == '__main__':
     unittest.main()

tests/perf/test_perf.py

@@ -7,7 +7,7 @@ os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 import unittest
 
 from evalscope.perf.main import run_perf_benchmark
-from evalscope.utils import test_level_list
+from tests.utils import test_level_list
 
 
 class TestPerf(unittest.TestCase):
@@ -126,7 +126,7 @@ class TestPerf(unittest.TestCase):
         from evalscope.perf.arguments import Arguments
         task_cfg = Arguments(
             parallel=[1, 2],
-            number=[2, 5],
+            number=[2, 4],
             model='qwen2.5-7b-instruct',
             url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
             api_key=env.get('DASHSCOPE_API_KEY'),

tests/rag/test_clip_benchmark.py

@@ -6,8 +6,9 @@ import subprocess
 import unittest
 
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger()
 

tests/rag/test_mteb.py

@@ -3,9 +3,11 @@
 import unittest
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 from evalscope.run import run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

tests/rag/test_ragas.py

@@ -2,11 +2,13 @@
 import os
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 import unittest
 
 from evalscope import TaskConfig, run_task
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()

tests/swift/test_run_swift_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.opencompass import OpenCompassBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/swift/test_run_swift_vlm_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/swift/test_run_swift_vlm_jugde_eval.py

@@ -10,8 +10,9 @@ import unittest
 from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
+from tests.utils import test_level_list
 
 logger = get_logger(__name__)
 

tests/utils.py

@@ -0,0 +1,13 @@
+import os
+
+TEST_LEVEL_LIST = [0, 1]
+# Example: export TEST_LEVEL_LIST=0,1
+TEST_LEVEL_LIST_STR = 'TEST_LEVEL_LIST'
+
+
+def test_level_list():
+    global TEST_LEVEL_LIST
+    if TEST_LEVEL_LIST_STR in os.environ:
+        TEST_LEVEL_LIST = [int(x) for x in os.environ[TEST_LEVEL_LIST_STR].split(',')]
+
+    return TEST_LEVEL_LIST

tests/vlm/test_vlmeval.py

@@ -1,12 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 from dotenv import dotenv_values
 
+from tests.utils import test_level_list
+
 env = dotenv_values('.env')
 import unittest
 
 from evalscope.run import run_task
 from evalscope.summarizer import Summarizer
-from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.import_utils import is_module_installed
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -62,7 +64,11 @@ class TestVLMEval(unittest.TestCase):
         task_cfg = {
             'eval_backend': 'VLMEvalKit',
             'eval_config': {
-                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'data': [
+                    # 'SEEDBench_IMG',
+                    # 'ChartQA_TEST',
+                    'MMDU'
+                    ],
                 'limit': 5,
                 'mode': 'all',
                 'model': [

evalscope/evaluator/rating_eval.py

@@ -1,157 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-import pandas as pd
-import pyarrow as pa
-from typing import List, Union
-
-from evalscope.constants import MetricMembers
-from evalscope.utils.arena_utils import compute_elo
-from evalscope.utils.io_utils import jsonl_to_list
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-DEFAULT_COLUMNS_MAPPING = {'model_a': 'model_a', 'model_b': 'model_b', 'win': 'win', 'tstamp': 'ts', 'language': 'lang'}
-
-
-class RatingEvaluate(object):
-
-    def __init__(self, metrics: list, baseline_model: str = None, **kwargs):
-        self.metrics = metrics
-        self.baseline_model = baseline_model
-        self.kwargs = kwargs
-
-    def preprocess(self, raw_data_df: pd.DataFrame, **kwargs):
-
-        # Get battles data
-        raw_data_df = raw_data_df.sort_values(ascending=True, by=['tstamp'])
-        battles = raw_data_df[raw_data_df['anony']].reset_index(drop=True)
-
-        return battles
-
-    def compute_elo_rating(self, raw_data):
-        battles = self.preprocess(raw_data_df=raw_data)
-        elo_ratings = compute_elo(battles)
-        col_model = 'Model'
-        col_elo_rating = 'Elo_Rating'
-        elo_ratings_res = pd.DataFrame([[n, elo_ratings[n]] for n in elo_ratings.keys()],
-                                       columns=[col_model, col_elo_rating]).sort_values(
-                                           col_elo_rating, ascending=False).reset_index(drop=True)
-        elo_ratings_res = elo_ratings_res.round({col_elo_rating: 1})
-        return elo_ratings_res
-
-    def get_single_pairwise_rating(self, row: pd.Series):
-        tie = False
-        if 'win' in row:
-            win = row['win']
-            if win == 'tie':
-                tie = True
-            else:
-                if win == 'model_a':
-                    winner = row['model_a']
-                    loser = row['model_b']
-                else:
-                    winner = row['model_b']
-                    loser = row['model_a']
-        elif 'win_1' in row:
-            win_1 = row['win_1']
-            win_2 = row['win_2']
-            if win_1 == 'tie' or win_1 != win_2:
-                tie = True
-            else:
-                if win_1 == 'model_a':
-                    winner = row['model_a']
-                    loser = row['model_b']
-                else:
-                    winner = row['model_b']
-                    loser = row['model_a']
-        else:
-            raise ValueError('Unsupported data format')
-
-        if tie:
-            return [{
-                'model': row['model_a'],
-                'win': 0,
-                'loss': 0,
-                'tie': 1
-            }, {
-                'model': row['model_b'],
-                'win': 0,
-                'loss': 0,
-                'tie': 1
-            }]
-        else:
-            return [{'model': winner, 'win': 1, 'loss': 0, 'tie': 0}, {'model': loser, 'win': 0, 'loss': 1, 'tie': 0}]
-
-    def compute_pairwise_rating(self, raw_data):
-        df_all = self.preprocess(raw_data_df=raw_data)
-        model_list = (df_all['model_a'].unique().tolist() + df_all['model_b'].unique().tolist())
-        model_list = list(set(model_list))
-
-        list_res = []
-        # traverse df row by row
-        for index, row in df_all.iterrows():
-            if self.baseline_model is not None:
-                if self.baseline_model not in [row['model_a'], row['model_b']]:
-                    logger.warning(
-                        f'One of the models in the battle should be the baseline model: {self.baseline_model}')
-                    continue
-            rating = self.get_single_pairwise_rating(row)
-            list_res = list_res + rating
-
-        df = pd.DataFrame(list_res)
-        df = df.groupby(['model']).sum()
-
-        # remove baseline model
-        if self.baseline_model is not None:
-            df = df[df.index != self.baseline_model]
-        # add win rate
-        df['win_rate'] = df['win'] / (df['win'] + df['loss'] + df['tie'])
-        df['loss_rate'] = df['loss'] / (df['win'] + df['loss'] + df['tie'])
-        df['tie_rate'] = df['tie'] / (df['win'] + df['loss'] + df['tie'])
-        return df.sort_values(by='win_rate', ascending=False)
-
-    def compute_score_rating(self, raw_data):
-        df_all = self.preprocess(raw_data_df=raw_data)
-        df = df_all[['model', 'score']]
-
-        df_score = df.groupby(['model']).mean()
-        return df_score.sort_values(by='score', ascending=False)
-
-    def eval_samples(self, data_list: list):
-        res_all = []
-
-        raw_data: pd.DataFrame = None
-
-        if len(data_list) > 0:
-            raw_data = data_list[0]
-
-        for metric in self.metrics:
-
-            if metric == MetricMembers.ELO:
-                res = self.compute_elo_rating(raw_data)
-                res_all.append(res)
-
-            elif metric == MetricMembers.PAIRWISE:
-                res = self.compute_pairwise_rating(raw_data)
-                res_all.append(res)
-
-            elif metric == MetricMembers.SCORE:
-                res = self.compute_score_rating(raw_data)
-                res_all.append(res)
-
-            else:
-                raise ValueError(f'Unsupported metric: {metric}')
-
-        return res_all
-
-    def run(self, prompts: Union[str, list], **kwargs) -> List[pd.DataFrame]:
-        """
-        Load the predicted samples and evaluate them in arena mode.
-        """
-        # raw_data = pd.read_json(prompts)
-        data_list = jsonl_to_list(prompts)
-        data_df = pa.Table.from_pylist(data_list).to_pandas()
-        res_list = self.eval_samples([data_df])
-
-        return res_list

evalscope/evaluator/reviewer/__init__.py

@@ -1 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.