evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/models/local_model.py

@@ -1,128 +0,0 @@
-import importlib
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Optional
-
-from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType, ModelTask
-from evalscope.utils.logger import get_logger
-from evalscope.utils.model_utils import get_device
-
-if TYPE_CHECKING:
-    from evalscope.config import TaskConfig
-
-logger = get_logger()
-
-
-class LocalModel(ABC):
-
-    def __init__(self,
-                 model_id: str,
-                 model_revision: str = None,
-                 device_map: str = None,
-                 torch_dtype: str = 'auto',
-                 cache_dir: str = None,
-                 **kwargs):
-
-        self.model_id = model_id
-        self.model_revision = model_revision or DEFAULT_MODEL_REVISION
-        self.device = device_map or get_device()
-        self.cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
-        self.kwargs = kwargs
-        self.model = None
-        self.tokenizer = None
-
-        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
-            import torch
-            torch_dtype = eval(torch_dtype)
-        self.torch_dtype = torch_dtype
-
-        self.model_cfg = {
-            'model_id': self.model_id,
-            'device_map': self.device,
-            'torch_dtype': str(self.torch_dtype),
-        }
-
-    @abstractmethod
-    def load_model(self):
-        pass
-
-
-class LocalChatModel(LocalModel):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-    def load_model(self):
-        from modelscope import AutoModelForCausalLM, AutoTokenizer
-
-        logger.info(f'Loading model {self.model_id} ...')
-
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            self.model_id,
-            revision=self.model_revision,
-            trust_remote_code=True,
-            cache_dir=self.cache_dir,
-        )
-
-        # Fix no padding
-        if self.tokenizer.pad_token is None:
-            self.tokenizer.pad_token = self.tokenizer.eos_token
-
-        self.model = AutoModelForCausalLM.from_pretrained(
-            self.model_id,
-            revision=self.model_revision,
-            device_map=self.device,
-            trust_remote_code=True,
-            torch_dtype=self.torch_dtype,
-            cache_dir=self.cache_dir,
-        )
-
-
-class LocalImageModel(LocalModel):
-
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        self.pipeline_cls = self.kwargs.pop('pipeline_cls', None)
-        # default to DiffusionPipeline if not specified
-        if self.pipeline_cls is None:
-            if 'flux' in self.model_id.lower():
-                self.pipeline_cls = 'FluxPipeline'
-            else:
-                self.pipeline_cls = 'DiffusionPipeline'
-
-    def load_model(self):
-        # from modelscope import pipeline_cls
-        module = getattr(importlib.import_module('modelscope'), self.pipeline_cls)
-
-        logger.info(f'Loading model {self.model_id} with {self.pipeline_cls} ...')
-
-        self.model = module.from_pretrained(
-            self.model_id,
-            revision=self.model_revision,
-            torch_dtype=self.torch_dtype,
-            cache_dir=self.cache_dir,
-            **self.kwargs,
-        )
-
-        self.model.to(self.device)
-
-    def __call__(self, *args, **kwargs):
-        return self.model(*args, **kwargs)
-
-
-def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
-    """Get the base local model for the task. If the task is not checkpoint-based, return None.
-       Avoids loading model multiple times for different datasets.
-    """
-    if task_cfg.eval_type != EvalType.CHECKPOINT:
-        return None
-    elif task_cfg.model_task == ModelTask.TEXT_GENERATION:
-        base_model = LocalChatModel(model_id=task_cfg.model, **task_cfg.model_args)
-        base_model.load_model()
-        return base_model
-    elif task_cfg.model_task == ModelTask.IMAGE_GENERATION:
-        base_model = LocalImageModel(model_id=task_cfg.model, **task_cfg.model_args)
-        base_model.load_model()
-        return base_model
-    else:
-        raise ValueError(f'Unsupported model task: {task_cfg.model_task} for model checkpoint.')

evalscope/models/register.py

@@ -1,41 +0,0 @@
-MODEL_ADAPTERS = {}
-
-
-def register_model_adapter(name):
-    """
-    Decorator to register a model adapter with a given name.
-    :param name: The name of the model adapter.
-    """
-
-    def decorator(adapter):
-        if name in MODEL_ADAPTERS:
-            raise ValueError(f"Model adapter '{name}' is already registered.")
-        MODEL_ADAPTERS[name] = adapter
-        return adapter
-
-    return decorator
-
-
-def get_model_adapter(name):
-    """
-    Retrieve a registered model adapter by name.
-    :param name: The name of the model adapter.
-    :return: The model adapter class or function.
-    """
-    if name not in MODEL_ADAPTERS:
-        raise ValueError(
-            f"Model adapter '{name}' is not registered. Available model adapters: {list(MODEL_ADAPTERS.keys())}")
-    return MODEL_ADAPTERS[name]
-
-
-def register_model_adapter_class(cls, name=None):
-    """
-    Register a model adapter class.
-    :param cls: The model adapter class to register
-    :param name: Optional name for the model adapter. If not provided, the class name will be used.
-    """
-    if name is None:
-        name = cls.__name__
-    if name in MODEL_ADAPTERS:
-        raise ValueError(f"Model adapter class '{name}' is already registered.")
-    MODEL_ADAPTERS[name] = cls

tests/cli/test_run.py

@@ -1,489 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-from dotenv import dotenv_values
-
-from tests.utils import test_level_list
-
-env = dotenv_values('.env')
-
-import os
-import subprocess
-import unittest
-
-from evalscope.config import TaskConfig
-from evalscope.constants import EvalStage, EvalType, JudgeStrategy, OutputType
-from evalscope.run import run_task
-from evalscope.utils.import_utils import is_module_installed
-from evalscope.utils.logger import get_logger
-
-os.environ['EVALSCOPE_LOG_LEVEL'] = 'DEBUG'
-
-logger = get_logger()
-
-
-class TestRun(unittest.TestCase):
-
-    def setUp(self) -> None:
-        logger.info('Init env for evalscope native run UTs ...\n')
-        self._check_env('evalscope')
-
-    def tearDown(self) -> None:
-        pass
-
-    @staticmethod
-    def _check_env(module_name: str):
-        if is_module_installed(module_name):
-            logger.info(f'{module_name} is installed.')
-        else:
-            raise ModuleNotFoundError(f'run: pip install {module_name}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_simple_eval(self):
-        model = 'qwen/Qwen2-0.5B-Instruct'
-        datasets = 'arc'  # arc ceval
-        limit = 10
-
-        cmd_simple = f'evalscope eval ' \
-                     f'--model {model} ' \
-                     f'--datasets {datasets} ' \
-                     f'--limit {limit}'
-
-        logger.info(f'Start to run command: {cmd_simple}')
-        run_res = subprocess.run(cmd_simple, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        assert run_res.returncode == 0, f'Failed to run command: {cmd_simple}'
-        logger.info(f'>>test_run_simple_eval stdout: {run_res.stdout}')
-        logger.error(f'>>test_run_simple_eval stderr: {run_res.stderr}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_eval_with_args(self):
-        model = 'qwen/Qwen2-0.5B-Instruct'
-        datasets = 'arc'  # arc ceval
-        limit = 5
-        dataset_args = '{"ceval": {"few_shot_num": 0, "few_shot_random": false}}'
-
-        cmd_with_args = f'evalscope eval ' \
-                        f'--model {model} ' \
-                        f'--datasets {datasets} ' \
-                        f'--limit {limit} ' \
-                        f'--generation-config do_sample=true,temperature=0.6,max_length=65535,max_new_tokens=65535,max_tokens=65535,n=1,top_p=0.95,top_k=20 ' \
-                        f"""--dataset-args \'{dataset_args}\' """
-
-        logger.info(f'Start to run command: {cmd_with_args}')
-        run_res = subprocess.run(cmd_with_args, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-
-        assert run_res.returncode == 0, f'Failed to run command: {cmd_with_args}'
-        logger.info(f'>>test_run_eval_with_args stdout: {run_res.stdout}')
-        logger.error(f'>>test_run_eval_with_args stderr: {run_res.stderr}')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_yaml_config(self):
-        from evalscope import run_task
-
-        run_task(task_cfg='examples/tasks/eval_native.yaml')
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_task(self):
-        task_cfg = TaskConfig(
-            model='qwen/Qwen2.5-0.5B-Instruct',
-            datasets=[
-                'iquiz',
-                # 'ifeval',
-                # 'mmlu',
-                # 'mmlu_pro',
-                # 'musr',
-                # 'process_bench',
-                # 'race',
-                # 'trivia_qa',
-                # 'cmmlu',
-                # 'humaneval',
-                # 'super_gpqa',
-                # 'gsm8k',
-                # 'bbh',
-                # 'competition_math',
-                # 'math_500',
-                'aime24',
-                'gpqa',
-                # 'arc',
-                # 'ceval',
-                # 'hellaswag',
-                # 'general_mcq',
-                # 'general_qa'
-            ],
-            dataset_args={
-                'mmlu': {
-                    'subset_list': ['elementary_mathematics'],
-                    'few_shot_num': 0
-                },
-                'mmlu_pro': {
-                    'subset_list': ['math', 'health'],
-                    'few_shot_num': 4
-                },
-                'ceval': {
-                    'subset_list': [
-                        'computer_network', 'operating_system', 'computer_architecture'
-                    ],
-                    'few_shot_num': 0
-                },
-                'cmmlu': {
-                    'subset_list': ['elementary_chinese'],
-                    'few_shot_num': 0
-                },
-                'bbh': {
-                    'subset_list': ['word_sorting', 'movie_recommendation'],
-                },
-                'gpqa': {
-                    'subset_list': ['gpqa_diamond'],
-                    'few_shot_num': 0
-                },
-                'humaneval': {
-                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
-                },
-                'competition_math': {
-                    'subset_list': ['Level 1']
-                },
-                'process_bench': {
-                    'subset_list': ['gsm8k'],
-                },
-                'musr': {
-                    'subset_list': ['murder_mysteries'],
-                },
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
-                        # 'test'
-                    ],
-                    'metric_list': ['AverageBLEU']
-                },
-                'super_gpqa': {
-                    'subset_list': ['Philosophy', 'Education'],
-                    'few_shot_num': 0
-                },
-                'ifeval': {
-                    'filters': {
-                        'remove_until': '</think>'
-                    }
-                }
-            },
-            limit=2,
-            eval_batch_size=2,
-            generation_config={
-                'max_new_tokens': 2048,
-                'temperature': 0.7,
-                'num_return_sequences': 1,
-            },
-            # debug=True
-        )
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_one_task(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='Qwen/Qwen3-1.7B',
-            datasets=[
-                # 'iquiz',
-                # 'math_500',
-                # 'aime24',
-                # 'competition_math',
-                # 'mmlu',
-                # 'simple_qa',
-                'truthful_qa',
-            ],
-            dataset_args={
-                'competition_math': {
-                    'subset_list': ['Level 4', 'Level 5']
-                },
-                'mmlu': {
-                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
-                    'few_shot_num': 0
-                },
-            },
-            limit=5,
-            eval_batch_size=5,
-            generation_config={
-                'max_new_tokens': 1000,  # 最大生成token数，建议设置为较大值避免输出截断
-                'temperature': 0.7,  # 采样温度 (qwen 报告推荐值)
-                'top_p': 0.8,  # top-p采样 (qwen 报告推荐值)
-                'top_k': 20,  # top-k采样 (qwen 报告推荐值)
-                'chat_template_kwargs': {'enable_thinking': False}  # 关闭思考模式
-            },
-            judge_strategy=JudgeStrategy.AUTO,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_task_loop(self):
-        os.environ['CUDA_VISIBLE_DEVICES'] = '2'
-        from evalscope.config import TaskConfig
-
-        task_cfg1 = TaskConfig(
-            model='Qwen/Qwen2.5-0.5B-Instruct',
-            model_id='model1',
-            datasets=['iquiz'],
-            limit=10
-        )
-        task_cfg2 = TaskConfig(
-            model='Qwen/Qwen2.5-0.5B-Instruct',
-            model_id='model2',
-            datasets=['iquiz'],
-            limit=10
-        )
-        task_cfg3 = TaskConfig(
-            model='Qwen/Qwen2.5-0.5B-Instruct',
-            model_id='model3',
-            datasets=['iquiz'],
-            limit=10
-        )
-
-        run_task(task_cfg=[task_cfg1, task_cfg2, task_cfg3])
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_server_model(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                # 'iquiz',
-                # 'ifeval',
-                # 'mmlu',
-                # 'mmlu_pro',
-                # 'musr',
-                # 'process_bench',
-                # 'race',
-                'trivia_qa',
-                # 'cmmlu',
-                # 'humaneval',
-                # 'gsm8k',
-                # 'bbh',
-                # 'competition_math',
-                # 'math_500',
-                # 'aime24',
-                # 'gpqa',
-                # 'arc',
-                # 'ceval',
-                # 'hellaswag',
-                # 'general_mcq',
-                # 'general_qa',
-                # 'super_gpqa',
-                # 'mmlu_redux',
-                # 'maritime_bench',
-                # 'drop',
-                # 'winogrande',
-                # 'tool_bench',
-                # 'frames',
-                # 'bfcl_v3',
-                # 'truthful_qa',
-                # 'tau_bench',
-                # 'hle'
-            ],
-            dataset_args={
-                'mmlu': {
-                    'subset_list': ['elementary_mathematics', 'high_school_european_history', 'nutrition'],
-                    'few_shot_num': 0
-                },
-                'mmlu_pro': {
-                    'subset_list': ['math', 'health'],
-                    'few_shot_num': 0
-                },
-                'ceval': {
-                    'subset_list': [
-                        'computer_network', 'operating_system', 'computer_architecture'
-                    ],
-                    'few_shot_num': 0
-                },
-                'cmmlu': {
-                    'subset_list': ['elementary_chinese'],
-                    'few_shot_num': 0
-                },
-                'bbh': {
-                    'subset_list': ['word_sorting', 'movie_recommendation'],
-                },
-                'gpqa': {
-                    # 'subset_list': ['gpqa_diamond'],
-                    'few_shot_num': 0,
-                    'local_path': './data/data/gpqa',
-                },
-                'humaneval': {
-                    'metric_list': ['Pass@1', 'Pass@2', 'Pass@5'],
-                },
-                'competition_math': {
-                    'subset_list': ['Level 1']
-                },
-                'process_bench': {
-                    'subset_list': ['gsm8k'],
-                },
-                'musr': {
-                    'subset_list': ['murder_mysteries'],
-                },
-                'general_mcq': {
-                    'local_path': 'custom_eval/text/mcq',  # 自定义数据集路径
-                    'subset_list': [
-                        'example'  # 评测数据集名称，上述 *_dev.csv 中的 *
-                    ],
-                    'query_template': 'Question: {question}\n{choices}\nAnswer: {answer}'  # 问题模板
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
-                        # 'test'
-                    ],
-                    'metric_list': ['AverageRouge']
-                },
-                'super_gpqa': {
-                    'subset_list': ['Philosophy', 'Education'],
-                    'few_shot_num': 0
-                },
-                'mmlu_redux':{
-                    'subset_list': ['abstract_algebra']
-                },
-                'frames':{
-                    'local_path': 'data/iic/frames',
-                },
-                'bfcl_v3': {
-                    'subset_list': ['parallel'],
-                    'extra_params': {
-                        # 'is_fc_model': False,
-                    }
-                },
-                'tau_bench': {
-                    'extra_params': {
-                        'user_model': 'qwen-plus',
-                        'api_key': env.get('DASHSCOPE_API_KEY'),
-                        'api_base': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                    }
-                },
-                'hle': {
-                    'subset_list': ['Math', 'Other'],
-                },
-            },
-            eval_batch_size=10,
-            limit=10,
-            # debug=True,
-            stream=True,
-            generation_config={
-                'temperature': 0.6,
-                'n': 1,
-                'max_tokens': 4096,
-                # 'extra_headers':{'key': 'value'},
-            },
-            ignore_errors=False,
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-
-    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
-    def test_run_judge_model(self):
-        from evalscope.config import TaskConfig
-
-        task_cfg = TaskConfig(
-            model='qwen-plus',
-            api_url='https://dashscope.aliyuncs.com/compatible-mode/v1',
-            api_key= env.get('DASHSCOPE_API_KEY'),
-            eval_type=EvalType.SERVICE,
-            datasets=[
-                # 'math_500',
-                # 'aime24',
-                # 'competition_math',
-                # 'arc',
-                # 'gsm8k',
-                # 'truthful_qa',
-                # 'simple_qa',
-                # 'chinese_simpleqa',
-                # 'live_code_bench',
-                # 'humaneval',
-                # 'general_qa',
-                # 'alpaca_eval',
-                # 'arena_hard',
-                # 'frames',
-                # 'docmath',
-                # 'needle_haystack',
-                # 'ifeval',
-                'hle'
-            ],
-            dataset_args={
-                'needle_haystack': {
-                    'subset_list': ['english'],
-                    'extra_params': {
-                        'show_score': True,
-                    }
-                },
-                'competition_math': {
-                    'subset_list': ['Level 4']
-                },
-                'live_code_bench': {
-                    'extra_params': {
-                        'start_date': '2024-08-01',
-                        'end_date': '2025-02-28'
-                    },
-                    'local_path': '/root/.cache/modelscope/hub/datasets/AI-ModelScope/code_generation_lite'
-                },
-                'general_qa': {
-                    'local_path': 'custom_eval/text/qa',  # 自定义数据集路径
-                    'subset_list': [
-                        'example',  # 评测数据集名称，上述 *_dev.csv 中的 *
-                        # 'test'
-                    ]
-                },
-                'chinese_simpleqa': {
-                    'subset_list': [
-                        '中华文化'
-                    ]
-                },
-                'frames': {
-                    'local_path': '/root/.cache/modelscope/hub/datasets/iic/frames'
-                },
-                'hle': {
-                    'subset_list': ['Math', 'Other'],
-                },
-            },
-            eval_batch_size=10,
-            limit=3,
-            judge_strategy=JudgeStrategy.LLM,
-            judge_worker_num=5,
-            judge_model_args={
-                'model_id': 'qwen2.5-72b-instruct',
-                'api_url': 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-                'api_key': env.get('DASHSCOPE_API_KEY'),
-                'generation_config': {
-                    'temperature': 0.0,
-                    'max_tokens': 4096
-                }
-            },
-            generation_config={
-                'max_new_tokens': 20000,
-                'temperature': 0.0,
-                'seed': 42,
-                'n': 1
-            },
-            timeout=60000,
-            stream=True,
-            use_cache='outputs/20250714_150626'
-            # analysis_report=True,
-            # debug=True,
-            # use_cache='outputs/20250616_161931'
-        )
-
-        run_task(task_cfg=task_cfg)
-
-
-if __name__ == '__main__':
-    unittest.main()