evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/registry/tasks/ceval_mini.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -20,8 +20,7 @@ model: null   # Note: to be implemented as CustomModel
 eval_type: custom
 datasets:
   - ceval
-outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/cmmlu.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -23,5 +23,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/eval_qwen-7b-chat_v100.yaml

@@ -24,5 +24,5 @@ datasets:
 outputs: ./outputs/eval_qwen-7b-chat_v100    # Directory to save the outputs, structure: logs, predictions, reviews, reports
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
+dataset_hub: modelscope    # `Local` or `ModelScope`
 limit: 10

evalscope/registry/tasks/general_qa.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat

evalscope/registry/tasks/gsm8k.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -25,5 +25,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
+dataset_hub: modelscope    # `Local` or `ModelScope`
 limit: null

evalscope/registry/tasks/mmlu.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -25,5 +25,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: true
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/mmlu_mini.yaml

@@ -1,5 +1,5 @@
 model_args:    # model args should be followed by benchmark requirements
-  revision: default
+  revision: master
   precision: torch.float16
   device_map: auto
 #  model_name_or_path: qwen/qwen-7b-chat
@@ -23,5 +23,5 @@ datasets:
 outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
 use_cache: false
 stage: all
-dataset_hub: ModelScope    # `Local` or `ModelScope`
-limit: null
+dataset_hub: modelscope    # `Local` or `ModelScope`
+limit: null

evalscope/run.py

@@ -1,408 +1,180 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-# flake8: noqa
-import copy
-import json
-import argparse
+"""
+Run evaluation for LLMs.
+"""
+import logging
 import os.path
-from typing import Union, List
-import torch        # noqa
-
-from evalscope.config import TaskConfig
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+import torch
+from argparse import Namespace
+from datetime import datetime
+from typing import List, Optional, Union
+
+from evalscope.arguments import parse_args
+from evalscope.config import TaskConfig, parse_task_config
+from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType
 from evalscope.evaluator import Evaluator
-from evalscope.evaluator.evaluator import HumanevalEvaluator
 from evalscope.models.custom import CustomModel
-from evalscope.utils import import_module_util, yaml_to_dict, make_outputs_dir, gen_hash, json_to_dict, EvalBackend
-from evalscope.utils.logger import get_logger
+from evalscope.utils import import_module_util, seed_everything
+from evalscope.utils.io_utils import OutputsStructure, are_paths_same
+from evalscope.utils.logger import configure_logging, get_logger
 
 logger = get_logger()
 
-"""
-Run evaluation for LLMs.
-"""
-
 BENCHMARK_PATH_PREFIX = 'evalscope.benchmarks.'
 MEMBERS_TO_IMPORT = ['DATASET_ID', 'SUBSET_LIST', 'DataAdapterClass', 'ModelAdapterClass']
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(description='Run evaluation on benchmarks for LLMs.')
-
-    parser.add_argument('--model',
-                        help='The model id on modelscope, or local model dir.',
-                        type=str,
-                        # required=True,
-                        required=False,
-                        )
-    parser.add_argument('--model-type',
-                        help='Deprecated. See `--template-type`',
-                        type=str,
-                        required=False,
-                        default=None)
-    parser.add_argument('--template-type',
-                        type=str,
-                        help='The template type for generation, should be a string.'
-                             'Refer to `https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md` for more details.',
-                        required=False,
-                        )
-    parser.add_argument('--eval-type',
-                        type=str,
-                        help='The type for evaluating. '
-                             'service - for APIs, TO-DO'
-                             'checkpoint - for models on ModelScope or local model dir, '
-                             'custom - for custom models.'
-                             '         Need to set `--model` to evalscope.models.custom.CustomModel format.'
-                             'default to `checkpoint`.',
-                        required=False,
-                        default='checkpoint',
-                        )
-    parser.add_argument('--model-args',
-                        type=str,
-                        help='The model args, should be a string.',
-                        required=False,
-                        default='revision=None,precision=torch.float16,device_map=auto'
-                        )
-    parser.add_argument('--generation-config',
-                        type=str,
-                        help='The generation config, should be a string.',
-                        required=False,
-                        default='do_sample=False,repetition_penalty=1.0,max_new_tokens=512',
-                        )
-    parser.add_argument('--datasets',
-                        help='Dataset id list, align to the module name in evalscope.benchmarks',
-                        type=str,
-                        nargs='+',
-                        required=False,
-                        )
-    parser.add_argument('--dataset-args',
-                        type=json.loads,
-                        help='The dataset args, should be a json string. The key of dict should be aligned to datasets,'
-                             'e.g. {"humaneval": {"local_path": "/to/your/path"}}',
-                        required=False,
-                        default='{}')
-    parser.add_argument('--dataset-dir',
-                        help='The datasets dir. Use to specify the local datasets or datasets cache dir.'
-                             'See --dataset-hub for more details.',
-                        required=False,
-                        default=DEFAULT_ROOT_CACHE_DIR)
-    parser.add_argument('--dataset-hub',
-                        help='The datasets hub, can be `ModelScope` or `HuggingFace` or `Local`. '
-                             'Default to `ModelScope`.'
-                             'If `Local`, the --dataset-dir should be local input data dir.'
-                             'Otherwise, the --dataset-dir should be the cache dir for datasets.',
-                        required=False,
-                        default='ModelScope')
-    parser.add_argument('--outputs',
-                        help='Outputs dir. Default to `outputs`, which means dump to current path: ./outputs',
-                        required=False,
-                        default='outputs')
-    parser.add_argument('--work-dir',
-                        help='The root cache dir.',
-                        required=False,
-                        default=DEFAULT_ROOT_CACHE_DIR)
-    parser.add_argument('--limit',
-                        type=int,
-                        help='Max evaluation samples num for each subset. Default to None, which means no limit.',
-                        default=None)
-    parser.add_argument('--debug',
-                        help='Debug mode, will print information for debugging.',
-                        action='store_true',
-                        default=False)
-    parser.add_argument('--dry-run',
-                        help='Dry run in single processing mode.',
-                        action='store_true',
-                        default=False)
-    parser.add_argument('--mem-cache',
-                        help='To use memory cache or not.',
-                        action='store_true',
-                        default=False)
-    parser.add_argument('--use-cache',
-                        help='To reuse the cache or not. Default to `true`.',
-                        type=str,
-                        default='false')
-    parser.add_argument('--stage',
-                        help='The stage of evaluation pipeline, '
-                             'can be `all`, `infer`, `review`. Default to `all`.',
-                        type=str,
-                        default='all')
-
-    parser.add_argument('--eval-backend',
-                        help='The evaluation backend to use. Default to None.'
-                             'can be `Native`, `OpenCompass` and `ThirdParty`. '
-                             'Default to `Native`.',
-                        type=str,
-                        default=EvalBackend.NATIVE.value,
-                        required=False)
-
-    parser.add_argument('--eval-config',
-                        help='The eval task config file path for evaluation backend, should be a yaml or json file.',
-                        type=str,
-                        default=None,
-                        required=False)
-
-    args = parser.parse_args()
-
-    return args
-
-
-def parse_str_args(str_args: str) -> dict:
-    assert isinstance(str_args, str), 'args should be a string.'
-    arg_list: list = str_args.strip().split(',')
-    arg_list = [arg.strip() for arg in arg_list]
-    arg_dict: dict = dict([arg.split('=') for arg in arg_list])
-
-    final_args = dict()
-    for k, v in arg_dict.items():
-        try:
-            final_args[k] = eval(v)
-        except:
-            if v.lower() == 'true':
-                v = True
-            if v.lower() == 'false':
-                v = False
-            final_args[k] = v
-
-    return final_args
-
-
-def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig]]) -> Union[dict, List[dict]]:
+def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]) -> Union[dict, List[dict]]:
+    """Run evaluation task(s) based on the provided configuration."""
+    run_time = datetime.now().strftime('%Y%m%d_%H%M%S')
 
+    # If task_cfg is a list, run each task individually
     if isinstance(task_cfg, list):
-        eval_results = []
-        for one_task_cfg in task_cfg:
-            eval_results.append(run_task(one_task_cfg))
-        return eval_results
-
-    if isinstance(task_cfg, TaskConfig):
-        task_cfg = task_cfg.to_dict()
-    elif isinstance(task_cfg, str):
-        if task_cfg.endswith('.yaml'):
-            task_cfg = yaml_to_dict(task_cfg)
-        elif task_cfg.endswith('.json'):
-            task_cfg = json_to_dict(task_cfg)
-        else:
-            raise ValueError(f'Unsupported file format: {task_cfg}, should be a yaml or json file.')
-    elif isinstance(task_cfg, dict):
-        logger.info('** Args: Task config is provided with dictionary type. **')
-    else:
-        raise ValueError('** Args: Please provide a valid task config. **')
-
-    # Check and run evaluation backend
-    if task_cfg.get('eval_backend') is None:
-        task_cfg['eval_backend'] = EvalBackend.NATIVE.value
-
-    eval_backend = task_cfg.get('eval_backend')
-    eval_config: Union[str, dict] = task_cfg.get('eval_config')
-
-    if eval_backend != EvalBackend.NATIVE.value:
-
-        if eval_config is None:
-            logger.warning(f'Got eval_backend {eval_backend}, but eval_config is not provided.')
-
-        if eval_backend == EvalBackend.OPEN_COMPASS.value:
-            from evalscope.backend.opencompass import OpenCompassBackendManager
-            oc_backend_manager = OpenCompassBackendManager(config=eval_config)
-            oc_backend_manager.run()
-        elif eval_backend == EvalBackend.VLM_EVAL_KIT.value:
-            from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
-            vlm_eval_kit_backend_manager = VLMEvalKitBackendManager(config=eval_config)
-            vlm_eval_kit_backend_manager.run()
-        elif eval_backend == EvalBackend.RAG_EVAL.value:
-            from evalscope.backend.rag_eval import RAGEvalBackendManager
-            rag_eval_backend_manager = RAGEvalBackendManager(config=eval_config)
-            rag_eval_backend_manager.run()
-        # TODO: Add other evaluation backends
-        elif eval_backend == EvalBackend.THIRD_PARTY.value:
-            raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')
-
-        return dict()
-
-    # Get the output task config
-    output_task_cfg = copy.copy(task_cfg)
-    logger.info(output_task_cfg)
-
-    model_args: dict = task_cfg.get('model_args',
-                                    {'revision': 'default', 'precision': torch.float16, 'device_map': 'auto'})
-    # Get the GLOBAL default config (infer_cfg) for prediction
-    generation_config: dict = task_cfg.get('generation_config',
-                                           {'do_sample': False,
-                                            'repetition_penalty': 1.0,
-                                            'max_length': 2048,
-                                            'max_new_tokens': 512,
-                                            'temperature': 0.3,
-                                            'top_k': 50,
-                                            'top_p': 0.8, }
-                                           )
-    dataset_args: dict = task_cfg.get('dataset_args', {})
-    dry_run: bool = task_cfg.get('dry_run', False)
-    model: Union[str, CustomModel] = task_cfg.get('model', None)
-    model_type: str = task_cfg.get('model_type', None)
-    template_type: str = task_cfg.get('template_type', None)
-    eval_type: str = task_cfg.get('eval_type', 'checkpoint')
-    datasets: list = task_cfg.get('datasets', None)
-    work_dir: str = task_cfg.get('work_dir', DEFAULT_ROOT_CACHE_DIR)
-    outputs: str = task_cfg.get('outputs', 'outputs')
-    mem_cache: bool = task_cfg.get('mem_cache', False)
-    use_cache: bool = task_cfg.get('use_cache', True)
-    dataset_hub: str = task_cfg.get('dataset_hub', 'ModelScope')
-    dataset_dir: str = task_cfg.get('dataset_dir', DEFAULT_ROOT_CACHE_DIR)
-    stage: str = task_cfg.get('stage', 'all')
-    limit: int = task_cfg.get('limit', None)
-    debug: str = task_cfg.get('debug', False)
-
-    if model is None or datasets is None:
-        if not task_cfg.get('eval_backend'):
-            raise ValueError('** Args: Please provide model and datasets. **')
-
-    if model_type:
-        logger.warning('** DeprecatedWarning: `--model-type` is deprecated, please use `--template-type` instead.')
-
-    model_precision = model_args.get('precision', torch.float16)
-    if isinstance(model_precision, str):
-        model_precision = eval(model_precision)
-
-    if mem_cache:
-        logger.warning('** DeprecatedWarning: `--mem-cache` is deprecated, please use `--use-cache` instead.')
-
-    logger.info(f'** Set use_cache to {use_cache}.')
-
-    # Get model args
-    if dry_run:
-        from evalscope.models.dummy_chat_model import DummyChatModel
-        model_id: str = 'dummy'
-        model_revision: str = 'v1.0.0'
-    elif eval_type == 'custom':
-        model_id: str = None
-        model_revision: str = None
-    else:
-        model_id: str = model
-        model_revision: str = model_args.get('revision', 'default')
+        return [run_single_task(cfg, run_time) for cfg in task_cfg]
+
+    task_cfg = parse_task_config(task_cfg)
+    return run_single_task(task_cfg, run_time)
+
 
-    # Get outputs directory
-    if isinstance(model_id, str) and os.path.isdir(os.path.expanduser(model_id)):
-        # get the output_model_id when model_id is a local model dir
-        output_model_id: str = gen_hash(model_id)
+def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
+    """Run a single evaluation task."""
+    seed_everything(task_cfg.seed)
+    outputs = setup_work_directory(task_cfg, run_time)
+    configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
+
+    task_cfg.dump_yaml(outputs.configs_dir)
+    logger.info(task_cfg)
+
+    if task_cfg.eval_backend != EvalBackend.NATIVE:
+        return run_non_native_backend(task_cfg)
     else:
-        output_model_id: str = model_id
-    if outputs == 'outputs':
-        outputs = make_outputs_dir(root_dir=os.path.join(work_dir, 'outputs'),
-                                   datasets=datasets,
-                                   model_id=output_model_id,
-                                   model_revision=model_revision,)
-
-    eval_results = dict()
-    for dataset_name in datasets:
-        # Get imported_modules
-        imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
-
-        if dataset_name == 'humaneval' and dataset_args.get('humaneval', {}).get('local_path') is None:
-            raise ValueError('Please specify the local problem path of humaneval dataset in --dataset-args,'
-                             'e.g. {"humaneval": {"local_path": "/to/your/path"}}, '
-                             'And refer to https://github.com/openai/human-eval/tree/master#installation to install it,'
-                             'Note that you need to enable the execution code in the human_eval/execution.py first.')
-
-        if dry_run:
-            from evalscope.models.dummy_chat_model import DummyChatModel
-            model_adapter = DummyChatModel(model_cfg=dict())
-        elif eval_type == 'custom':
-            if not isinstance(model, CustomModel):
-                raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(model)}.')
-            from evalscope.models.model_adapter import CustomModelAdapter
-            model_adapter = CustomModelAdapter(custom_model=model)
-        else:
-            # Init model adapter
-            device_map = model_args.get('device_map', 'auto') if torch.cuda.is_available() else None
-            model_adapter = imported_modules['ModelAdapterClass'](model_id=model_id,
-                                                                  model_revision=model_revision,
-                                                                  device_map=device_map,
-                                                                  torch_dtype=model_precision,
-                                                                  cache_dir=work_dir,
-                                                                  template_type=template_type)
-
-        if dataset_name == 'humaneval':
-            problem_file: str = dataset_args.get('humaneval', {}).get('local_path')
-
-            evaluator = HumanevalEvaluator(problem_file=problem_file,
-                                           model_id=model_id,
-                                           model_revision=model_revision,
-                                           model_adapter=model_adapter,
-                                           outputs_dir=outputs,
-                                           is_custom_outputs_dir=False, )
-        else:
-            # TODO: CHECK dataset_args
-            dataset_name_or_path: str = dataset_args.get(dataset_name, {}).get('local_path') or imported_modules[
-                'DATASET_ID']
-
-            in_prompt_template: str = dataset_args.get(dataset_name, {}).get('prompt_template', '')
-
-            # Init data adapter
-            few_shot_num: int = dataset_args.get(dataset_name, {}).get('few_shot_num', None)
-            few_shot_random: bool = dataset_args.get(dataset_name, {}).get('few_shot_random', True)
-            data_adapter = imported_modules['DataAdapterClass'](few_shot_num=few_shot_num,
-                                                                few_shot_random=few_shot_random,
-                                                                prompt_template=in_prompt_template,)
-
-            in_subset_list: list = dataset_args.get(dataset_name, {})\
-                .get('subset_list', imported_modules['SUBSET_LIST'])
-            logger.info(f'\n** Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
-
-            evaluator = Evaluator(
-                dataset_name_or_path=dataset_name_or_path,
-                subset_list=in_subset_list,
-                data_adapter=data_adapter,
-                model_adapter=model_adapter,
-                use_cache=use_cache,
-                root_cache_dir=work_dir,
-                outputs_dir=outputs,
-                is_custom_outputs_dir=outputs != 'outputs',
-                datasets_dir=dataset_dir,
-                datasets_hub=dataset_hub,
-                stage=stage,
-                eval_type=eval_type,
-                overall_task_cfg=output_task_cfg,
-            )
-
-        infer_cfg = generation_config or {}
-        infer_cfg.update(dict(limit=limit))
-        res_dict: dict = evaluator.eval(infer_cfg=infer_cfg, debug=debug)
+        return evaluate_model(task_cfg, outputs)
+
+
+def setup_work_directory(task_cfg: TaskConfig, run_time: str):
+    """Set the working directory for the task."""
+    if task_cfg.use_cache:
+        task_cfg.work_dir = task_cfg.use_cache
+        logger.info(f'Set resume from {task_cfg.work_dir}')
+    elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
+        task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
+    outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
+
+    if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
+        task_cfg.eval_config['time_str'] = run_time
+    elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
+        task_cfg.eval_config['work_dir'] = task_cfg.work_dir
+    return outputs
+
+
+def run_non_native_backend(task_cfg: TaskConfig) -> dict:
+    """Run evaluation using a non-native backend."""
+    eval_backend = task_cfg.eval_backend
+    eval_config = task_cfg.eval_config
+
+    if eval_config is None:
+        logger.warning(f'Got eval_backend {eval_backend}, but eval_config is not provided.')
+
+    backend_manager_class = get_backend_manager_class(eval_backend)
+    backend_manager = backend_manager_class(config=eval_config)
+    backend_manager.run()
+
+    return dict()
+
+
+def get_backend_manager_class(eval_backend: EvalBackend):
+    """Get the backend manager class based on the evaluation backend."""
+    if eval_backend == EvalBackend.OPEN_COMPASS:
+        from evalscope.backend.opencompass import OpenCompassBackendManager
+        return OpenCompassBackendManager
+    elif eval_backend == EvalBackend.VLM_EVAL_KIT:
+        from evalscope.backend.vlm_eval_kit import VLMEvalKitBackendManager
+        return VLMEvalKitBackendManager
+    elif eval_backend == EvalBackend.RAG_EVAL:
+        from evalscope.backend.rag_eval import RAGEvalBackendManager
+        return RAGEvalBackendManager
+    elif eval_backend == EvalBackend.THIRD_PARTY:
+        raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')
+
+
+def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
+    """Evaluate the model based on the provided task configuration."""
+    # Initialize evaluator
+    eval_results = {}
+
+    for dataset_name in task_cfg.datasets:
+        evaluator = create_evaluator(task_cfg, dataset_name, outputs)
+        res_dict = evaluator.eval(infer_cfg=task_cfg.generation_config, debug=task_cfg.debug, limit=task_cfg.limit)
         eval_results[dataset_name] = res_dict
 
     return eval_results
 
 
+def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure):
+    """Create an evaluator object for the specified dataset."""
+    imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
+    model_adapter = initialize_model_adapter(task_cfg, dataset_name, imported_modules)
+
+    dataset_config = task_cfg.dataset_args.get(dataset_name, {})
+    dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
+    in_prompt_template = dataset_config.get('prompt_template', '')
+    few_shot_num = dataset_config.get('few_shot_num', None)
+    few_shot_random = dataset_config.get('few_shot_random', True)
+
+    data_adapter = imported_modules['DataAdapterClass'](
+        few_shot_num=few_shot_num,
+        few_shot_random=few_shot_random,
+        prompt_template=in_prompt_template,
+        outputs=outputs,
+    )
+    in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
+
+    logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
+
+    return Evaluator(
+        dataset_name_or_path=dataset_name_or_path,
+        subset_list=in_subset_list,
+        data_adapter=data_adapter,
+        model_adapter=model_adapter,
+        use_cache=task_cfg.use_cache,
+        outputs=outputs,
+        datasets_dir=task_cfg.dataset_dir,
+        datasets_hub=task_cfg.dataset_hub,
+        stage=task_cfg.stage,
+        eval_type=task_cfg.eval_type,
+        overall_task_cfg=task_cfg,
+    )
+
+
+def initialize_model_adapter(task_cfg: TaskConfig, dataset_name: str, imported_modules):
+    """Initialize the model adapter based on the task configuration."""
+    if task_cfg.dry_run:
+        from evalscope.models.dummy_chat_model import DummyChatModel
+        return DummyChatModel(model_cfg=dict())
+    elif task_cfg.eval_type == EvalType.CUSTOM:
+        if not isinstance(task_cfg.model, CustomModel):
+            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
+        from evalscope.models.model_adapter import CustomModelAdapter
+        return CustomModelAdapter(custom_model=task_cfg.model)
+    else:
+        device_map = task_cfg.model_args.get('device_map', 'auto') if torch.cuda.is_available() else None
+        model_precision = task_cfg.model_args.get('precision', torch.float16)
+        if isinstance(model_precision, str) and model_precision != 'auto':
+            model_precision = eval(model_precision)
+        return imported_modules['ModelAdapterClass'](
+            model_id=task_cfg.model,
+            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
+            device_map=device_map,
+            torch_dtype=model_precision,
+            generation_config=task_cfg.generation_config,
+            chat_template=task_cfg.chat_template)
+
+
 def main():
     args = parse_args()
-
-    # Get task_cfg
-    use_cache: bool = False if args.use_cache.lower() == 'false' else True
-    task_cfg = {
-        'model_args': parse_str_args(args.model_args),
-        'generation_config': parse_str_args(args.generation_config),
-        'dataset_args': args.dataset_args,
-        'dry_run': args.dry_run,
-        'model': args.model,
-        'template_type': args.template_type,
-        'eval_type': args.eval_type,
-        'datasets': args.datasets,
-        'work_dir': args.work_dir,
-        'outputs': args.outputs,
-        'mem_cache': args.mem_cache,
-        'use_cache': use_cache,
-        'dataset_hub': args.dataset_hub,
-        'dataset_dir': args.dataset_dir,
-        'stage': args.stage,
-        'limit': args.limit,
-        'debug': args.debug,
-
-        'eval_backend': args.eval_backend,
-        'eval_config': args.eval_config,
-    }
-
-    run_task(task_cfg)
+    run_task(args)
 
 
 if __name__ == '__main__':
-    # Usage: python3 evalscope/run.py --model ZhipuAI/chatglm2-6b --datasets mmlu hellaswag --limit 10
-    # Usage: python3 evalscope/run.py --model qwen/Qwen-1_8B --generation-config do_sample=false,temperature=0.0 --datasets ceval --dataset-args '{"ceval": {"few_shot_num": 0}}' --limit 10
     main()