evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/config.py

@@ -1,85 +1,253 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
+# flake8: noqa: E501
 import copy
-import json
 import os
 from argparse import Namespace
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType
-from evalscope.models.custom import CustomModel
-from evalscope.utils import gen_hash
-from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
+from evalscope.api.model import GenerateConfig, Model, ModelAPI
+from evalscope.constants import (
+    DEFAULT_DATASET_CACHE_DIR,
+    DEFAULT_WORK_DIR,
+    EvalBackend,
+    EvalType,
+    HubType,
+    JudgeStrategy,
+    ModelTask,
+)
+from evalscope.utils.argument_utils import BaseArgument, parse_int_or_float
+from evalscope.utils.deprecation_utils import deprecated_warning
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.io_utils import dict_to_yaml, gen_hash, safe_filename
 from evalscope.utils.logger import get_logger
+from evalscope.version import __version__ as evalscope_version
 
 logger = get_logger()
 
-cur_path = os.path.dirname(os.path.abspath(__file__))
-
-DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16', 'device': 'auto'}
-DEFAULT_GENERATION_CONFIG = {
-    'max_length': 2048,
-    'max_new_tokens': 512,
-    'do_sample': False,
-    'top_k': 50,
-    'top_p': 1.0,
-    'temperature': 1.0,
-}
-
 
 @dataclass
-class TaskConfig:
+class TaskConfig(BaseArgument):
     # Model-related arguments
-    model: Union[str, 'CustomModel', None] = None
+    model: Optional[Union[str, Model, ModelAPI]] = None
+    """The model to be evaluated. Can be a string path, Model object, or ModelAPI object."""
+
     model_id: Optional[str] = None
-    model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
+    """Unique identifier for the model. Auto-generated from model name if not provided."""
+
+    model_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to the model during initialization."""
+
+    model_task: str = ModelTask.TEXT_GENERATION
+    """The type of task the model performs (e.g., text generation, image generation)."""
 
     # Template-related arguments
-    template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
     chat_template: Optional[str] = None
+    """Chat template to use for formatting conversations with the model."""
 
     # Dataset-related arguments
     datasets: List[str] = field(default_factory=list)
+    """List of dataset names to evaluate the model on."""
+
     dataset_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to datasets during loading."""
+
     dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
+    """Directory where datasets are cached locally."""
+
     dataset_hub: str = HubType.MODELSCOPE
+    """Hub platform to download datasets from (e.g., ModelScope, HuggingFace)."""
+
+    repeats: int = 1
+    """Number of times to repeat the dataset items for k-metrics evaluation."""
 
     # Generation configuration arguments
-    generation_config: Optional[Dict] = field(default_factory=lambda: DEFAULT_GENERATION_CONFIG | {})
+    generation_config: Union[Dict, GenerateConfig] = field(default_factory=dict)
+    """Configuration parameters for text/image generation."""
 
     # Evaluation-related arguments
     eval_type: str = EvalType.CHECKPOINT
+    """Type of evaluation: checkpoint, service, or mock."""
+
     eval_backend: str = EvalBackend.NATIVE
+    """Backend framework to use for evaluation."""
+
     eval_config: Union[str, Dict, None] = None
-    stage: str = EvalStage.ALL
-    limit: Optional[int] = None
+    """Additional evaluation configuration parameters."""
+
+    limit: Optional[Union[int, float]] = None
+    """Maximum number of samples to evaluate. Can be int (count) or float (fraction)."""
+
+    eval_batch_size: int = 1
+    """Batch size for evaluation processing."""
 
     # Cache and working directory arguments
-    mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
     use_cache: Optional[str] = None
+    """Whether to use cached results and which cache strategy to apply."""
+
+    rerun_review: bool = False
+    """Whether to rerun the review process even if results exist."""
+
     work_dir: str = DEFAULT_WORK_DIR
-    outputs: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+    """Working directory for storing evaluation results and temporary files."""
 
     # Debug and runtime mode arguments
+    ignore_errors: bool = False
+    """Whether to continue evaluation when encountering errors."""
+
     debug: bool = False
-    dry_run: bool = False
+    """Enable debug mode for detailed logging and error reporting."""
+
     seed: Optional[int] = 42
-    api_url: Optional[str] = None  # Only used for server model
-    api_key: Optional[str] = 'EMPTY'  # Only used for server model
+    """Random seed for reproducible results."""
+
+    api_url: Optional[str] = None
+    """API endpoint URL for server-based model evaluation."""
+
+    api_key: Optional[str] = 'EMPTY'
+    """API key for authenticating with server-based models."""
+
+    timeout: Optional[float] = None
+    """Request timeout in seconds for server-based models."""
+
+    stream: Optional[bool] = None
+    """Whether to use streaming responses for server-based models."""
+
+    # LLMJudge arguments
+    judge_strategy: str = JudgeStrategy.AUTO
+    """Strategy for LLM-based judgment (auto, single, pairwise)."""
+
+    judge_worker_num: int = 1
+    """Number of worker processes for parallel LLM judging."""
+
+    judge_model_args: Optional[Dict] = field(default_factory=dict)
+    """Additional arguments for the judge model configuration."""
+
+    analysis_report: bool = False
+    """Whether to generate detailed analysis reports after evaluation."""
+
+    # Sandbox configuration arguments
+    use_sandbox: bool = False
+    """Whether to execute code in a sandboxed environment."""
+
+    sandbox_type: Optional[str] = 'docker'
+    """Type of sandbox environment for code execution (e.g., docker). Default is 'docker'."""
+
+    sandbox_manager_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for the sandbox manager. Default is local manager. If url is provided, it will use remote manager."""
+
+    sandbox_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for sandboxed code execution environments."""
+
+    evalscope_version: Optional[str] = evalscope_version
+    """EvalScope version used for the evaluation."""
 
     def __post_init__(self):
-        if (not self.model_id) and self.model:
-            if isinstance(self.model, CustomModel):
-                self.model_id = type(self.model).__name__
+        self.__init_model_and_id()
+
+        self.__init_eval_data_config()
+
+        # Set default generation_config and model_args
+        self.__init_default_generation_config()
+        self.__init_default_model_args()
+        self.__init_default_sandbox_config()
+
+    def __init_model_and_id(self):
+        # Set model to DummyCustomModel if not provided
+        if self.model is None:
+            self.model = self.model_task
+            self.eval_type = EvalType.MOCK_LLM
+
+        # Set model_id if not provided
+        if not self.model_id:
+            if isinstance(self.model, str):
+                self.model_id = safe_filename(os.path.basename(self.model))
+            elif isinstance(self.model, Model):
+                self.model_id = safe_filename(self.model.name)
+            elif isinstance(self.model, ModelAPI):
+                self.model_id = safe_filename(self.model.model_name)
             else:
-                self.model_id = os.path.basename(self.model).rstrip(os.sep)
-
-    def to_dict(self):
-        return self.__dict__
+                self.model_id = 'dummy_model'
+
+    def __init_eval_data_config(self):
+        # Post process limit
+        if self.limit is not None:
+            self.limit = parse_int_or_float(self.limit)
+
+    def __init_default_generation_config(self):
+        if not self.generation_config:
+            if self.model_task == ModelTask.IMAGE_GENERATION:
+                self.generation_config = {
+                    'height': 1024,
+                    'width': 1024,
+                    'num_inference_steps': 50,
+                    'guidance_scale': 9.0,
+                }
+                if self.eval_batch_size != 1:
+                    logger.warning(
+                        'For image generation task, we only support eval_batch_size=1 for now, changed to 1.'
+                    )
+                    self.eval_batch_size = 1
+            elif self.model_task == ModelTask.TEXT_GENERATION:
+                if self.eval_type == EvalType.CHECKPOINT:
+                    self.generation_config = {
+                        'max_tokens': 2048,
+                        'do_sample': False,
+                        'top_k': 50,
+                        'top_p': 1.0,
+                        'temperature': 1.0,
+                        'n': 1,
+                    }
+                elif self.eval_type == EvalType.SERVICE:
+                    self.generation_config = {
+                        'temperature': 0.0,
+                    }
+        if isinstance(self.generation_config, dict):
+            self.generation_config = GenerateConfig.model_validate(self.generation_config)
+
+        # Set eval_batch_size to generation_config.batch_size
+        self.generation_config.batch_size = self.eval_batch_size
+
+        # Set default values for generation_config
+        if self.timeout is not None:
+            deprecated_warning(
+                logger,
+                'The `timeout` parameter is deprecated and will be removed in v2.0.0. Use `generation_config.timeout` instead.'
+            )
+            self.generation_config.timeout = self.timeout
+
+        if self.stream is not None:
+            deprecated_warning(
+                logger,
+                'The `stream` parameter is deprecated and will be removed in v2.0.0. Use `generation_config.stream` instead.'
+            )
+            self.generation_config.stream = self.stream
+
+        if self.generation_config.n is not None and self.generation_config.n > 1:
+            self.repeats = self.generation_config.n
+            self.generation_config.n = 1
+            deprecated_warning(
+                logger,
+                'The `n` parameter in generation_config is deprecated and will be removed in v2.0.0. Use `TaskConfig.repeats` instead.'
+            )
+
+    def __init_default_model_args(self):
+        if self.model_args:
+            return
+        if self.model_task == ModelTask.TEXT_GENERATION:
+            if self.eval_type == EvalType.CHECKPOINT:
+                self.model_args = {
+                    'revision': 'master',
+                    'precision': 'torch.float16',
+                }
+
+    def __init_default_sandbox_config(self):
+        if not self.use_sandbox:
+            return
+        check_import('ms_enclave', 'ms_enclave[docker]', raise_error=True)
 
-    def __str__(self):
-        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+        if not self.sandbox_type:
+            self.sandbox_type = 'docker'
 
     def update(self, other: Union['TaskConfig', dict]):
         if isinstance(other, TaskConfig):
@@ -95,91 +263,16 @@ class TaskConfig:
         except Exception as e:
             logger.warning(f'Failed to dump overall task config: {e}')
 
-    @staticmethod
-    def list():
-        return list(registry_tasks.keys())
-
-    @staticmethod
-    def from_yaml(yaml_file: str):
-        return TaskConfig.from_dict(yaml_to_dict(yaml_file))
-
-    @staticmethod
-    def from_dict(d: dict):
-        return TaskConfig(**d)
-
-    @staticmethod
-    def from_json(json_file: str):
-        return TaskConfig.from_dict(json_to_dict(json_file))
-
-    @staticmethod
-    def from_args(args: Namespace):
-        # Convert Namespace to a dictionary and filter out None values
-        args_dict = {k: v for k, v in vars(args).items() if v is not None}
-
-        if 'func' in args_dict:
-            del args_dict['func']  # Note: compat CLI arguments
-
-        return TaskConfig.from_dict(args_dict)
-
-    @staticmethod
-    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
-        res_list = []
-        for task_name in tasks:
-            task = registry_tasks.get(task_name, None)
-            if task is None:
-                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
-                continue
-
-            task.model = custom_model
-            task.model_args = custom_model.config
-            task.model_id = type(custom_model).__name__
-            res_list.append(task)
-
-        return res_list
-
-    @staticmethod
-    def registry(name: str, data_pattern: str, dataset_dir: str = None, subset_list: list = None) -> None:
-        """
-        Register a new task (dataset) for evaluation.
-
-        Args:
-            name: str, the dataset name.
-            data_pattern: str, the data pattern for the task.
-                    e.g. `mmlu`, `ceval`, `gsm8k`, ...
-                    refer to task_config.list() for all available datasets.
-            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data,
-                then your specific custom dataset directory will be /path/to/data/{name}
-            subset_list: list, the subset list for the dataset.
-                e.g. ['middle_school_politics', 'operating_system']
-                refer to the mmlu for example.  https://github.com/hendrycks/test/blob/master/categories.py
-        """
-        available_datasets = list(registry_tasks.keys())
-        if data_pattern not in available_datasets:
-            logger.error(
-                f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
-            return
-
-        # Reuse the existing task config and update the datasets
-        pattern_config = registry_tasks[data_pattern]
-
-        custom_config = copy.deepcopy(pattern_config)
-        custom_config.datasets = [data_pattern]
-        custom_config.dataset_args = {data_pattern: {}}
-        custom_config.eval_type = EvalType.CHECKPOINT
-
-        if dataset_dir is not None:
-            custom_config.dataset_args[data_pattern].update({'local_path': dataset_dir})
-
-        if subset_list is not None:
-            custom_config.dataset_args[data_pattern].update({'subset_list': subset_list})
-
-        registry_tasks.update({name: custom_config})
-        logger.info(f'** Registered task: {name} with data pattern: {data_pattern}')
-
+    def to_dict(self):
+        result = copy.copy(self.__dict__)
+        del result['api_key']  # Do not expose api_key in the config
 
-tasks = ['arc', 'gsm8k', 'mmlu', 'cmmlu', 'ceval', 'bbh', 'general_qa']
+        if isinstance(self.model, (Model, ModelAPI)):
+            result['model'] = self.model.__class__.__name__
 
-registry_tasks = {task: TaskConfig.from_yaml(os.path.join(cur_path, f'registry/tasks/{task}.yaml')) for task in tasks}
+        if isinstance(self.generation_config, GenerateConfig):
+            result['generation_config'] = self.generation_config.model_dump(exclude_unset=True)
+        return result
 
 
 def parse_task_config(task_cfg) -> TaskConfig:
@@ -193,36 +286,14 @@ def parse_task_config(task_cfg) -> TaskConfig:
         logger.info('Args: Task config is provided with CommandLine type.')
         task_cfg = TaskConfig.from_args(task_cfg)
     elif isinstance(task_cfg, str):
-        extension = task_cfg.split('.')[-1]
+        extension = os.path.splitext(task_cfg)[-1]
         logger.info(f'Args: Task config is provided with {extension} file type.')
-        if extension in ['yaml', 'yml']:
+        if extension in ['.yaml', '.yml']:
             task_cfg = TaskConfig.from_yaml(task_cfg)
-        elif extension == 'json':
+        elif extension == '.json':
             task_cfg = TaskConfig.from_json(task_cfg)
         else:
             raise ValueError('Args: Unsupported file extension.')
     else:
         raise ValueError('Args: Please provide a valid task config.')
     return task_cfg
-
-
-class TempModel(CustomModel):
-
-    def __init__(self, config: dict):
-        super().__init__(config=config)
-
-    def predict(self, prompts: str, **kwargs):
-        return [item + ': response' for item in prompts]
-
-
-if __name__ == '__main__':
-    model = TempModel(config={'model_id': 'test-swift-dummy-model'})
-    task_config = TaskConfig()
-
-    # Register a new task
-    TaskConfig.registry(name='arc_swift', data_pattern='arc', dataset_dir='/path/to/swift_custom_work')
-
-    swift_eval_task: List[TaskConfig] = TaskConfig.load(custom_model=model, tasks=['gsm8k', 'arc', 'arc_swift'])
-    for item in swift_eval_task:
-        print(item)
-        print()

evalscope/constants.py

@@ -1,12 +1,22 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa
+import os
+
+os.environ['MODELSCOPE_LOG_LEVEL'] = '40'  # Set default log level to ERROR
+
 from modelscope.utils.constant import DEFAULT_REPOSITORY_REVISION
 from modelscope.utils.file_utils import get_dataset_cache_root, get_model_cache_root
 
 DEFAULT_WORK_DIR = './outputs'
 DEFAULT_MODEL_REVISION = DEFAULT_REPOSITORY_REVISION  # master
-DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub
-DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/datasets
+DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub/models
+DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/hub/datasets
 DEFAULT_ROOT_CACHE_DIR = DEFAULT_DATASET_CACHE_DIR  # compatible with old version
+DEFAULT_EVALSCOPE_CACHE_DIR = os.path.expanduser(
+    os.getenv('EVALSCOPE_CACHE', '~/.cache/evalscope')
+)  # ~/.cache/evalscope
+IS_BUILD_DOC = os.getenv('BUILD_DOC', '0') == '1'  # To avoid some heavy dependencies when building doc
+HEARTBEAT_INTERVAL_SEC = 60  # 60 seconds
 
 
 class HubType:
@@ -36,47 +46,17 @@ class MetricsConstant:
     ]
 
 
-class MetricMembers:
-
-    # Math accuracy metric
-    MATH_ACCURACY = 'math_accuracy'
-
-    # Code pass@k metric
-    CODE_PASS_K = 'code_pass_k'
-
-    # Code rouge metric
-    ROUGE = 'rouge'
-
-    # ELO rating system for pairwise comparison
-    ELO = 'elo'
-
-    # Pairwise comparison win/lose and tie(optional)
-    PAIRWISE = 'pairwise'
-
-    # Rating score for single model
-    SCORE = 'score'
-
-
 class ArenaWinner:
 
     MODEL_A = 'model_a'
-
     MODEL_B = 'model_b'
-
     TIE = 'tie'
-
     TIE_BOTH_BAD = 'tie_both_bad'
-
     UNKNOWN = 'unknown'
 
 
-class ArenaMode:
-    SINGLE = 'single'
-    PAIRWISE = 'pairwise'
-    PAIRWISE_BASELINE = 'pairwise_baseline'
-
-
 class AnswerKeys:
+    INDEX = 'index'
     ANSWER_ID = 'answer_id'
     RAW_INPUT = 'raw_input'
     ORIGIN_PROMPT = 'origin_prompt'
@@ -85,58 +65,22 @@ class AnswerKeys:
     CHOICES = 'choices'
 
 
-class ReviewKeys:
-    REVIEW_ID = 'review_id'
-    REVIEWED = 'reviewed'
-    REVIEWER_SPEC = 'reviewer_spec'
-    REVIEW_TIME = 'review_time'
-    MESSAGE = 'message'
-    CONTENT = 'content'
-    GOLD = 'gold'
-    PRED = 'pred'
-    RESULT = 'result'
-    REVIEW = 'review'
-
-
-class EvalConfigKeys:
-    CLASS_REF = 'ref'
-    CLASS_ARGS = 'args'
-    ENABLE = 'enable'
-    POSITION_BIAS_MITIGATION = 'position_bias_mitigation'
-    RANDOM_SEED = 'random_seed'
-    FN_COMPLETION_PARSER = 'fn_completion_parser'
-    COMPLETION_PARSER_KWARGS = 'completion_parser_kwargs'
-    OUTPUT_FILE = 'output_file'
-    MODEL_ID_OR_PATH = 'model_id_or_path'
-    MODEL_REVISION = 'revision'
-    GENERATION_CONFIG = 'generation_config'
-    PRECISION = 'precision'
-    TEMPLATE_TYPE = 'template_type'
-
-
-class FnCompletionParser:
-    LMSYS_PARSER: str = 'lmsys_parser'
-    RANKING_PARSER: str = 'ranking_parser'
-
-
-class PositionBiasMitigation:
-    NONE = 'none'
-    RANDOMIZE_ORDER = 'randomize_order'
-    SWAP_POSITION = 'swap_position'
-
-
-class EvalStage:
-    # Enums: `all`, `infer`, `review`
-    ALL = 'all'
-    INFER = 'infer'
-    REVIEW = 'review'
-
-
 class EvalType:
 
     CUSTOM = 'custom'
-    CHECKPOINT = 'checkpoint'  # native model checkpoint
-    SERVICE = 'service'  # model service
+    MOCK_LLM = 'mock_llm'
+    CHECKPOINT = 'llm_ckpt'  # native model checkpoint
+    SERVICE = 'openai_api'  # model service
+    TEXT2IMAGE = 'text2image'  # image generation service
+    IMAGE_EDITING = 'image_editing'  # image editing service
+
+
+class OutputType:
+    LOGITS = 'logits'  # for logits output tasks
+    GENERATION = 'generation'  # for text generation tasks and general tasks
+    MULTIPLE_CHOICE = 'multiple_choice_logits'  # for multiple choice tasks
+    CONTINUOUS = 'continuous_logits'  # for continuous tasks
+    IMAGE_GENERATION = 'image_generation'  # for image generation tasks
 
 
 class EvalBackend:
@@ -149,3 +93,55 @@ class EvalBackend:
 
 class DataCollection:
     NAME = 'data_collection'
+    INFO = 'collection_info'
+
+
+class JudgeStrategy:
+    AUTO = 'auto'
+    RULE = 'rule'
+    LLM = 'llm'
+    LLM_RECALL = 'llm_recall'
+
+
+class JudgeScoreType:
+    NUMERIC = 'numeric'  # numeric score
+    PATTERN = 'pattern'  # pattern matching score
+
+
+class ModelTask:
+    TEXT_GENERATION = 'text_generation'
+    IMAGE_GENERATION = 'image_generation'
+
+
+class Tags:
+    KNOWLEDGE = 'Knowledge'
+    MULTIPLE_CHOICE = 'MCQ'
+    MATH = 'Math'
+    REASONING = 'Reasoning'
+    CODING = 'Coding'
+    CHINESE = 'Chinese'
+    COMMONSENSE = 'Commonsense'
+    QA = 'QA'
+    NER = 'NER'
+    READING_COMPREHENSION = 'ReadingComprehension'
+    CUSTOM = 'Custom'
+    INSTRUCTION_FOLLOWING = 'InstructionFollowing'
+    ARENA = 'Arena'
+    LONG_CONTEXT = 'LongContext'
+    RETRIEVAL = 'Retrieval'
+    FUNCTION_CALLING = 'FunctionCalling'
+    TEXT_TO_IMAGE = 'TextToImage'
+    IMAGE_EDITING = 'ImageEditing'
+    MULTI_MODAL = 'MultiModal'
+    MULTI_LINGUAL = 'MultiLingual'
+    MULTI_TURN = 'MultiTurn'
+    YES_NO = 'Yes/No'
+    HALLUCINATION = 'Hallucination'
+    MEDICAL = 'Medical'
+    AGENT = 'Agent'
+    MT = 'MachineTranslation'
+
+
+class FileConstants:
+    IMAGE_PATH = 'image_path'
+    ID = 'id'

evalscope/evaluator/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.evaluator.evaluator import Evaluator
+from .evaluator import DefaultEvaluator