evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/cli/start_app.py

@@ -28,6 +28,12 @@ class StartAppCMD(CLICommand):
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        from evalscope.app import create_app
+        try:
+            from evalscope.app import create_app
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import create_app from evalscope.app, due to {e}. '
+                "Please run `pip install 'evalscope[app]'`."
+            )
 
         create_app(self.args)

evalscope/cli/start_perf.py

@@ -28,6 +28,12 @@ class PerfBenchCMD(CLICommand):
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        from evalscope.perf.main import run_perf_benchmark
+        try:
+            from evalscope.perf.main import run_perf_benchmark
+        except ImportError as e:
+            raise ImportError(
+                f'Failed to import run_perf_benchmark from evalscope.perf.main, due to {e}. '
+                "Please run `pip install 'evalscope[perf]'`."
+            )
 
         run_perf_benchmark(self.args)

evalscope/cli/start_server.py

@@ -25,14 +25,16 @@ def add_perf_args(parser):
         '--logdir',
         required=True,
         type=str,
-        help='The monitor log save dir, tensorboard start at this path for display!')
+        help='The monitor log save dir, tensorboard start at this path for display!'
+    )
     parser.add_argument('--host', type=str, default='0.0.0.0', help='The tensorboard host')
     parser.add_argument('--tensorboard-port', type=str, default='6006', help='The tensorboard port')
 
 
 def async_run_command_with_popen(cmd):
     sub_process = subprocess.Popen(
-        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8')
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8'
+    )
     return sub_process
 
 
@@ -61,7 +63,8 @@ def start_server(args):
         bufsize=1,
         shell=True,
         universal_newlines=True,
-        encoding='utf8')
+        encoding='utf8'
+    )
 
     os.set_blocking(sub_process.stdout.fileno(), False)
     return sub_process

evalscope/collections/__init__.py

@@ -4,20 +4,12 @@ from typing import TYPE_CHECKING
 from evalscope.utils.import_utils import _LazyModule
 
 if TYPE_CHECKING:
-    from .evaluator import EvaluatorCollection
-    from .sampler import StratifiedSampler, UniformSampler, WeightedSampler
+    from .sampler import DatasetEntry, StratifiedSampler, UniformSampler, WeightedSampler
     from .schema import CollectionSchema, DatasetInfo
 
 else:
     _import_structure = {
-        'evaluator': [
-            'EvaluatorCollection',
-        ],
-        'sampler': [
-            'StratifiedSampler',
-            'UniformSampler',
-            'WeightedSampler',
-        ],
+        'sampler': ['StratifiedSampler', 'UniformSampler', 'WeightedSampler', 'DatasetEntry'],
         'schema': [
             'CollectionSchema',
             'DatasetInfo',

evalscope/collections/sampler.py

@@ -1,18 +1,17 @@
 import random
 from abc import ABC, abstractmethod
-from dataclasses import asdict, dataclass, field
+from pydantic import BaseModel, Field
 from tqdm import tqdm
 from typing import List, Optional
 
 from evalscope.collections.schema import CollectionSchema, DatasetInfo
 
 
-@dataclass
-class DatasetEntry:
+class DatasetEntry(BaseModel):
     index: int = 0
-    prompt: dict = field(default_factory=dict)
-    tags: List[str] = field(default_factory=list)
-    categories: List[str] = field(default_factory=list)
+    prompt: dict = Field(default_factory=dict)
+    tags: List[str] = Field(default_factory=list)
+    categories: List[str] = Field(default_factory=list)
     task_type: str = ''
     weight: float = 0.0
     dataset_name: str = ''
@@ -33,17 +32,18 @@ class Sampler(ABC):
         all_data = []
         data_dict = dataset.get_data()
         for subset_name, subset_data in data_dict.items():
-            for prompt in subset_data:
+            for sample in subset_data:
                 all_data.append(
                     DatasetEntry(
-                        prompt=prompt,
+                        prompt=sample.model_dump(exclude_none=True),
                         tags=dataset.tags,
                         categories=dataset.hierarchy,
                         task_type=dataset.task_type,
                         weight=dataset.weight,
                         dataset_name=dataset.name,
                         subset_name=subset_name,
-                    ))
+                    )
+                )
         count = min(count, len(all_data))  # avoid sampling more than the dataset size
         sampled_data = random.sample(all_data, k=count)
         return sampled_data
@@ -52,7 +52,7 @@ class Sampler(ABC):
         result = []
         for i, entry in enumerate(all_data):
             entry.index = i
-            result.append(asdict(entry))
+            result.append(entry.model_dump())
         return result
 
 

evalscope/collections/schema.py

@@ -3,6 +3,10 @@ import json
 from dataclasses import asdict, dataclass, field
 from typing import List, Union
 
+from evalscope.api.dataset import DatasetDict
+from evalscope.api.registry import get_benchmark
+from evalscope.config import TaskConfig
+
 
 @dataclass
 class DatasetInfo:
@@ -13,15 +17,11 @@ class DatasetInfo:
     args: dict = field(default_factory=dict)
     hierarchy: List[str] = field(default_factory=list)
 
-    def get_data(self) -> dict:
-        from evalscope.benchmarks import Benchmark
-
-        benchmark_meta = Benchmark.get(self.name)
-
-        data_adapter = benchmark_meta.get_data_adapter(config=self.args)
-        data_dict = data_adapter.load()
-        prompts = data_adapter.gen_prompts(data_dict)
-        return prompts
+    def get_data(self) -> DatasetDict:
+        dataset_args = {self.name: self.args}
+        benchmark_meta = get_benchmark(self.name, config=TaskConfig(dataset_args=dataset_args))
+        data_dict = benchmark_meta.load_dataset()
+        return data_dict
 
 
 def flatten_weight(collection: 'CollectionSchema', base_weight=1):
@@ -111,8 +111,10 @@ if __name__ == '__main__':
             ]),
             CollectionSchema(
                 name='chinese',
-                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})])
-        ])
+                datasets=[DatasetInfo(name='ceval', weight=1, tags=['zh'], args={'subset_list': ['logic']})]
+            )
+        ]
+    )
     print(schema)
     print(schema.flatten())
     schema.dump_json('outputs/schema.json')

evalscope/config.py

@@ -1,16 +1,24 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-
+# flake8: noqa: E501
 import copy
 import os
 from argparse import Namespace
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from evalscope.constants import (DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType,
-                                 JudgeStrategy, ModelTask, OutputType)
-from evalscope.models import CustomModel, DummyCustomModel
+from evalscope.api.model import GenerateConfig, Model, ModelAPI
+from evalscope.constants import (
+    DEFAULT_DATASET_CACHE_DIR,
+    DEFAULT_WORK_DIR,
+    EvalBackend,
+    EvalType,
+    HubType,
+    JudgeStrategy,
+    ModelTask,
+)
 from evalscope.utils.argument_utils import BaseArgument, parse_int_or_float
-from evalscope.utils.io_utils import dict_to_yaml, gen_hash
+from evalscope.utils.deprecation_utils import deprecated_warning
+from evalscope.utils.io_utils import dict_to_yaml, gen_hash, safe_filename
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -19,104 +27,191 @@ logger = get_logger()
 @dataclass
 class TaskConfig(BaseArgument):
     # Model-related arguments
-    model: Union[str, 'CustomModel', None] = None
+    model: Optional[Union[str, Model, ModelAPI]] = None
+    """The model to be evaluated. Can be a string path, Model object, or ModelAPI object."""
+
     model_id: Optional[str] = None
+    """Unique identifier for the model. Auto-generated from model name if not provided."""
+
     model_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to the model during initialization."""
+
     model_task: str = ModelTask.TEXT_GENERATION
+    """The type of task the model performs (e.g., text generation, image generation)."""
 
     # Template-related arguments
-    template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
     chat_template: Optional[str] = None
+    """Chat template to use for formatting conversations with the model."""
 
     # Dataset-related arguments
     datasets: List[str] = field(default_factory=list)
+    """List of dataset names to evaluate the model on."""
+
     dataset_args: Dict = field(default_factory=dict)
+    """Additional arguments to pass to datasets during loading."""
+
     dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
+    """Directory where datasets are cached locally."""
+
     dataset_hub: str = HubType.MODELSCOPE
+    """Hub platform to download datasets from (e.g., ModelScope, HuggingFace)."""
+
+    repeats: int = 1
+    """Number of times to repeat the dataset items for k-metrics evaluation."""
 
     # Generation configuration arguments
-    generation_config: Dict = field(default_factory=dict)
+    generation_config: Union[Dict, GenerateConfig] = field(default_factory=dict)
+    """Configuration parameters for text/image generation."""
 
     # Evaluation-related arguments
     eval_type: str = EvalType.CHECKPOINT
+    """Type of evaluation: checkpoint, service, or mock."""
+
     eval_backend: str = EvalBackend.NATIVE
+    """Backend framework to use for evaluation."""
+
     eval_config: Union[str, Dict, None] = None
-    stage: str = EvalStage.ALL
+    """Additional evaluation configuration parameters."""
+
     limit: Optional[Union[int, float]] = None
-    eval_batch_size: Optional[int] = None
+    """Maximum number of samples to evaluate. Can be int (count) or float (fraction)."""
+
+    eval_batch_size: int = 1
+    """Batch size for evaluation processing."""
 
     # Cache and working directory arguments
-    mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
     use_cache: Optional[str] = None
+    """Whether to use cached results and which cache strategy to apply."""
+
+    rerun_review: bool = False
+    """Whether to rerun the review process even if results exist."""
+
     work_dir: str = DEFAULT_WORK_DIR
-    outputs: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+    """Working directory for storing evaluation results and temporary files."""
 
     # Debug and runtime mode arguments
     ignore_errors: bool = False
+    """Whether to continue evaluation when encountering errors."""
+
     debug: bool = False
-    dry_run: bool = False
+    """Enable debug mode for detailed logging and error reporting."""
+
     seed: Optional[int] = 42
-    api_url: Optional[str] = None  # Only used for server model
-    api_key: Optional[str] = 'EMPTY'  # Only used for server model
-    timeout: Optional[float] = None  # Only used for server model
-    stream: bool = False  # Only used for server model
+    """Random seed for reproducible results."""
+
+    api_url: Optional[str] = None
+    """API endpoint URL for server-based model evaluation."""
+
+    api_key: Optional[str] = 'EMPTY'
+    """API key for authenticating with server-based models."""
+
+    timeout: Optional[float] = None
+    """Request timeout in seconds for server-based models."""
+
+    stream: Optional[bool] = None
+    """Whether to use streaming responses for server-based models."""
 
     # LLMJudge arguments
     judge_strategy: str = JudgeStrategy.AUTO
+    """Strategy for LLM-based judgment (auto, single, pairwise)."""
+
     judge_worker_num: int = 1
+    """Number of worker processes for parallel LLM judging."""
+
     judge_model_args: Optional[Dict] = field(default_factory=dict)
+    """Additional arguments for the judge model configuration."""
+
     analysis_report: bool = False
+    """Whether to generate detailed analysis reports after evaluation."""
 
     def __post_init__(self):
+        self.__init_model_and_id()
+
+        self.__init_eval_data_config()
+
+        # Set default generation_config and model_args
+        self.__init_default_generation_config()
+        self.__init_default_model_args()
+
+    def __init_model_and_id(self):
+        # Set model to DummyCustomModel if not provided
         if self.model is None:
-            self.model = DummyCustomModel()
-            self.eval_type = EvalType.CUSTOM
+            self.model = self.model_task
+            self.eval_type = EvalType.MOCK_LLM
 
-        if (not self.model_id) and self.model:
-            if isinstance(self.model, CustomModel):
-                self.model_id = self.model.config.get('model_id', 'custom_model')
+        # Set model_id if not provided
+        if not self.model_id:
+            if isinstance(self.model, str):
+                self.model_id = safe_filename(os.path.basename(self.model))
+            elif isinstance(self.model, Model):
+                self.model_id = safe_filename(self.model.name)
+            elif isinstance(self.model, ModelAPI):
+                self.model_id = safe_filename(self.model.model_name)
             else:
-                self.model_id = os.path.basename(self.model).rstrip(os.sep)
-            # fix path error, see http://github.com/modelscope/evalscope/issues/377
-            self.model_id = self.model_id.replace(':', '-')
-
-        # Set default eval_batch_size based on eval_type
-        if self.eval_batch_size is None:
-            self.eval_batch_size = 8 if self.eval_type == EvalType.SERVICE else 1
+                self.model_id = 'dummy_model'
 
+    def __init_eval_data_config(self):
         # Post process limit
         if self.limit is not None:
             self.limit = parse_int_or_float(self.limit)
 
-        # Set default generation_config and model_args
-        self.__init_default_generation_config()
-        self.__init_default_model_args()
-
     def __init_default_generation_config(self):
-        if self.generation_config:
-            return
-        if self.model_task == ModelTask.IMAGE_GENERATION:
-            self.generation_config = {
-                'height': 1024,
-                'width': 1024,
-                'num_inference_steps': 50,
-                'guidance_scale': 9.0,
-            }
-        elif self.model_task == ModelTask.TEXT_GENERATION:
-            if self.eval_type == EvalType.CHECKPOINT:
+        if not self.generation_config:
+            if self.model_task == ModelTask.IMAGE_GENERATION:
                 self.generation_config = {
-                    'max_length': 2048,
-                    'max_new_tokens': 512,
-                    'do_sample': False,
-                    'top_k': 50,
-                    'top_p': 1.0,
-                    'temperature': 1.0,
-                }
-            elif self.eval_type == EvalType.SERVICE:
-                self.generation_config = {
-                    'max_tokens': 2048,
-                    'temperature': 0.0,
+                    'height': 1024,
+                    'width': 1024,
+                    'num_inference_steps': 50,
+                    'guidance_scale': 9.0,
                 }
+                if self.eval_batch_size != 1:
+                    logger.warning(
+                        'For image generation task, we only support eval_batch_size=1 for now, changed to 1.'
+                    )
+                    self.eval_batch_size = 1
+            elif self.model_task == ModelTask.TEXT_GENERATION:
+                if self.eval_type == EvalType.CHECKPOINT:
+                    self.generation_config = {
+                        'max_tokens': 2048,
+                        'do_sample': False,
+                        'top_k': 50,
+                        'top_p': 1.0,
+                        'temperature': 1.0,
+                        'n': 1,
+                    }
+                elif self.eval_type == EvalType.SERVICE:
+                    self.generation_config = {
+                        'max_tokens': 2048,
+                        'temperature': 0.0,
+                    }
+        if isinstance(self.generation_config, dict):
+            self.generation_config = GenerateConfig.model_validate(self.generation_config)
+
+        # Set eval_batch_size to generation_config.batch_size
+        self.generation_config.batch_size = self.eval_batch_size
+
+        # Set default values for generation_config
+        if self.timeout is not None:
+            deprecated_warning(
+                logger,
+                'The `timeout` parameter is deprecated and will be removed in v1.1.0. Use `generation_config.timeout` instead.'
+            )
+            self.generation_config.timeout = self.timeout
+
+        if self.stream is not None:
+            deprecated_warning(
+                logger,
+                'The `stream` parameter is deprecated and will be removed in v1.1.0. Use `generation_config.stream` instead.'
+            )
+            self.generation_config.stream = self.stream
+
+        if self.generation_config.n is not None and self.generation_config.n > 1:
+            self.repeats = self.generation_config.n
+            self.generation_config.n = 1
+            deprecated_warning(
+                logger,
+                'The `n` parameter in generation_config is deprecated and will be removed in v1.1.0. Use `TaskConfig.repeats` instead.'
+            )
 
     def __init_default_model_args(self):
         if self.model_args:
@@ -143,9 +238,14 @@ class TaskConfig(BaseArgument):
             logger.warning(f'Failed to dump overall task config: {e}')
 
     def to_dict(self):
-        result = self.__dict__.copy()
-        if isinstance(self.model, CustomModel):
+        result = copy.deepcopy(self.__dict__)
+        del result['api_key']  # Do not expose api_key in the config
+
+        if isinstance(self.model, (Model, ModelAPI)):
             result['model'] = self.model.__class__.__name__
+
+        if isinstance(self.generation_config, GenerateConfig):
+            result['generation_config'] = self.generation_config.model_dump(exclude_unset=True)
         return result
 
 

evalscope/constants.py

@@ -9,9 +9,12 @@ from modelscope.utils.file_utils import get_dataset_cache_root, get_model_cache_
 
 DEFAULT_WORK_DIR = './outputs'
 DEFAULT_MODEL_REVISION = DEFAULT_REPOSITORY_REVISION  # master
-DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub
-DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/datasets
+DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub/models
+DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/hub/datasets
 DEFAULT_ROOT_CACHE_DIR = DEFAULT_DATASET_CACHE_DIR  # compatible with old version
+DEFAULT_EVALSCOPE_CACHE_DIR = os.path.expanduser(
+    os.getenv('EVALSCOPE_CACHE', '~/.cache/evalscope')
+)  # ~/.cache/evalscope
 
 
 class HubType:
@@ -44,22 +47,12 @@ class MetricsConstant:
 class ArenaWinner:
 
     MODEL_A = 'model_a'
-
     MODEL_B = 'model_b'
-
     TIE = 'tie'
-
     TIE_BOTH_BAD = 'tie_both_bad'
-
     UNKNOWN = 'unknown'
 
 
-class ArenaMode:
-    SINGLE = 'single'
-    PAIRWISE = 'pairwise'
-    PAIRWISE_BASELINE = 'pairwise_baseline'
-
-
 class AnswerKeys:
     INDEX = 'index'
     ANSWER_ID = 'answer_id'
@@ -70,58 +63,14 @@ class AnswerKeys:
     CHOICES = 'choices'
 
 
-class ReviewKeys:
-    REVIEW_ID = 'review_id'
-    REVIEWED = 'reviewed'
-    REVIEWER_SPEC = 'reviewer_spec'
-    REVIEW_TIME = 'review_time'
-    MESSAGE = 'message'
-    CONTENT = 'content'
-    GOLD = 'gold'
-    PRED = 'pred'
-    RESULT = 'result'
-    REVIEW = 'review'
-
-
-class EvalConfigKeys:
-    CLASS_REF = 'ref'
-    CLASS_ARGS = 'args'
-    ENABLE = 'enable'
-    POSITION_BIAS_MITIGATION = 'position_bias_mitigation'
-    RANDOM_SEED = 'random_seed'
-    FN_COMPLETION_PARSER = 'fn_completion_parser'
-    COMPLETION_PARSER_KWARGS = 'completion_parser_kwargs'
-    OUTPUT_FILE = 'output_file'
-    MODEL_ID_OR_PATH = 'model_id_or_path'
-    MODEL_REVISION = 'revision'
-    GENERATION_CONFIG = 'generation_config'
-    PRECISION = 'precision'
-    TEMPLATE_TYPE = 'template_type'
-
-
-class FnCompletionParser:
-    LMSYS_PARSER: str = 'lmsys_parser'
-    RANKING_PARSER: str = 'ranking_parser'
-
-
-class PositionBiasMitigation:
-    NONE = 'none'
-    RANDOMIZE_ORDER = 'randomize_order'
-    SWAP_POSITION = 'swap_position'
-
-
-class EvalStage:
-    # Enums: `all`, `infer`, `review`
-    ALL = 'all'
-    INFER = 'infer'
-    REVIEW = 'review'
-
-
 class EvalType:
 
     CUSTOM = 'custom'
-    CHECKPOINT = 'checkpoint'  # native model checkpoint
-    SERVICE = 'service'  # model service
+    MOCK_LLM = 'mock_llm'
+    CHECKPOINT = 'llm_ckpt'  # native model checkpoint
+    SERVICE = 'openai_api'  # model service
+    TEXT2IMAGE = 'text2image'  # image generation service
+    IMAGE_EDITING = 'image_editing'  # image editing service
 
 
 class OutputType:
@@ -142,6 +91,7 @@ class EvalBackend:
 
 class DataCollection:
     NAME = 'data_collection'
+    INFO = 'collection_info'
 
 
 class JudgeStrategy:
@@ -159,3 +109,29 @@ class JudgeScoreType:
 class ModelTask:
     TEXT_GENERATION = 'text_generation'
     IMAGE_GENERATION = 'image_generation'
+
+
+class Tags:
+    KNOWLEDGE = 'Knowledge'
+    MULTIPLE_CHOICE = 'MCQ'
+    MATH = 'Math'
+    REASONING = 'Reasoning'
+    CODING = 'Coding'
+    CHINESE = 'Chinese'
+    COMMONSENSE = 'Commonsense'
+    QA = 'QA'
+    READING_COMPREHENSION = 'ReadingComprehension'
+    CUSTOM = 'Custom'
+    INSTRUCTION_FOLLOWING = 'InstructionFollowing'
+    ARENA = 'Arena'
+    LONG_CONTEXT = 'LongContext'
+    RETRIEVAL = 'Retrieval'
+    FUNCTION_CALLING = 'FunctionCalling'
+    TEXT_TO_IMAGE = 'TextToImage'
+    IMAGE_EDITING = 'ImageEditing'
+    MULTI_MODAL = 'MultiModal'
+
+
+class FileConstants:
+    IMAGE_PATH = 'image_path'
+    ID = 'id'

evalscope/evaluator/__init__.py

@@ -1,3 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from .evaluator import Evaluator
+from .evaluator import DefaultEvaluator