evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/api/metric/metric.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from typing import Callable, Iterable, List, Union
+
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+
+logger = get_logger()
+
+
+class Metric(ABC):
+    """
+    Metric classes operate on a sample level.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Metric class should have state.
+        """
+
+    @abstractmethod
+    def apply(self, predictions: List[str], references: List[str]) -> List[float]:
+        pass
+
+    def __call__(self, prediction: str, reference: str) -> float:
+        """
+        Allows the metric to be called like a function.
+        """
+        return self.apply([prediction], [reference])[0]
+
+
+class T2IMetric(Metric):
+    _instance = None
+
+    @thread_safe
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, *args, **kwargs):
+        cls = self.__class__
+        if hasattr(self, '_init_done'):
+            return
+        logger.info(f'Initializing {cls.__name__}...')
+        self._init_once(*args, **kwargs)
+        self._init_done = True
+
+    def _init_once(self, *args, **kwargs):
+        pass
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[Union[float, dict]]:
+        pass
+
+    def __call__(self, image: str, text: str, **kwargs) -> Union[float, dict]:
+        return self.apply([image], [text], **kwargs)[0]

evalscope/api/metric/scorer.py

@@ -0,0 +1,105 @@
+from pydantic import BaseModel, Field
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+Value = Dict[str, Union[int, float, bool]]
+
+
+class Score(BaseModel):
+    """Score generated by a scorer."""
+
+    value: Value = Field(default_factory=dict)
+    """Score value as a dictionary. Key is the score name, value is the score value.
+    The first key is considered the main score by default."""
+
+    extracted_prediction: Optional[str] = Field(default=None)
+    """Answer extracted from model output (optional)"""
+
+    prediction: Optional[str] = Field(default=None)
+    """Original prediction text from the model (optional)"""
+
+    explanation: Optional[str] = Field(default=None)
+    """Explanation of score (optional)."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict)
+    """Additional metadata related to the score"""
+
+    main_score_name: Optional[str] = Field(default=None)
+    """Main score name, if applicable. This is used to indicate which score is the primary score in a multi-score scenario."""  # noqa: E501
+
+    @property
+    def main_value(self) -> Union[int, float, bool]:
+        """Main score value."""
+        if self.main_score_name and self.main_score_name in self.value:
+            return self.value[self.main_score_name]
+        return next(iter(self.value.values()), None)
+
+    @main_value.setter
+    def main_value(self, value: Union[int, float, bool]):
+        """Set the main score value."""
+        if self.main_score_name:
+            self.value[self.main_score_name] = value
+        else:
+            # If no main score name is set, just update the first value
+            if self.value:
+                first_key = next(iter(self.value))
+                self.value[first_key] = value
+            else:
+                self.value['default'] = value
+
+
+class SampleScore(BaseModel):
+    """Score for a Sample."""
+
+    score: Score
+    """A score"""
+
+    sample_id: Optional[Union[str, int]] = Field(default=None)
+    """A sample id"""
+
+    group_id: Optional[Union[str, int]] = Field(default=None)
+    """A group id for the sample, used for grouping k repeated samples."""
+
+    sample_metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Metadata from the sample"""
+
+
+class AggScore(BaseModel):
+    """Output of an aggregation operation."""
+
+    score: float = Field(default=0.0)
+    """Aggregated value as a float."""
+
+    metric_name: str = Field(default='')
+    """Name of the metric being aggregated."""
+
+    aggregation_name: str = Field(default='')
+    """Name of the aggregation methods"""
+
+    num: int = Field(default=0)
+    """Number of samples used in the aggregation."""
+
+    ids: Optional[List[Union[str, int]]] = Field(default=None)
+    """List of sample IDs used in the aggregation, if applicable."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional metadata related to the aggregation."""
+
+
+class Aggregator:
+
+    name = 'default'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        r"""Aggregate a metric on a list of scores.
+
+        Args:
+          scores: List of scores.
+
+        Returns:
+          List[AggregatOutput]: List of aggregated outputs.
+        """
+        ...

evalscope/api/mixin/__init__.py

@@ -0,0 +1,2 @@
+from .dataset_mixin import DatasetLoaderMixin
+from .llm_judge_mixin import LLMJudgeMixin

evalscope/api/mixin/dataset_mixin.py

@@ -0,0 +1,105 @@
+from abc import ABC
+from collections import defaultdict
+from typing import Any, Callable, Dict
+
+from evalscope.api.dataset import Dataset, DatasetDict, RemoteDataLoader
+
+
+class DatasetLoaderMixin:
+    """
+    Mixin class providing dataset loading functionality for benchmarks.
+
+    This mixin provides common dataset loading methods that can be shared
+    across different data adapters, including support for:
+    - Loading multiple subsets
+    - Few-shot dataset loading
+    - Remote dataset loading with configuration
+    """
+
+    def load_subsets(self, load_func: Callable[[str], Dataset]) -> DatasetDict:
+        """
+        Load multiple subsets of the dataset using the provided loading function.
+
+        This method handles two loading strategies:
+        1. Reformat mode: Load only the default subset and reformat it
+        2. Multi-subset mode: Load all subsets specified in subset_list
+
+        Args:
+            load_func (Callable[[str], Dataset]): Function to load individual subsets
+
+        Returns:
+            DatasetDict: Dictionary containing all loaded subsets
+        """
+        if self.reformat_subset:
+            # Load only the default subset
+            subset_data = load_func(self.default_subset)
+            # Reformat the subset to create multiple subsets based on sample keys
+            # NOTE: subset_list and limit is applied here if specified
+            dataset_dict = DatasetDict.from_dataset(dataset=subset_data, subset_list=self.subset_list, limit=self.limit)
+        else:
+            # Load all specified subsets into separate entries
+            subset_dict = defaultdict()
+            for subset in self.subset_list:
+                subset_data = load_func(subset)
+                subset_dict[subset] = subset_data
+            dataset_dict = DatasetDict(subset_dict)
+        return dataset_dict
+
+    def load_subset(self, subset: str) -> Dataset:
+        """
+        Load a specific subset of the dataset for evaluation.
+
+        This method configures and executes the data loading for a single subset,
+        handling both split-as-subset and traditional subset configurations.
+
+        Args:
+            subset (str): The subset identifier to load
+
+        Returns:
+            Dataset: The loaded dataset subset with processed samples
+        """
+        # Determine the split and subset names based on configuration
+        split = subset if self.split_as_subset else self.eval_split
+        subset_name = self.default_subset if self.split_as_subset else subset
+
+        # Create and configure the remote data loader
+        loader = RemoteDataLoader(
+            data_id_or_path=self.dataset_id,
+            split=split,
+            subset=subset_name,
+            sample_fields=self.record_to_sample,  # Custom sample conversion function
+            limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
+            repeats=self._task_config.repeats,  # Number of repetitions for each sample
+            data_source=self._task_config.dataset_hub,  # Data source configuration
+        )
+        return loader.load()
+
+    def load_fewshot_subset(self, subset: str) -> Dataset:
+        """
+        Load a subset specifically for few-shot examples.
+
+        This method loads training data to be used as demonstrations in few-shot prompting.
+        It typically loads from the training split with limited samples and optional shuffling.
+
+        Args:
+            subset (str): The subset identifier to load few-shot examples from
+
+        Returns:
+            Dataset: The loaded few-shot dataset with demonstration examples
+        """
+        # Use training split for few-shot examples
+        split = subset if self.split_as_subset else self.train_split
+        subset_name = self.default_subset if self.split_as_subset else subset
+
+        # Create loader specifically configured for few-shot sampling
+        loader = RemoteDataLoader(
+            data_id_or_path=self.dataset_id,
+            split=split,
+            subset=subset_name,
+            sample_fields=self.record_to_sample,
+            limit=self.few_shot_num
+            if not self.reformat_subset else None,  # Limit to specified number of few-shot examples
+            shuffle=self.few_shot_random,  # Randomize selection if enabled
+            data_source=self._task_config.dataset_hub,
+        )
+        return loader.load()

evalscope/api/mixin/llm_judge_mixin.py

@@ -0,0 +1,168 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.constants import JudgeStrategy
+from evalscope.metrics import LLMJudge
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LLMJudgeMixin:
+    """
+    Mixin class for LLM Judge functionality.
+    """
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+        self._use_llm_judge = False
+        """Whether to use LLM as a judge"""
+
+        self._llm_judge: Optional[LLMJudge] = None
+
+    @property
+    def llm_judge(self) -> Optional[LLMJudge]:
+        """Get LLM judge instance with lazy initialization."""
+        if self._llm_judge is None and self.use_llm_judge:
+            self._llm_judge = self.init_llm_judge()
+        return self._llm_judge
+
+    @llm_judge.setter
+    def llm_judge(self, value: Optional[LLMJudge]):
+        """Set LLM judge instance."""
+        self._llm_judge = value
+
+    @property
+    def judge_strategy(self) -> str:
+        """Get the judge strategy from the task configuration."""
+        return self._task_config.judge_strategy
+
+    @property
+    def use_llm_judge(self) -> bool:
+        """Check if LLM judge is enabled."""
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return False
+        elif self.judge_strategy == JudgeStrategy.LLM:
+            return True
+        elif self.judge_strategy == JudgeStrategy.LLM_RECALL:
+            return True
+        elif self.judge_strategy == JudgeStrategy.AUTO:
+            return self._use_llm_judge
+        else:
+            logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.')
+            return False
+
+    def init_llm_judge(self) -> Optional[LLMJudge]:
+        """
+        Initialize the LLM judge for the benchmark.
+
+        Returns:
+            Optional[LLMJudge]: The initialized LLM judge instance or None
+        """
+
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return None
+        else:
+            return LLMJudge(**self._task_config.judge_model_args)
+
+    def maybe_llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+        rule_based_score: Optional[Score] = None,
+    ) -> Score:
+        """
+        Compute the match score between the original and filtered predictions against the reference.
+
+        Args:
+            original_prediction: The original prediction output from the model.
+            filtered_prediction: The filtered prediction output from the model.
+            reference: The ground truth reference output.
+            task_state: The current task state.
+            original_score: Optional original score to be used for comparison.
+
+        Returns:
+            Score: The computed match score.
+        """
+        # If LLM judge is not used, return the rule-based score directly
+        if not self.use_llm_judge:
+            return rule_based_score
+
+        # For LLM_RECALL, if rule-based score is already perfect, skip LLM judge
+        if float(rule_based_score.main_value) > 0.99:
+            return rule_based_score
+
+        # Compute LLM judge score
+        llm_score = self.llm_match_score(
+            original_prediction=original_prediction,
+            filtered_prediction=filtered_prediction,
+            reference=reference,
+            task_state=task_state,
+        )
+
+        # For LLM RECALL, merge the scores
+        return self._merge_scores(rule_based_score, llm_score)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """Compute the LLM match score.
+
+        Args:
+            original_prediction (str): The original prediction output from the model.
+            filtered_prediction (str): The filtered prediction output from the model.
+            reference (str): The ground truth reference output.
+            task_state (TaskState): The current task state.
+
+        Returns:
+            Score: The computed match score.
+        """
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = self.llm_judge.build_prompt(pred=original_prediction, gold=reference, question=question)
+        judge_response = self.llm_judge.judge(prompt)
+        judge_score = self.llm_judge.get_score(judge_response)
+
+        score.value = {'acc': judge_score}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+
+        return score
+
+    def _merge_scores(self, rule_based_score: Score, llm_score: Score) -> Score:
+        """
+        Merge rule-based score with LLM judge score for LLM_RECALL strategy.
+
+        Args:
+            rule_based_score: The original rule-based score
+            llm_score: The LLM judge score
+
+        Returns:
+            Score: The merged score
+        """
+        # Update the main value with LLM judge result
+        rule_based_score.main_value = llm_score.main_value
+        rule_based_score.explanation = llm_score.explanation
+        rule_based_score.metadata = llm_score.metadata
+
+        return rule_based_score

evalscope/api/model/__init__.py

@@ -0,0 +1,12 @@
+from .generate_config import GenerateConfig
+from .model import Model, ModelAPI, get_model, get_model_with_task_config
+from .model_output import (
+    ChatCompletionChoice,
+    Logprob,
+    Logprobs,
+    ModelOutput,
+    ModelUsage,
+    StopReason,
+    TopLogprob,
+    as_stop_reason,
+)

evalscope/api/model/generate_config.py

@@ -0,0 +1,157 @@
+# flake8: noqa: E501
+from copy import deepcopy
+from pydantic import BaseModel, Field, model_validator
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from evalscope.utils.json_schema import JSONSchema
+
+
+class ResponseSchema(BaseModel):
+    """Schema for model response when using Structured Output."""
+
+    name: str
+    """The name of the response schema. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64."""
+
+    json_schema: JSONSchema
+    """The schema for the response format, described as a JSON Schema object."""
+
+    description: Optional[str] = Field(default=None)
+    """A description of what the response format is for, used by the model to determine how to respond in the format."""
+
+    strict: Optional[bool] = Field(default=None)
+    """Whether to enable strict schema adherence when generating the output. If set to true, the model will always follow the exact schema defined in the schema field.
+    OpenAI and Mistral only."""
+
+
+class GenerateConfig(BaseModel):
+    """Model generation options."""
+
+    max_retries: Optional[int] = Field(default=None)
+    """Maximum number of times to retry request (defaults to unlimited)."""
+
+    timeout: Optional[int] = Field(default=None)
+    """Request timeout (in seconds)."""
+
+    batch_size: Optional[int] = Field(default=None)
+    """Maximum number of concurrent connections to Model API (default is model specific) or batch size for generation."""
+
+    stream: Optional[bool] = Field(default=None)
+    """Whether to stream the response (default is model specific)."""
+
+    system_message: Optional[str] = Field(default=None)
+    """Override the default system message."""
+
+    max_tokens: Optional[int] = Field(default=None)
+    """The maximum number of tokens that can be generated in the completion (default is model specific)."""
+
+    top_p: Optional[float] = Field(default=None)
+    """An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass."""
+
+    temperature: Optional[float] = Field(default=None)
+    """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic."""
+
+    stop_seqs: Optional[List[str]] = Field(default=None)
+    """Sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."""
+
+    best_of: Optional[int] = Field(default=None)
+    """Generates best_of completions server-side and returns the 'best' (the one with the highest log probability per token). vLLM only."""
+
+    frequency_penalty: Optional[float] = Field(default=None)
+    """Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. OpenAI, Google, Grok, Groq, vLLM, and SGLang only."""
+
+    presence_penalty: Optional[float] = Field(default=None)
+    """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. OpenAI, Google, Grok, Groq, vLLM, and SGLang only."""
+
+    logit_bias: Optional[Dict[int, float]] = Field(default=None)
+    """Map token Ids to an associated bias value from -100 to 100 (e.g. "42=10,43=-10"). OpenAI, Grok, Grok, and vLLM only."""
+
+    seed: Optional[int] = Field(default=None)
+    """Random seed. OpenAI, Google, Mistral, Groq, HuggingFace, and vLLM only."""
+
+    do_sample: Optional[bool] = Field(default=None)
+    """Whether to use sampling; use greedy decoding otherwise. Only transformers models support this parameter."""
+
+    top_k: Optional[int] = Field(default=None)
+    """Randomly sample the next word from the top_k most likely next words. Anthropic, Google, HuggingFace, vLLM, and SGLang only."""
+
+    n: Optional[int] = Field(default=None)
+    """How many chat completion choices to generate for each input message. OpenAI, Grok, Google, TogetherAI, vLLM, and SGLang only."""
+
+    logprobs: Optional[bool] = Field(default=None)
+    """Return log probabilities of the output tokens. OpenAI, Grok, TogetherAI, Huggingface, llama-cpp-python, vLLM, and SGLang only."""
+
+    top_logprobs: Optional[int] = Field(default=None)
+    """Number of most likely tokens (0-20) to return at each token position, each with an associated log probability. OpenAI, Grok, Huggingface, vLLM, and SGLang only."""
+
+    parallel_tool_calls: Optional[bool] = Field(default=None)
+    """Whether to enable parallel function calling during tool use (defaults to True). OpenAI and Groq only."""
+
+    internal_tools: Optional[bool] = Field(default=None)
+    """Whether to automatically map tools to model internal implementations (e.g. 'computer' for anthropic)."""
+
+    max_tool_output: Optional[int] = Field(default=None)
+    """Maximum tool output (in bytes). Defaults to 16 * 1024."""
+
+    cache_prompt: Union[Literal['auto'], bool, None] = Field(default=None)
+    """Whether to cache the prompt prefix. Defaults to "auto", which will enable caching for requests with tools. Anthropic only."""
+
+    reasoning_effort: Optional[Literal['low', 'medium', 'high']] = Field(default=None)
+    """Constrains effort on reasoning for reasoning models (defaults to `medium`). Open AI o1 models only."""
+
+    reasoning_tokens: Optional[int] = Field(default=None)
+    """Maximum number of tokens to use for reasoning. Anthropic Claude models only."""
+
+    reasoning_summary: Optional[Literal['concise', 'detailed', 'auto']] = Field(default=None)
+    """Provide summary of reasoning steps (defaults to no summary). Use 'auto' to access the most detailed summarizer available for the current model. OpenAI reasoning models only."""
+
+    reasoning_history: Optional[Literal['none', 'all', 'last', 'auto']] = Field(default=None)
+    """Include reasoning in chat message history sent to generate."""
+
+    response_schema: Optional[ResponseSchema] = Field(default=None)
+    """Request a response format as JSONSchema (output should still be validated). OpenAI, Google, and Mistral only."""
+
+    extra_body: Optional[Dict[str, Any]] = Field(default=None)
+    """Extra body to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
+    height: Optional[int] = Field(default=None)
+    """Image height for image generation model only"""
+
+    width: Optional[int] = Field(default=None)
+    """Image width for image generation model only"""
+
+    num_inference_steps: Optional[int] = Field(default=None)
+    """Number of inference steps for image generation model only"""
+
+    guidance_scale: Optional[float] = Field(default=None)
+    """Guidance scale for image generation model only"""
+
+    # migrate reasoning_history as a bool
+    @model_validator(mode='before')
+    @classmethod
+    def migrate_reasoning(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            reasoning_history = data.get('reasoning_history', None)
+            if reasoning_history is True:
+                data['reasoning_history'] = 'all'
+            elif reasoning_history is False:
+                data['reasoning_history'] = 'none'
+
+        return data
+
+    def merge(self, other: 'GenerateConfig') -> 'GenerateConfig':
+        """Merge another model configuration into this one.
+
+        Args:
+           other (GenerateConfig):
+              Configuration to merge.
+
+        Returns:
+           Merged configuration.
+        """
+        config_keys = [field for field in self.__class__.model_fields.keys()]
+        config = deepcopy(self)
+        for key in config_keys:
+            value = getattr(other, key, None)
+            if value is not None:
+                setattr(config, key, value)
+        return config