evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/mixin/llm_judge_mixin.py

@@ -0,0 +1,168 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.constants import JudgeStrategy
+from evalscope.metrics import LLMJudge
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LLMJudgeMixin:
+    """
+    Mixin class for LLM Judge functionality.
+    """
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+        self._use_llm_judge = False
+        """Whether to use LLM as a judge"""
+
+        self._llm_judge: Optional[LLMJudge] = None
+
+    @property
+    def llm_judge(self) -> Optional[LLMJudge]:
+        """Get LLM judge instance with lazy initialization."""
+        if self._llm_judge is None and self.use_llm_judge:
+            self._llm_judge = self.init_llm_judge()
+        return self._llm_judge
+
+    @llm_judge.setter
+    def llm_judge(self, value: Optional[LLMJudge]):
+        """Set LLM judge instance."""
+        self._llm_judge = value
+
+    @property
+    def judge_strategy(self) -> str:
+        """Get the judge strategy from the task configuration."""
+        return self._task_config.judge_strategy
+
+    @property
+    def use_llm_judge(self) -> bool:
+        """Check if LLM judge is enabled."""
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return False
+        elif self.judge_strategy == JudgeStrategy.LLM:
+            return True
+        elif self.judge_strategy == JudgeStrategy.LLM_RECALL:
+            return True
+        elif self.judge_strategy == JudgeStrategy.AUTO:
+            return self._use_llm_judge
+        else:
+            logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.')
+            return False
+
+    def init_llm_judge(self) -> Optional[LLMJudge]:
+        """
+        Initialize the LLM judge for the benchmark.
+
+        Returns:
+            Optional[LLMJudge]: The initialized LLM judge instance or None
+        """
+
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return None
+        else:
+            return LLMJudge(**self._task_config.judge_model_args)
+
+    def maybe_llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+        rule_based_score: Optional[Score] = None,
+    ) -> Score:
+        """
+        Compute the match score between the original and filtered predictions against the reference.
+
+        Args:
+            original_prediction: The original prediction output from the model.
+            filtered_prediction: The filtered prediction output from the model.
+            reference: The ground truth reference output.
+            task_state: The current task state.
+            original_score: Optional original score to be used for comparison.
+
+        Returns:
+            Score: The computed match score.
+        """
+        # If LLM judge is not used, return the rule-based score directly
+        if not self.use_llm_judge:
+            return rule_based_score
+
+        # For LLM_RECALL, if rule-based score is already perfect, skip LLM judge
+        if float(rule_based_score.main_value) > 0.99:
+            return rule_based_score
+
+        # Compute LLM judge score
+        llm_score = self.llm_match_score(
+            original_prediction=original_prediction,
+            filtered_prediction=filtered_prediction,
+            reference=reference,
+            task_state=task_state,
+        )
+
+        # For LLM RECALL, merge the scores
+        return self._merge_scores(rule_based_score, llm_score)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """Compute the LLM match score.
+
+        Args:
+            original_prediction (str): The original prediction output from the model.
+            filtered_prediction (str): The filtered prediction output from the model.
+            reference (str): The ground truth reference output.
+            task_state (TaskState): The current task state.
+
+        Returns:
+            Score: The computed match score.
+        """
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = self.llm_judge.build_prompt(pred=original_prediction, gold=reference, question=question)
+        judge_response = self.llm_judge.judge(prompt)
+        judge_score = self.llm_judge.get_score(judge_response)
+
+        score.value = {'acc': judge_score}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+
+        return score
+
+    def _merge_scores(self, rule_based_score: Score, llm_score: Score) -> Score:
+        """
+        Merge rule-based score with LLM judge score for LLM_RECALL strategy.
+
+        Args:
+            rule_based_score: The original rule-based score
+            llm_score: The LLM judge score
+
+        Returns:
+            Score: The merged score
+        """
+        # Update the main value with LLM judge result
+        rule_based_score.main_value = llm_score.main_value
+        rule_based_score.explanation = llm_score.explanation
+        rule_based_score.metadata = llm_score.metadata
+
+        return rule_based_score

evalscope/api/model/__init__.py

@@ -0,0 +1,12 @@
+from .generate_config import GenerateConfig
+from .model import Model, ModelAPI, get_model, get_model_with_task_config
+from .model_output import (
+    ChatCompletionChoice,
+    Logprob,
+    Logprobs,
+    ModelOutput,
+    ModelUsage,
+    StopReason,
+    TopLogprob,
+    as_stop_reason,
+)

evalscope/api/model/generate_config.py

@@ -0,0 +1,155 @@
+# flake8: noqa: E501
+from copy import deepcopy
+from pydantic import BaseModel, Field, model_validator
+from typing import Any, Dict, List, Literal, Optional, Union
+
+from evalscope.utils.json_schema import JSONSchema
+
+
+class ResponseSchema(BaseModel):
+    """Schema for model response when using Structured Output."""
+
+    name: str
+    """The name of the response schema. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64."""
+
+    json_schema: JSONSchema
+    """The schema for the response format, described as a JSON Schema object."""
+
+    description: Optional[str] = Field(default=None)
+    """A description of what the response format is for, used by the model to determine how to respond in the format."""
+
+    strict: Optional[bool] = Field(default=None)
+    """Whether to enable strict schema adherence when generating the output. If set to true, the model will always follow the exact schema defined in the schema field.
+    OpenAI and Mistral only."""
+
+
+class GenerateConfig(BaseModel):
+    """Model generation options."""
+    model_config = {'extra': 'allow'}
+
+    timeout: Optional[int] = Field(default=None)
+    """Request timeout (in seconds)."""
+
+    batch_size: Optional[int] = Field(default=None)
+    """Maximum number of concurrent connections to Model API (default is model specific) or batch size for generation."""
+
+    stream: Optional[bool] = Field(default=None)
+    """Whether to stream the response (default is model specific)."""
+
+    system_message: Optional[str] = Field(default=None)
+    """Override the default system message."""
+
+    max_tokens: Optional[int] = Field(default=None)
+    """The maximum number of tokens that can be generated in the completion (default is model specific)."""
+
+    top_p: Optional[float] = Field(default=None)
+    """An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass."""
+
+    temperature: Optional[float] = Field(default=None)
+    """What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic."""
+
+    stop_seqs: Optional[List[str]] = Field(default=None)
+    """Sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence."""
+
+    best_of: Optional[int] = Field(default=None)
+    """Generates best_of completions server-side and returns the 'best' (the one with the highest log probability per token). vLLM only."""
+
+    frequency_penalty: Optional[float] = Field(default=None)
+    """Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. OpenAI, Google, Grok, Groq, vLLM, and SGLang only."""
+
+    presence_penalty: Optional[float] = Field(default=None)
+    """Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics. OpenAI, Google, Grok, Groq, vLLM, and SGLang only."""
+
+    logit_bias: Optional[Dict[int, float]] = Field(default=None)
+    """Map token Ids to an associated bias value from -100 to 100 (e.g. "42=10,43=-10"). OpenAI, Grok, Grok, and vLLM only."""
+
+    seed: Optional[int] = Field(default=None)
+    """Random seed. OpenAI, Google, Mistral, Groq, HuggingFace, and vLLM only."""
+
+    do_sample: Optional[bool] = Field(default=None)
+    """Whether to use sampling; use greedy decoding otherwise. Only transformers models support this parameter."""
+
+    top_k: Optional[int] = Field(default=None)
+    """Randomly sample the next word from the top_k most likely next words. Anthropic, Google, HuggingFace, vLLM, and SGLang only."""
+
+    n: Optional[int] = Field(default=None)
+    """How many chat completion choices to generate for each input message. OpenAI, Grok, Google, TogetherAI, vLLM, and SGLang only."""
+
+    logprobs: Optional[bool] = Field(default=None)
+    """Return log probabilities of the output tokens. OpenAI, Grok, TogetherAI, Huggingface, llama-cpp-python, vLLM, and SGLang only."""
+
+    top_logprobs: Optional[int] = Field(default=None)
+    """Number of most likely tokens (0-20) to return at each token position, each with an associated log probability. OpenAI, Grok, Huggingface, vLLM, and SGLang only."""
+
+    parallel_tool_calls: Optional[bool] = Field(default=None)
+    """Whether to enable parallel function calling during tool use (defaults to True). OpenAI and Groq only."""
+
+    internal_tools: Optional[bool] = Field(default=None)
+    """Whether to automatically map tools to model internal implementations (e.g. 'computer' for anthropic)."""
+
+    max_tool_output: Optional[int] = Field(default=None)
+    """Maximum tool output (in bytes). Defaults to 16 * 1024."""
+
+    cache_prompt: Union[Literal['auto'], bool, None] = Field(default=None)
+    """Whether to cache the prompt prefix. Defaults to "auto", which will enable caching for requests with tools. Anthropic only."""
+
+    reasoning_effort: Optional[Literal['low', 'medium', 'high']] = Field(default=None)
+    """Constrains effort on reasoning for reasoning models (defaults to `medium`). Open AI o1 models only."""
+
+    reasoning_tokens: Optional[int] = Field(default=None)
+    """Maximum number of tokens to use for reasoning. Anthropic Claude models only."""
+
+    reasoning_summary: Optional[Literal['concise', 'detailed', 'auto']] = Field(default=None)
+    """Provide summary of reasoning steps (defaults to no summary). Use 'auto' to access the most detailed summarizer available for the current model. OpenAI reasoning models only."""
+
+    reasoning_history: Optional[Literal['none', 'all', 'last', 'auto']] = Field(default=None)
+    """Include reasoning in chat message history sent to generate."""
+
+    response_schema: Optional[ResponseSchema] = Field(default=None)
+    """Request a response format as JSONSchema (output should still be validated). OpenAI, Google, and Mistral only."""
+
+    extra_body: Optional[Dict[str, Any]] = Field(default=None)
+    """Extra body to be sent with requests to OpenAI compatible servers. OpenAI, vLLM, and SGLang only."""
+
+    height: Optional[int] = Field(default=None)
+    """Image height for image generation model only"""
+
+    width: Optional[int] = Field(default=None)
+    """Image width for image generation model only"""
+
+    num_inference_steps: Optional[int] = Field(default=None)
+    """Number of inference steps for image generation model only"""
+
+    guidance_scale: Optional[float] = Field(default=None)
+    """Guidance scale for image generation model only"""
+
+    # migrate reasoning_history as a bool
+    @model_validator(mode='before')
+    @classmethod
+    def migrate_reasoning(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            reasoning_history = data.get('reasoning_history', None)
+            if reasoning_history is True:
+                data['reasoning_history'] = 'all'
+            elif reasoning_history is False:
+                data['reasoning_history'] = 'none'
+
+        return data
+
+    def merge(self, other: 'GenerateConfig') -> 'GenerateConfig':
+        """Merge another model configuration into this one.
+
+        Args:
+           other (GenerateConfig):
+              Configuration to merge.
+
+        Returns:
+           Merged configuration.
+        """
+        config_keys = [field for field in self.__class__.model_fields.keys()]
+        config = deepcopy(self)
+        for key in config_keys:
+            value = getattr(other, key, None)
+            if value is not None:
+                setattr(config, key, value)
+        return config