evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/filter/__init__.py

@@ -0,0 +1 @@
+from .filter import Filter, FilterEnsemble, build_filter_ensemble

evalscope/api/filter/filter.py

@@ -0,0 +1,72 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Iterable, List, Union
+
+from evalscope.api.registry import get_filter
+
+
+class Filter(ABC):
+    """
+    Filter classes operate on a sample level.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+        """
+
+    @abstractmethod
+    def apply(self, instance: List[str]) -> List[str]:
+
+        return instance
+
+    def __call__(self, instance: str) -> str:
+        """
+        Allows the filter to be called like a function.
+        """
+        return self.apply([instance])[0]
+
+
+@dataclass
+class FilterEnsemble:
+    """
+    FilterEnsemble creates a pipeline applying multiple filters.
+    Its intended usage is to stack multiple post-processing steps in order.
+    """
+
+    name: str
+    filters: List[Callable[[], Filter]]
+
+    def apply(self, instance: List[str]) -> List[str]:
+
+        for f in self.filters:
+            # apply filters in sequence
+            instance = f.apply(instance)
+
+        return instance
+
+    def __call__(self, instance: str) -> str:
+        """
+        Allows the filter ensemble to be called like a function.
+        """
+        return self.apply([instance])[0]
+
+
+def build_filter_ensemble(name: str = 'default', filters: Dict[str, Any] = {}) -> FilterEnsemble:
+    """
+    Create a filtering pipeline.
+    """
+    filter_funcs = []
+    for filter_name, filter_args in filters.items():
+        filter_cls = get_filter(filter_name)
+        if isinstance(filter_args, list):
+            filter_function = filter_cls(*filter_args)
+        elif isinstance(filter_args, dict):
+            filter_function = filter_cls(**filter_args)
+        else:
+            # Assume single value for simple filters
+            filter_function = filter_cls(filter_args)
+        # add the filter as a pipeline step
+        filter_funcs.append(filter_function)
+
+    return FilterEnsemble(name=name, filters=filter_funcs)

evalscope/api/messages/__init__.py

@@ -0,0 +1,12 @@
+from .chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageSystem,
+    ChatMessageTool,
+    ChatMessageUser,
+    dict_to_chat_message,
+    messages_pretty_str,
+    messages_to_markdown,
+)
+from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo
+from .utils import parse_content_with_reasoning

evalscope/api/messages/chat_message.py

@@ -0,0 +1,243 @@
+import uuid
+from pydantic import BaseModel, Field, JsonValue, model_validator
+from typing import Any, Dict, List, Literal, Optional, Type, Union
+
+from evalscope.api.tool import ToolCall, ToolCallError
+from .content import Content, ContentImage, ContentReasoning, ContentText
+from .utils import parse_content_with_reasoning
+
+
+class ChatMessageBase(BaseModel):
+    """Base class for chat messages."""
+
+    id: Optional[str] = Field(default=None)
+    """Unique identifer for message."""
+
+    content: Union[str, List[Content]]
+    """Content (simple string or list of content objects)"""
+
+    source: Optional[Literal['input', 'generate']] = Field(default=None)
+    """Source of message."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional message metadata."""
+
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+    def model_post_init(self, __context: Any) -> None:
+        # Generate ID
+        if self.id is None:
+            self.id = uuid.uuid4().hex[:8]  # Shorten to 8 characters for simplicity
+
+    @property
+    def text(self) -> str:
+        """Get the text content of this message.
+
+        ChatMessage content is very general and can contain either
+        a simple text value or a list of content parts (each of which
+        can either be text or an image). Solvers (e.g. for prompt
+        engineering) often need to interact with chat messages with
+        the assumption that they are a simple string. The text
+        property returns either the plain str content, or if the
+        content is a list of text and images, the text items
+        concatenated together (separated by newline)
+        """
+        if isinstance(self.content, str):
+            return self.content
+        else:
+            all_text = [content.text for content in self.content if content.type == 'text']
+            return '\n'.join(all_text)
+
+    @text.setter
+    def text(self, text: str) -> None:
+        """Set the primary text content for this message.
+
+        ChatMessage content is very general and can contain either
+        a simple text value or a list of content parts (each of which
+        can either be text or an image). Solvers (e.g. for prompt
+        engineering) often need to interact with chat messages with
+        the assumption that they are a simple string. The text property
+        sets text either to content directly (if it is a `str`) or to
+        the first text content item in the message (inserting one at
+        the beginning if necessary). If there are multiple text content
+        items in the message then after the set there will be only
+        one remaining (image content will remain).
+        """
+        if isinstance(self.content, str):
+            self.content = text
+        else:
+            all_other = [content for content in self.content if content.type != 'text']
+            self.content = all_other + [ContentText(text=text)]
+
+
+class ChatMessageSystem(ChatMessageBase):
+    """System chat message."""
+
+    role: Literal['system'] = Field(default='system')
+    """Conversation role."""
+
+
+class ChatMessageUser(ChatMessageBase):
+    """User chat message."""
+
+    role: Literal['user'] = Field(default='user')
+    """Conversation role."""
+
+    tool_call_id: Optional[List[str]] = Field(default=None)
+    """ID(s) of tool call(s) this message has the content payload for."""
+
+
+class ChatMessageAssistant(ChatMessageBase):
+    """Assistant chat message."""
+
+    role: Literal['assistant'] = Field(default='assistant')
+    """Conversation role."""
+
+    tool_calls: Optional[List[ToolCall]] = Field(default=None)
+    """Tool calls made by the model."""
+
+    model: Optional[str] = Field(default=None)
+    """Model used to generate assistant message."""
+
+    # Some OpenAI compatible REST endpoints include reasoning as a field alongside
+    # content, however since this field doesn't exist in the OpenAI interface,
+    # hosting providers (so far we've seen this with Together and Groq) may
+    # include the reasoning in a <think></think> tag before the main response.
+    # We expect this pattern to be repeated elsewhere, so include this hook to
+    # automatically extract the reasoning content when the response is prefaced
+    # with a <think> block. If this ends up being an overeach we can fall back
+    # to each provider manually parsing out <think> using a helper function.
+    # The implementation isn't important here, the critical thing to establish
+    # is that EvalScope makes reasoning content available separately.
+    @model_validator(mode='before')
+    @classmethod
+    def extract_reasoning(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            # cleave apart <think> blocks
+            content = data.get('content', None)
+            if isinstance(content, str):
+                content_text, content_reasoning = parse_content_with_reasoning(content)
+                if content_reasoning:
+                    data['content'] = [
+                        content_reasoning,
+                        ContentText(text=content_text),
+                    ]
+            # migrate messages that has explicit 'reasoning' field
+            # (which was our original representation of reasoning)
+            reasoning = data.get('reasoning', None)
+            if isinstance(reasoning, str):
+                # ensure that content is a list
+                content = data.get('content', None)
+                if content is None:
+                    data['content'] = []
+                elif isinstance(content, str):
+                    data['content'] = [ContentText(text=content)]
+                elif not isinstance(content, list):
+                    data['content'] = []
+                data['content'].insert(0, ContentReasoning(reasoning=reasoning))
+
+                del data['reasoning']
+        return data
+
+
+class ChatMessageTool(ChatMessageBase):
+    """Tool chat message."""
+
+    role: Literal['tool'] = Field(default='tool')
+    """Conversation role."""
+
+    tool_call_id: Optional[str] = Field(default=None)
+    """ID of tool call."""
+
+    function: Optional[str] = Field(default=None)
+    """Name of function called."""
+
+    error: Optional[ToolCallError] = Field(default=None)
+    """Error which occurred during tool call."""
+
+
+ChatMessage = Union[ChatMessageSystem, ChatMessageUser, ChatMessageAssistant, ChatMessageTool]
+"""Message in a chat conversation"""
+
+
+def dict_to_chat_message(data: Dict[str, Any]) -> ChatMessage:
+    """Convert a dictionary to a ChatMessage."""
+
+    if isinstance(data, ChatMessage):
+        return data
+
+    if 'role' not in data:
+        raise ValueError('ChatMessage must have a "role" field')
+
+    role = data['role']
+    if role == 'system':
+        return ChatMessageSystem.model_validate(data)
+    elif role == 'user':
+        return ChatMessageUser.model_validate(data)
+    elif role == 'assistant':
+        return ChatMessageAssistant.model_validate(data)
+    elif role == 'tool':
+        return ChatMessageTool.model_validate(data)
+    else:
+        raise ValueError(f'Unknown chat message role: {role}')
+
+
+def messages_pretty_str(messages: List[ChatMessage]) -> str:
+    """Pretty print a list of chat messages. Without images or other multi-modal contents."""
+    output = []
+    for message in messages:
+        role = message.role.capitalize()
+        content = message.text
+        if isinstance(message, ChatMessageTool):
+            if message.error:
+                content += f'\nError: {message.error.message}'
+            if message.function:
+                content += f'\nFunction: {message.function}'
+        output.append(f'**{role}**: {content}')
+    return '\n\n'.join(output)
+
+
+def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int] = None) -> str:
+    """Convert a list of chat messages to markdown format.
+
+    Args:
+        messages (List[ChatMessage]): The list of chat messages to convert.
+        max_length (Optional[int]): If provided, truncates the base64 string of images to this length.
+    """
+    output = []
+    for message in messages:
+        role = message.role.capitalize()
+
+        # Start with role header
+        content_parts = [f'**{role}**: ']
+
+        # Handle content based on type
+        if isinstance(message.content, str):
+            content_parts.append(message.content)
+        else:
+            for content_item in message.content:
+                if isinstance(content_item, ContentText):
+                    content_parts.append(content_item.text)
+                elif isinstance(content_item, ContentImage):
+                    # Use markdown image syntax
+                    image_base64 = content_item.image
+                    if max_length and len(image_base64) > max_length:
+                        image_base64 = image_base64[:max_length]
+                    content_parts.append(f'![image]({image_base64})')
+                elif isinstance(content_item, ContentReasoning):
+                    content_parts.append(f'**Reasoning:** {content_item.reasoning}')
+
+        # Add tool-specific information
+        if isinstance(message, ChatMessageTool):
+            if message.error:
+                content_parts.append(f'**Error:** {message.error.message}')
+            if message.function:
+                content_parts.append(f'**Function:** {message.function}')
+        elif isinstance(message, ChatMessageAssistant) and message.tool_calls:
+            for tool_call in message.tool_calls:
+                content_parts.append(f'**Tool Call:** {tool_call.function}')
+
+        output.append('\n'.join(content_parts))
+
+    return '\n\n'.join(output)

evalscope/api/messages/content.py

@@ -0,0 +1,102 @@
+from pydantic import BaseModel, Field, JsonValue
+from typing import Dict, Literal, Optional, Sequence, Union
+
+
+class ContentBase(BaseModel):
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentText(ContentBase):
+    """Text content."""
+
+    type: Literal['text'] = Field(default='text')
+    """Type."""
+
+    text: str
+    """Text content."""
+
+    refusal: Optional[bool] = Field(default=None)
+    """Was this a refusal message?"""
+
+
+class ContentReasoning(ContentBase):
+    """Reasoning content.
+
+    See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
+    """  # noqa: E501
+
+    type: Literal['reasoning'] = Field(default='reasoning')
+    """Type."""
+
+    reasoning: str
+    """Reasoning content."""
+
+    signature: Optional[str] = Field(default=None)
+    """Signature for reasoning content (used by some models to ensure that reasoning content is not modified for replay)"""  # noqa: E501
+
+    redacted: bool = Field(default=False)
+    """Indicates that the explicit content of this reasoning block has been redacted."""
+
+
+class ContentImage(ContentBase):
+    """Image content."""
+
+    type: Literal['image'] = Field(default='image')
+    """Type."""
+
+    image: str
+    """Either a URL of the image or the base64 encoded image data."""
+
+    detail: Literal['auto', 'low', 'high'] = Field(default='auto')
+    """Specifies the detail level of the image.
+
+    Currently only supported for OpenAI. Learn more in the    [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+    """  # noqa: E501
+
+
+class ContentAudio(ContentBase):
+    """Audio content."""
+
+    type: Literal['audio'] = Field(default='audio')
+    """Type."""
+
+    audio: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['wav', 'mp3']
+    """Format of audio data ('mp3' or 'wav')"""
+
+
+class ContentVideo(ContentBase):
+    """Video content."""
+
+    type: Literal['video'] = Field(default='video')
+    """Type."""
+
+    video: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['mp4', 'mpeg', 'mov']
+    """Format of video data ('mp4', 'mpeg', or 'mov')"""
+
+
+class ContentData(ContentBase):
+    """Model internal."""
+
+    type: Literal['data'] = Field(default='data')
+    """Type."""
+
+    data: Dict[str, JsonValue]
+    """Model provider specific payload - required for internal content."""
+
+
+Content = Union[
+    ContentText,
+    ContentReasoning,
+    ContentImage,
+    ContentAudio,
+    ContentVideo,
+    ContentData,
+]
+"""Content sent to or received from a model."""

evalscope/api/messages/utils.py

@@ -0,0 +1,35 @@
+import re
+from typing import Optional
+
+from .content import ContentReasoning
+
+
+def parse_content_with_reasoning(content: str) -> tuple[str, Optional[ContentReasoning]]:
+    """
+    Looks for and extracts <think/> tags into reasoning text.
+
+    Returns a tuple:
+    - The first element is the input content with the <think> tag and its contents fully removed.
+    - The second element is a ContentReasoning object (or None if no <think> tag is found).
+    """
+    # Match <think> tag with optional attributes anywhere in the string
+    pattern = (r'<think(?:\s+signature="([^"]*)")?(?:\s+redacted="(true)")?\s*>(.*?)</think>')
+    match = re.search(pattern, content, re.DOTALL)
+
+    if match:
+        signature = match.group(1)  # This will be None if not present
+        redacted_value = match.group(2)  # This will be "true" or None
+        reasoning = match.group(3).strip()
+        # Remove the matched <think>...</think> from the input
+        start, end = match.span()
+
+        return (
+            (content[:start] + content[end:]).strip(),
+            ContentReasoning(
+                reasoning=reasoning,
+                signature=signature,
+                redacted=redacted_value == 'true',
+            ),
+        )
+    else:
+        return content, None

evalscope/api/metric/__init__.py

@@ -0,0 +1,2 @@
+from .metric import Metric, T2IMetric
+from .scorer import Aggregator, AggScore, SampleScore, Score, Value

evalscope/api/metric/metric.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from typing import Callable, Iterable, List, Union
+
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+
+logger = get_logger()
+
+
+class Metric(ABC):
+    """
+    Metric classes operate on a sample level.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Metric class should have state.
+        """
+
+    @abstractmethod
+    def apply(self, predictions: List[str], references: List[str]) -> List[float]:
+        pass
+
+    def __call__(self, prediction: str, reference: str) -> float:
+        """
+        Allows the metric to be called like a function.
+        """
+        return self.apply([prediction], [reference])[0]
+
+
+class T2IMetric(Metric):
+    _instance = None
+
+    @thread_safe
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, *args, **kwargs):
+        cls = self.__class__
+        if hasattr(self, '_init_done'):
+            return
+        logger.info(f'Initializing {cls.__name__}...')
+        self._init_once(*args, **kwargs)
+        self._init_done = True
+
+    def _init_once(self, *args, **kwargs):
+        pass
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[Union[float, dict]]:
+        pass
+
+    def __call__(self, image: str, text: str, **kwargs) -> Union[float, dict]:
+        return self.apply([image], [text], **kwargs)[0]

evalscope/api/metric/scorer.py

@@ -0,0 +1,113 @@
+from pydantic import BaseModel, Field
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+Value = Dict[str, Union[int, float, bool]]
+
+
+class Score(BaseModel):
+    """Score generated by a scorer."""
+
+    value: Value = Field(default_factory=dict)
+    """Score value as a dictionary. Key is the score name, value is the score value.
+    The first key is considered the main score by default."""
+
+    extracted_prediction: Optional[str] = Field(default=None)
+    """Answer extracted from model output (optional)"""
+
+    prediction: Optional[str] = Field(default=None)
+    """Original prediction text from the model (optional)"""
+
+    explanation: Optional[str] = Field(default=None)
+    """Explanation of score (optional)."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict)
+    """Additional metadata related to the score"""
+
+    main_score_name: Optional[str] = Field(default=None)
+    """Main score name, if applicable. This is used to indicate which score is the primary score in a multi-score scenario."""  # noqa: E501
+
+    @property
+    def main_value(self) -> Union[int, float, bool]:
+        """Main score value."""
+        if self.main_score_name and self.main_score_name in self.value:
+            return self.value[self.main_score_name]
+        elif self.value:
+            # If main_score_name is not set or not found, use the first value and update main_score_name
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            return self.value[first_key]
+        return None
+
+    @main_value.setter
+    def main_value(self, value: Union[int, float, bool]):
+        """Set the main score value."""
+        if self.main_score_name:
+            # If main_score_name is already set, use it
+            self.value[self.main_score_name] = value
+        elif self.value:
+            # If no main_score_name but value dict exists, use the first key
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            self.value[first_key] = value
+        else:
+            # If neither main_score_name nor value dict exists, initialize both
+            self.main_score_name = 'default'
+            self.value[self.main_score_name] = value
+
+
+class SampleScore(BaseModel):
+    """Score for a Sample."""
+
+    score: Score
+    """A score"""
+
+    sample_id: Optional[Union[str, int]] = Field(default=None)
+    """A sample id"""
+
+    group_id: Optional[Union[str, int]] = Field(default=None)
+    """A group id for the sample, used for grouping k repeated samples."""
+
+    sample_metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Metadata from the sample"""
+
+
+class AggScore(BaseModel):
+    """Output of an aggregation operation."""
+
+    score: float = Field(default=0.0)
+    """Aggregated value as a float."""
+
+    metric_name: str = Field(default='')
+    """Name of the metric being aggregated."""
+
+    aggregation_name: str = Field(default='')
+    """Name of the aggregation methods"""
+
+    num: int = Field(default=0)
+    """Number of samples used in the aggregation."""
+
+    ids: Optional[List[Union[str, int]]] = Field(default=None)
+    """List of sample IDs used in the aggregation, if applicable."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional metadata related to the aggregation."""
+
+
+class Aggregator:
+
+    name = 'default'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        r"""Aggregate a metric on a list of scores.
+
+        Args:
+          scores: List of scores.
+
+        Returns:
+          List[AggregatOutput]: List of aggregated outputs.
+        """
+        ...

evalscope/api/mixin/__init__.py

@@ -0,0 +1 @@
+from .llm_judge_mixin import LLMJudgeMixin