evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/api/evaluator/state.py

@@ -0,0 +1,264 @@
+from dataclasses import dataclass
+from random import Random
+from typing import Any, Dict, List, Optional, Sequence, Union, overload
+
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessage, ChatMessageUser, messages_pretty_str
+from evalscope.api.model import ModelOutput
+
+
+class Target(Sequence[str]):
+    """Target for scoring against the current TaskState.
+
+    Target is a sequence of one or more strings. Use the
+    `text` property to access the value as a single string.
+    """
+
+    def __init__(self, target: Union[str, List[str]]) -> None:
+        self.target = target if isinstance(target, list) else [target]
+
+    @overload
+    def __getitem__(self, index: int) -> str:
+        ...
+
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[str]:
+        ...
+
+    def __getitem__(self, index: Union[int, slice]) -> Union[str, Sequence[str]]:
+        return self.target[index]
+
+    def __len__(self) -> int:
+        return len(self.target)
+
+    @property
+    def text(self) -> str:
+        return ''.join(self.target)
+
+
+@dataclass
+class Choice:
+    """
+    A `Choice` represents a single choice in a multiple choice question.
+
+    It is only relevant for the `multiple_choice` solver and corresponding
+    `choice` scorer.
+    """
+
+    value: str
+    """The original value of the choice from the `Sample`."""
+
+    correct: Optional[bool]
+    """Did the model think this choice satisfies the question? `None`
+    indicates this has not been set yet"""
+
+    original_position: int
+    """Choices may be re-ordered during processing, this represents the
+    original position in the sample's list of choices"""
+
+
+class Choices(Sequence[Choice]):
+    """
+    Wrapper class for a list of `Choice` objects.
+
+    Primarily simply to abstract away implementations of choice-specific
+    functionality from the already-big `TaskState` class.
+    """
+
+    def __init__(self, choices: Union[List[str], List[Choice]]) -> None:
+        """
+        Setter for choices, intended to only be used with the `multiple_choice` scorer.
+
+        Choices come from a list of choices for the sample, specifically used by
+        the `multiple_choice` scorer.
+
+        For example, if the sample was a multiple choice question like "What is
+        the capital of France? A) Paris B) London C) Berlin", we would store the
+        possible answers here.
+        """
+        self._choices: List[Choice] = []
+
+        for i, choice in enumerate(choices):
+            if isinstance(choice, str):
+                self._choices.append(Choice(value=choice, correct=None, original_position=i))
+            elif isinstance(choice, Choice):
+                self._choices.append(choice)
+
+    @overload
+    def __getitem__(self, index: int) -> Choice:
+        ...
+
+    @overload
+    def __getitem__(self, index: slice) -> Sequence[Choice]:
+        ...
+
+    def __getitem__(self, index: Union[int, slice]) -> Union[Choice, Sequence[Choice]]:
+        return self._choices[index]
+
+    def __len__(self) -> int:
+        return len(self._choices)
+
+    def mark_choice(self, index: int, correct: bool) -> None:
+        """Set the value of a specific choice"""
+        self._choices[index].correct = correct
+
+    def shuffle(self, rand: Random = Random()) -> None:
+        """
+        Shuffle the choice order, setting the `original_position` so they can be mapped back to their original order.
+
+        Some evals will shuffle the choices from the original sample to try to
+        avoid the model answering correctly due to fine-tuning (or similar) on
+        specific datasets.
+        """
+        shuffled_positions = list(range(len(self._choices)))
+        rand.shuffle(shuffled_positions)
+
+        shuffled_choices = [Choice('notachoice', None, -1)] * len(self._choices)
+
+        for i, shuffled_position in enumerate(shuffled_positions):
+            shuffled_choices[i] = self._choices[shuffled_position]
+            shuffled_choices[i].original_position = shuffled_position
+
+        self._choices = shuffled_choices
+
+
+class TaskState:
+    """
+    The `TaskState` represents the internal state of the `Task` being run for a single `Sample`.
+
+    The `TaskState` is passed to and returned from each solver during a sample's
+    evaluation. It allows us to maintain the manipulated message history, the tools
+    available to the model, the final output of the model, and whether the task
+    is completed or has hit a limit.
+    """
+
+    def __init__(
+        self,
+        model: str,
+        sample: Sample,
+        messages: List[ChatMessage] = [],
+        output: Optional[ModelOutput] = None,
+        completed: bool = False,
+    ) -> None:
+        self._model = model
+        self._sample = sample
+        self._sample_id = sample.id
+        self._group_id = sample.group_id
+        self._input = sample.input
+        self._target = Target(sample.target)
+        self._metadata = sample.metadata
+        self._messages: List[ChatMessage] = messages
+        self._output = output if output else ModelOutput(model=str(model))
+        self._completed = completed
+        if sample.choices:
+            self._choices = Choices(sample.choices)
+        else:
+            self._choices = Choices([])
+
+    @property
+    def model(self) -> str:
+        """Name of model being evaluated."""
+        return self._model
+
+    @property
+    def sample_id(self) -> int:
+        """Unique id for sample."""
+        return self._sample_id
+
+    @property
+    def group_id(self) -> int:
+        """Group id for sample."""
+        return self._group_id
+
+    @property
+    def input(self) -> Union[str, List[ChatMessage]]:
+        """Input from the `Sample`, should be considered immutable."""
+        return self._input
+
+    @property
+    def input_text(self) -> str:
+        """
+        Convenience function for accessing the initial input from the `Sample` as a string.
+
+        If the `input` is a `List[ChatMessage]`, this will return the text from
+        the last chat message
+        """
+        if isinstance(self._input, str):
+            return self._input
+        else:
+            return messages_pretty_str(self._input)
+
+    @property
+    def choices(self) -> Choices:
+        """Choices for the sample, if applicable."""
+        return self._choices
+
+    @property
+    def user_prompt(self) -> ChatMessageUser:
+        """User prompt for this state.
+
+        Tasks are very general and can have may types of inputs.
+        However, in many cases solvers assume they can interact with
+        the state as a "chat" in a predictable fashion (e.g. prompt
+        engineering solvers). This property enables easy read and
+        write access to the user chat prompt. Raises an
+        exception if there is no user prompt
+        """
+        prompt = next((m for m in reversed(self.messages) if m.role == 'user'), None)
+        if prompt:
+            return prompt
+        else:
+            raise ValueError('user_prompt requested from TaskState but none available')
+
+    @property
+    def metadata(self) -> Dict[str, Any]:
+        """Metadata from the `Sample` for this `TaskState`"""
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, metadata: Dict[str, Any]) -> None:
+        self._metadata = metadata
+
+    @property
+    def messages(self) -> List[ChatMessage]:
+        """
+        Chat conversation history for sample.
+
+        This will generally get appended to every time a `generate` call is made
+        to the model. Useful for both debug and for solvers/scorers to assess
+        model performance or choose the next step.
+        """
+        return self._messages
+
+    @messages.setter
+    def messages(self, messages: List[ChatMessage]) -> None:
+        self._messages = messages
+
+    @property
+    def output(self) -> ModelOutput:
+        """
+        The 'final' model output once we've completed all solving.
+
+        For simple evals this may just be the last `message` from the
+        conversation history, but more complex solvers may set this directly.
+        """
+        return self._output
+
+    @output.setter
+    def output(self, output: ModelOutput) -> None:
+        self._output = output
+
+    @property
+    def completed(self) -> bool:
+        """Is the task completed."""
+        return self._completed
+
+    @completed.setter
+    def completed(self, completed: bool) -> None:
+        """Set the completed status."""
+        self._completed = completed
+
+    @property
+    def target(self) -> str:
+        """The scoring target for this `Sample`."""
+        return self._target.text

evalscope/api/filter/__init__.py

@@ -0,0 +1 @@
+from .filter import Filter, FilterEnsemble, build_filter_ensemble

evalscope/api/filter/filter.py

@@ -0,0 +1,72 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, Iterable, List, Union
+
+from evalscope.api.registry import get_filter
+
+
+class Filter(ABC):
+    """
+    Filter classes operate on a sample level.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Filter class should have state.
+        """
+
+    @abstractmethod
+    def apply(self, instance: List[str]) -> List[str]:
+
+        return instance
+
+    def __call__(self, instance: str) -> str:
+        """
+        Allows the filter to be called like a function.
+        """
+        return self.apply([instance])[0]
+
+
+@dataclass
+class FilterEnsemble:
+    """
+    FilterEnsemble creates a pipeline applying multiple filters.
+    Its intended usage is to stack multiple post-processing steps in order.
+    """
+
+    name: str
+    filters: List[Callable[[], Filter]]
+
+    def apply(self, instance: List[str]) -> List[str]:
+
+        for f in self.filters:
+            # apply filters in sequence
+            instance = f.apply(instance)
+
+        return instance
+
+    def __call__(self, instance: str) -> str:
+        """
+        Allows the filter ensemble to be called like a function.
+        """
+        return self.apply([instance])[0]
+
+
+def build_filter_ensemble(name: str = 'default', filters: Dict[str, Any] = {}) -> FilterEnsemble:
+    """
+    Create a filtering pipeline.
+    """
+    filter_funcs = []
+    for filter_name, filter_args in filters.items():
+        filter_cls = get_filter(filter_name)
+        if isinstance(filter_args, list):
+            filter_function = filter_cls(*filter_args)
+        elif isinstance(filter_args, dict):
+            filter_function = filter_cls(**filter_args)
+        else:
+            # Assume single value for simple filters
+            filter_function = filter_cls(filter_args)
+        # add the filter as a pipeline step
+        filter_funcs.append(filter_function)
+
+    return FilterEnsemble(name=name, filters=filter_funcs)

evalscope/api/messages/__init__.py

@@ -0,0 +1,11 @@
+from .chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageSystem,
+    ChatMessageTool,
+    ChatMessageUser,
+    dict_to_chat_message,
+    messages_pretty_str,
+)
+from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo
+from .utils import parse_content_with_reasoning

evalscope/api/messages/chat_message.py

@@ -0,0 +1,198 @@
+import uuid
+from pydantic import BaseModel, Field, JsonValue, model_validator
+from typing import Any, Dict, List, Literal, Optional, Type, Union
+
+from evalscope.api.tool import ToolCall, ToolCallError
+from .content import Content, ContentReasoning, ContentText
+from .utils import parse_content_with_reasoning
+
+
+class ChatMessageBase(BaseModel):
+    """Base class for chat messages."""
+
+    id: Optional[str] = Field(default=None)
+    """Unique identifer for message."""
+
+    content: Union[str, List[Content]]
+    """Content (simple string or list of content objects)"""
+
+    source: Optional[Literal['input', 'generate']] = Field(default=None)
+    """Source of message."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional message metadata."""
+
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+    def model_post_init(self, __context: Any) -> None:
+        # Generate ID
+        if self.id is None:
+            self.id = uuid.uuid4().hex[:8]  # Shorten to 8 characters for simplicity
+
+    @property
+    def text(self) -> str:
+        """Get the text content of this message.
+
+        ChatMessage content is very general and can contain either
+        a simple text value or a list of content parts (each of which
+        can either be text or an image). Solvers (e.g. for prompt
+        engineering) often need to interact with chat messages with
+        the assumption that they are a simple string. The text
+        property returns either the plain str content, or if the
+        content is a list of text and images, the text items
+        concatenated together (separated by newline)
+        """
+        if isinstance(self.content, str):
+            return self.content
+        else:
+            all_text = [content.text for content in self.content if content.type == 'text']
+            return '\n'.join(all_text)
+
+    @text.setter
+    def text(self, text: str) -> None:
+        """Set the primary text content for this message.
+
+        ChatMessage content is very general and can contain either
+        a simple text value or a list of content parts (each of which
+        can either be text or an image). Solvers (e.g. for prompt
+        engineering) often need to interact with chat messages with
+        the assumption that they are a simple string. The text property
+        sets text either to content directly (if it is a `str`) or to
+        the first text content item in the message (inserting one at
+        the beginning if necessary). If there are multiple text content
+        items in the message then after the set there will be only
+        one remaining (image content will remain).
+        """
+        if isinstance(self.content, str):
+            self.content = text
+        else:
+            all_other = [content for content in self.content if content.type != 'text']
+            self.content = all_other + [ContentText(text=text)]
+
+
+class ChatMessageSystem(ChatMessageBase):
+    """System chat message."""
+
+    role: Literal['system'] = Field(default='system')
+    """Conversation role."""
+
+
+class ChatMessageUser(ChatMessageBase):
+    """User chat message."""
+
+    role: Literal['user'] = Field(default='user')
+    """Conversation role."""
+
+    tool_call_id: Optional[List[str]] = Field(default=None)
+    """ID(s) of tool call(s) this message has the content payload for."""
+
+
+class ChatMessageAssistant(ChatMessageBase):
+    """Assistant chat message."""
+
+    role: Literal['assistant'] = Field(default='assistant')
+    """Conversation role."""
+
+    tool_calls: Optional[List[ToolCall]] = Field(default=None)
+    """Tool calls made by the model."""
+
+    model: Optional[str] = Field(default=None)
+    """Model used to generate assistant message."""
+
+    # Some OpenAI compatible REST endpoints include reasoning as a field alongside
+    # content, however since this field doesn't exist in the OpenAI interface,
+    # hosting providers (so far we've seen this with Together and Groq) may
+    # include the reasoning in a <think></think> tag before the main response.
+    # We expect this pattern to be repeated elsewhere, so include this hook to
+    # automatically extract the reasoning content when the response is prefaced
+    # with a <think> block. If this ends up being an overeach we can fall back
+    # to each provider manually parsing out <think> using a helper function.
+    # The implementation isn't important here, the critical thing to establish
+    # is that EvalScope makes reasoning content available separately.
+    @model_validator(mode='before')
+    @classmethod
+    def extract_reasoning(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            # cleave apart <think> blocks
+            content = data.get('content', None)
+            if isinstance(content, str):
+                content_text, content_reasoning = parse_content_with_reasoning(content)
+                if content_reasoning:
+                    data['content'] = [
+                        content_reasoning,
+                        ContentText(text=content_text),
+                    ]
+            # migrate messages that has explicit 'reasoning' field
+            # (which was our original representation of reasoning)
+            reasoning = data.get('reasoning', None)
+            if isinstance(reasoning, str):
+                # ensure that content is a list
+                content = data.get('content', None)
+                if content is None:
+                    data['content'] = []
+                elif isinstance(content, str):
+                    data['content'] = [ContentText(text=content)]
+                elif not isinstance(content, list):
+                    data['content'] = []
+                data['content'].insert(0, ContentReasoning(reasoning=reasoning))
+
+                del data['reasoning']
+        return data
+
+
+class ChatMessageTool(ChatMessageBase):
+    """Tool chat message."""
+
+    role: Literal['tool'] = Field(default='tool')
+    """Conversation role."""
+
+    tool_call_id: Optional[str] = Field(default=None)
+    """ID of tool call."""
+
+    function: Optional[str] = Field(default=None)
+    """Name of function called."""
+
+    error: Optional[ToolCallError] = Field(default=None)
+    """Error which occurred during tool call."""
+
+
+ChatMessage = Union[ChatMessageSystem, ChatMessageUser, ChatMessageAssistant, ChatMessageTool]
+"""Message in a chat conversation"""
+
+
+def dict_to_chat_message(data: Dict[str, Any]) -> ChatMessage:
+    """Convert a dictionary to a ChatMessage."""
+
+    if isinstance(data, ChatMessage):
+        return data
+
+    if 'role' not in data:
+        raise ValueError('ChatMessage must have a "role" field')
+
+    role = data['role']
+    if role == 'system':
+        return ChatMessageSystem.model_validate(data)
+    elif role == 'user':
+        return ChatMessageUser.model_validate(data)
+    elif role == 'assistant':
+        return ChatMessageAssistant.model_validate(data)
+    elif role == 'tool':
+        return ChatMessageTool.model_validate(data)
+    else:
+        raise ValueError(f'Unknown chat message role: {role}')
+
+
+def messages_pretty_str(messages: List[ChatMessage]) -> str:
+    """Pretty print a list of chat messages."""
+    output = []
+    for message in messages:
+        role = message.role.capitalize()
+        content = message.text
+        if isinstance(message, ChatMessageTool):
+            if message.error:
+                content += f'\nError: {message.error.message}'
+            if message.function:
+                content += f'\nFunction: {message.function}'
+        output.append(f'**{role}**: {content}')
+    return '\n\n'.join(output)

evalscope/api/messages/content.py

@@ -0,0 +1,102 @@
+from pydantic import BaseModel, Field, JsonValue
+from typing import Dict, Literal, Optional, Sequence, Union
+
+
+class ContentBase(BaseModel):
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentText(ContentBase):
+    """Text content."""
+
+    type: Literal['text'] = Field(default='text')
+    """Type."""
+
+    text: str
+    """Text content."""
+
+    refusal: Optional[bool] = Field(default=None)
+    """Was this a refusal message?"""
+
+
+class ContentReasoning(ContentBase):
+    """Reasoning content.
+
+    See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
+    """  # noqa: E501
+
+    type: Literal['reasoning'] = Field(default='reasoning')
+    """Type."""
+
+    reasoning: str
+    """Reasoning content."""
+
+    signature: Optional[str] = Field(default=None)
+    """Signature for reasoning content (used by some models to ensure that reasoning content is not modified for replay)"""  # noqa: E501
+
+    redacted: bool = Field(default=False)
+    """Indicates that the explicit content of this reasoning block has been redacted."""
+
+
+class ContentImage(ContentBase):
+    """Image content."""
+
+    type: Literal['image'] = Field(default='image')
+    """Type."""
+
+    image: str
+    """Either a URL of the image or the base64 encoded image data."""
+
+    detail: Literal['auto', 'low', 'high'] = Field(default='auto')
+    """Specifies the detail level of the image.
+
+    Currently only supported for OpenAI. Learn more in the    [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+    """  # noqa: E501
+
+
+class ContentAudio(ContentBase):
+    """Audio content."""
+
+    type: Literal['audio'] = Field(default='audio')
+    """Type."""
+
+    audio: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['wav', 'mp3']
+    """Format of audio data ('mp3' or 'wav')"""
+
+
+class ContentVideo(ContentBase):
+    """Video content."""
+
+    type: Literal['video'] = Field(default='video')
+    """Type."""
+
+    video: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['mp4', 'mpeg', 'mov']
+    """Format of video data ('mp4', 'mpeg', or 'mov')"""
+
+
+class ContentData(ContentBase):
+    """Model internal."""
+
+    type: Literal['data'] = Field(default='data')
+    """Type."""
+
+    data: Dict[str, JsonValue]
+    """Model provider specific payload - required for internal content."""
+
+
+Content = Union[
+    ContentText,
+    ContentReasoning,
+    ContentImage,
+    ContentAudio,
+    ContentVideo,
+    ContentData,
+]
+"""Content sent to or received from a model."""

evalscope/api/messages/utils.py

@@ -0,0 +1,35 @@
+import re
+from typing import Optional
+
+from .content import ContentReasoning
+
+
+def parse_content_with_reasoning(content: str) -> tuple[str, Optional[ContentReasoning]]:
+    """
+    Looks for and extracts <think/> tags into reasoning text.
+
+    Returns a tuple:
+    - The first element is the input content with the <think> tag and its contents fully removed.
+    - The second element is a ContentReasoning object (or None if no <think> tag is found).
+    """
+    # Match <think> tag with optional attributes anywhere in the string
+    pattern = (r'<think(?:\s+signature="([^"]*)")?(?:\s+redacted="(true)")?\s*>(.*?)</think>')
+    match = re.search(pattern, content, re.DOTALL)
+
+    if match:
+        signature = match.group(1)  # This will be None if not present
+        redacted_value = match.group(2)  # This will be "true" or None
+        reasoning = match.group(3).strip()
+        # Remove the matched <think>...</think> from the input
+        start, end = match.span()
+
+        return (
+            (content[:start] + content[end:]).strip(),
+            ContentReasoning(
+                reasoning=reasoning,
+                signature=signature,
+                redacted=redacted_value == 'true',
+            ),
+        )
+    else:
+        return content, None

evalscope/api/metric/__init__.py

@@ -0,0 +1,2 @@
+from .metric import Metric, T2IMetric
+from .scorer import Aggregator, AggScore, SampleScore, Score, Value