evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/messages/content.py

@@ -0,0 +1,102 @@
+from pydantic import BaseModel, Field, JsonValue
+from typing import Dict, Literal, Optional, Sequence, Union
+
+
+class ContentBase(BaseModel):
+    internal: Optional[JsonValue] = Field(default=None)
+    """Model provider specific payload - typically used to aid transformation back to model types."""
+
+
+class ContentText(ContentBase):
+    """Text content."""
+
+    type: Literal['text'] = Field(default='text')
+    """Type."""
+
+    text: str
+    """Text content."""
+
+    refusal: Optional[bool] = Field(default=None)
+    """Was this a refusal message?"""
+
+
+class ContentReasoning(ContentBase):
+    """Reasoning content.
+
+    See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
+    """  # noqa: E501
+
+    type: Literal['reasoning'] = Field(default='reasoning')
+    """Type."""
+
+    reasoning: str
+    """Reasoning content."""
+
+    signature: Optional[str] = Field(default=None)
+    """Signature for reasoning content (used by some models to ensure that reasoning content is not modified for replay)"""  # noqa: E501
+
+    redacted: bool = Field(default=False)
+    """Indicates that the explicit content of this reasoning block has been redacted."""
+
+
+class ContentImage(ContentBase):
+    """Image content."""
+
+    type: Literal['image'] = Field(default='image')
+    """Type."""
+
+    image: str
+    """Either a URL of the image or the base64 encoded image data."""
+
+    detail: Literal['auto', 'low', 'high'] = Field(default='auto')
+    """Specifies the detail level of the image.
+
+    Currently only supported for OpenAI. Learn more in the    [Vision guide](https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding).
+    """  # noqa: E501
+
+
+class ContentAudio(ContentBase):
+    """Audio content."""
+
+    type: Literal['audio'] = Field(default='audio')
+    """Type."""
+
+    audio: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['wav', 'mp3']
+    """Format of audio data ('mp3' or 'wav')"""
+
+
+class ContentVideo(ContentBase):
+    """Video content."""
+
+    type: Literal['video'] = Field(default='video')
+    """Type."""
+
+    video: str
+    """Audio file path or base64 encoded data URL."""
+
+    format: Literal['mp4', 'mpeg', 'mov']
+    """Format of video data ('mp4', 'mpeg', or 'mov')"""
+
+
+class ContentData(ContentBase):
+    """Model internal."""
+
+    type: Literal['data'] = Field(default='data')
+    """Type."""
+
+    data: Dict[str, JsonValue]
+    """Model provider specific payload - required for internal content."""
+
+
+Content = Union[
+    ContentText,
+    ContentReasoning,
+    ContentImage,
+    ContentAudio,
+    ContentVideo,
+    ContentData,
+]
+"""Content sent to or received from a model."""

evalscope/api/messages/utils.py

@@ -0,0 +1,35 @@
+import re
+from typing import Optional
+
+from .content import ContentReasoning
+
+
+def parse_content_with_reasoning(content: str) -> tuple[str, Optional[ContentReasoning]]:
+    """
+    Looks for and extracts <think/> tags into reasoning text.
+
+    Returns a tuple:
+    - The first element is the input content with the <think> tag and its contents fully removed.
+    - The second element is a ContentReasoning object (or None if no <think> tag is found).
+    """
+    # Match <think> tag with optional attributes anywhere in the string
+    pattern = (r'<think(?:\s+signature="([^"]*)")?(?:\s+redacted="(true)")?\s*>(.*?)</think>')
+    match = re.search(pattern, content, re.DOTALL)
+
+    if match:
+        signature = match.group(1)  # This will be None if not present
+        redacted_value = match.group(2)  # This will be "true" or None
+        reasoning = match.group(3).strip()
+        # Remove the matched <think>...</think> from the input
+        start, end = match.span()
+
+        return (
+            (content[:start] + content[end:]).strip(),
+            ContentReasoning(
+                reasoning=reasoning,
+                signature=signature,
+                redacted=redacted_value == 'true',
+            ),
+        )
+    else:
+        return content, None

evalscope/api/metric/__init__.py

@@ -0,0 +1,2 @@
+from .metric import Metric, SingletonMetric, T2IMetric
+from .scorer import Aggregator, AggScore, SampleScore, Score, Value

evalscope/api/metric/metric.py

@@ -0,0 +1,60 @@
+from abc import ABC, abstractmethod
+from typing import Callable, Iterable, List, Union
+
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+
+logger = get_logger()
+
+
+class Metric(ABC):
+    """
+    Metric classes operate on a sample level.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        """
+        Can define custom behavior here, if an individual instantiation of a Metric class should have state.
+        """
+
+    @abstractmethod
+    def apply(self, predictions: List[str], references: List[str]) -> List[float]:
+        pass
+
+    def __call__(self, prediction: str, reference: str) -> float:
+        """
+        Allows the metric to be called like a function.
+        """
+        return self.apply([prediction], [reference])[0]
+
+
+class SingletonMetric(Metric):
+    """Singleton base class for metrics."""
+    _instance = None
+
+    @thread_safe
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+        return cls._instance
+
+    def __init__(self, *args, **kwargs):
+        cls = self.__class__
+        if hasattr(self, '_init_done'):
+            return
+        logger.info(f'Initializing {cls.__name__}...')
+        self._init_once(*args, **kwargs)
+        self._init_done = True
+
+    def _init_once(self, *args, **kwargs):
+        pass
+
+
+class T2IMetric(SingletonMetric):
+    """Singleton base class for T2I metrics."""
+
+    def apply(self, images: List[str], texts: List[str], **kwargs) -> List[Union[float, dict]]:
+        pass
+
+    def __call__(self, image: str, text: str, **kwargs) -> Union[float, dict]:
+        return self.apply([image], [text], **kwargs)[0]

evalscope/api/metric/scorer.py

@@ -0,0 +1,113 @@
+from pydantic import BaseModel, Field
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+Value = Dict[str, Union[int, float, bool]]
+
+
+class Score(BaseModel):
+    """Score generated by a scorer."""
+
+    value: Value = Field(default_factory=dict)
+    """Score value as a dictionary. Key is the score name, value is the score value.
+    The first key is considered the main score by default."""
+
+    extracted_prediction: Optional[str] = Field(default=None)
+    """Answer extracted from model output (optional)"""
+
+    prediction: Optional[str] = Field(default=None)
+    """Original prediction text from the model (optional)"""
+
+    explanation: Optional[str] = Field(default=None)
+    """Explanation of score (optional)."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default_factory=dict)
+    """Additional metadata related to the score"""
+
+    main_score_name: Optional[str] = Field(default=None)
+    """Main score name, if applicable. This is used to indicate which score is the primary score in a multi-score scenario."""  # noqa: E501
+
+    @property
+    def main_value(self) -> Union[int, float, bool]:
+        """Main score value."""
+        if self.main_score_name and self.main_score_name in self.value:
+            return self.value[self.main_score_name]
+        elif self.value:
+            # If main_score_name is not set or not found, use the first value and update main_score_name
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            return self.value[first_key]
+        return None
+
+    @main_value.setter
+    def main_value(self, value: Union[int, float, bool]):
+        """Set the main score value."""
+        if self.main_score_name:
+            # If main_score_name is already set, use it
+            self.value[self.main_score_name] = value
+        elif self.value:
+            # If no main_score_name but value dict exists, use the first key
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            self.value[first_key] = value
+        else:
+            # If neither main_score_name nor value dict exists, initialize both
+            self.main_score_name = 'default'
+            self.value[self.main_score_name] = value
+
+
+class SampleScore(BaseModel):
+    """Score for a Sample."""
+
+    score: Score
+    """A score"""
+
+    sample_id: Optional[Union[str, int]] = Field(default=None)
+    """A sample id"""
+
+    group_id: Optional[Union[str, int]] = Field(default=None)
+    """A group id for the sample, used for grouping k repeated samples."""
+
+    sample_metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Metadata from the sample"""
+
+
+class AggScore(BaseModel):
+    """Output of an aggregation operation."""
+
+    score: float = Field(default=0.0)
+    """Aggregated value as a float."""
+
+    metric_name: str = Field(default='')
+    """Name of the metric being aggregated."""
+
+    aggregation_name: str = Field(default='')
+    """Name of the aggregation methods"""
+
+    num: int = Field(default=0)
+    """Number of samples used in the aggregation."""
+
+    ids: Optional[List[Union[str, int]]] = Field(default=None)
+    """List of sample IDs used in the aggregation, if applicable."""
+
+    metadata: Optional[Dict[str, Any]] = Field(default=None)
+    """Additional metadata related to the aggregation."""
+
+
+class Aggregator:
+
+    name = 'default'
+
+    def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
+        r"""Aggregate a metric on a list of scores.
+
+        Args:
+          scores: List of scores.
+
+        Returns:
+          List[AggregatOutput]: List of aggregated outputs.
+        """
+        ...

evalscope/api/mixin/__init__.py

@@ -0,0 +1,2 @@
+from .llm_judge_mixin import LLMJudgeMixin
+from .sandbox_mixin import SandboxMixin

evalscope/api/mixin/llm_judge_mixin.py

@@ -0,0 +1,170 @@
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.constants import JudgeStrategy
+from evalscope.metrics import LLMJudge
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LLMJudgeMixin:
+    """
+    Mixin class for LLM Judge functionality.
+    """
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+        self._use_llm_judge = False
+        """Whether to use LLM as a judge"""
+
+        self._llm_judge: Optional[LLMJudge] = None
+
+        super().__init__(task_config=task_config)
+
+    @property
+    def llm_judge(self) -> Optional[LLMJudge]:
+        """Get LLM judge instance with lazy initialization."""
+        if self._llm_judge is None and self.use_llm_judge:
+            self._llm_judge = self.init_llm_judge()
+        return self._llm_judge
+
+    @llm_judge.setter
+    def llm_judge(self, value: Optional[LLMJudge]):
+        """Set LLM judge instance."""
+        self._llm_judge = value
+
+    @property
+    def judge_strategy(self) -> str:
+        """Get the judge strategy from the task configuration."""
+        return self._task_config.judge_strategy
+
+    @property
+    def use_llm_judge(self) -> bool:
+        """Check if LLM judge is enabled."""
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return False
+        elif self.judge_strategy == JudgeStrategy.LLM:
+            return True
+        elif self.judge_strategy == JudgeStrategy.LLM_RECALL:
+            return True
+        elif self.judge_strategy == JudgeStrategy.AUTO:
+            return self._use_llm_judge
+        else:
+            logger.warning(f'Unknown judge strategy: {self.judge_strategy}. Defaulting to False.')
+            return False
+
+    def init_llm_judge(self) -> Optional[LLMJudge]:
+        """
+        Initialize the LLM judge for the benchmark.
+
+        Returns:
+            Optional[LLMJudge]: The initialized LLM judge instance or None
+        """
+
+        if self.judge_strategy == JudgeStrategy.RULE:
+            return None
+        else:
+            return LLMJudge(**self._task_config.judge_model_args)
+
+    def maybe_llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+        rule_based_score: Optional[Score] = None,
+    ) -> Score:
+        """
+        Compute the match score between the original and filtered predictions against the reference.
+
+        Args:
+            original_prediction: The original prediction output from the model.
+            filtered_prediction: The filtered prediction output from the model.
+            reference: The ground truth reference output.
+            task_state: The current task state.
+            original_score: Optional original score to be used for comparison.
+
+        Returns:
+            Score: The computed match score.
+        """
+        # If LLM judge is not used, return the rule-based score directly
+        if not self.use_llm_judge:
+            return rule_based_score
+
+        # For LLM_RECALL, if rule-based score is already perfect, skip LLM judge
+        if float(rule_based_score.main_value) > 0.99:
+            return rule_based_score
+
+        # Compute LLM judge score
+        llm_score = self.llm_match_score(
+            original_prediction=original_prediction,
+            filtered_prediction=filtered_prediction,
+            reference=reference,
+            task_state=task_state,
+        )
+
+        # For LLM RECALL, merge the scores
+        return self._merge_scores(rule_based_score, llm_score)
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """Compute the LLM match score.
+
+        Args:
+            original_prediction (str): The original prediction output from the model.
+            filtered_prediction (str): The filtered prediction output from the model.
+            reference (str): The ground truth reference output.
+            task_state (TaskState): The current task state.
+
+        Returns:
+            Score: The computed match score.
+        """
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Request judge and obtain score
+        prompt = self.llm_judge.build_prompt(pred=original_prediction, gold=reference, question=question)
+        judge_response = self.llm_judge.judge(prompt)
+        judge_score = self.llm_judge.get_score(judge_response)
+
+        score.value = {'acc': judge_score}
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+
+        return score
+
+    def _merge_scores(self, rule_based_score: Score, llm_score: Score) -> Score:
+        """
+        Merge rule-based score with LLM judge score for LLM_RECALL strategy.
+
+        Args:
+            rule_based_score: The original rule-based score
+            llm_score: The LLM judge score
+
+        Returns:
+            Score: The merged score
+        """
+        # Update the main value with LLM judge result
+        rule_based_score.main_value = llm_score.main_value
+        rule_based_score.explanation = llm_score.explanation
+        rule_based_score.metadata = llm_score.metadata
+
+        return rule_based_score

evalscope/api/mixin/sandbox_mixin.py

@@ -0,0 +1,182 @@
+from typing import TYPE_CHECKING, Any, Dict, Optional
+
+from evalscope.utils.function_utils import AsyncioLoopRunner, thread_safe
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from ms_enclave.sandbox.manager import SandboxManager
+
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class SandboxMixin:
+    """Sandbox mixin for sandboxed code execution."""
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+
+        self._manager: Optional['SandboxManager'] = None
+        """Sandbox manager instance."""
+
+        self._sandbox_id: Optional[str] = None
+        """Sandbox ID."""
+
+        # Lazy init state
+        self._initialized: bool = False
+
+        # NOTE: Initialization is deferred.
+        super().__init__()
+
+    async def _async_init(self):
+        """Async initialization helper."""
+        await self.init_sandbox_manager_async()
+        await self.init_sandbox_async()
+
+    @property
+    def use_sandbox(self) -> bool:
+        """
+        Return whether to use sandbox for the benchmark.
+        """
+        if not self._task_config:
+            return False
+        else:
+            return self._task_config.use_sandbox
+
+    @property
+    def sandbox_manager(self) -> Optional['SandboxManager']:
+        """Get the sandbox manager instance."""
+        return self._manager
+
+    @property
+    def sandbox_id(self) -> Optional[str]:
+        """Get the sandbox ID."""
+        return self._sandbox_id
+
+    @thread_safe
+    def ensure_sandbox_ready(self) -> bool:
+        """
+        Ensure the sandbox loop, manager, and sandbox instance are initialized.
+        This method is thread-safe and idempotent.
+        """
+        if not self.use_sandbox:
+            return False
+
+        if self._initialized and self._manager and self._sandbox_id:
+            return True
+
+        # Initialize manager and sandbox using the class-level runner
+        AsyncioLoopRunner.run(self.init_sandbox_manager_async())
+        AsyncioLoopRunner.run(self.init_sandbox_async())
+
+        self._initialized = True
+        return True
+
+    async def init_sandbox_manager_async(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager asynchronously."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.manager import HttpSandboxManager, LocalSandboxManager
+
+        manager_config = self._task_config.sandbox_manager_config or {}
+        if manager_config.get('base_url'):
+            # Remote manager
+            self._manager = HttpSandboxManager(**manager_config)
+        else:
+            # Local manager
+            self._manager = LocalSandboxManager(**manager_config)
+
+        await self._manager.start()
+        logger.info('Sandbox manager initialized.')
+        return self._manager
+
+    def init_sandbox_manager(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        return AsyncioLoopRunner.run(self.init_sandbox_manager_async())
+
+    async def init_sandbox_async(self) -> Optional[str]:
+        """Initialize the sandbox instance asynchronously."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.model import DockerSandboxConfig, SandboxType
+
+        sandbox_config = self._task_config.sandbox_config or DockerSandboxConfig(
+            image='python:3.11-slim', tools_config={
+                'shell_executor': {},
+                'python_executor': {}
+            }
+        )
+        sandbox_type = self._task_config.sandbox_type or SandboxType.DOCKER
+
+        self._sandbox_id = await self._manager.create_sandbox(sandbox_type=sandbox_type, config=sandbox_config)
+
+        sandbox_info = await self._manager.get_sandbox_info(self._sandbox_id)
+
+        logger.info(f'Sandbox of type {sandbox_type} initialized. Info: {sandbox_info.model_dump(exclude_none=True)}')
+        return self._sandbox_id
+
+    def init_sandbox(self) -> Optional[str]:
+        """Initialize the sandbox instance."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        return AsyncioLoopRunner.run(self.init_sandbox_async())
+
+    def execute_code_in_sandbox(self, code: str, timeout: int = 60, language: str = 'python') -> Dict[str, Any]:
+        """Execute code in the sandbox."""
+        # Lazy, thread-safe initialization
+        if not self.ensure_sandbox_ready():
+            logger.warning('Sandbox is not initialized.')
+            return {'error': 'Sandbox is not initialized.'}
+
+        from ms_enclave.sandbox.model import ExecutionStatus, ToolResult
+
+        async def _execute_async():
+            if language.lower() == 'python':
+                tool_name = 'python_executor'
+                parameters = {'code': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            elif language.lower() == 'shell':
+                tool_name = 'shell_executor'
+                parameters = {'command': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            else:
+                logger.warning(f"Unsupported language: {language}. Supported languages are 'python' and 'shell'.")
+                result = ToolResult(
+                    status=ExecutionStatus.ERROR,
+                    tool_name='code_executor',
+                    output=f"Unsupported language: {language}. Supported languages are 'python' and 'shell'."
+                )
+            return result
+
+        # Execute in background loop via class-level runner
+        result = AsyncioLoopRunner.run(_execute_async(), timeout=timeout + 10)
+        return result.model_dump(exclude_none=True)
+
+    def sandbox_finalize(self, *args, **kwargs):
+        """Finalize the sandbox manager."""
+        if self._manager:
+            try:
+                # Stop the manager but keep the shared loop alive
+                AsyncioLoopRunner.run(self._manager.stop(), timeout=30)
+                logger.info('Sandbox manager finalized.')
+            except Exception as e:
+                logger.warning(f'Error finalizing sandbox manager: {e}')

evalscope/api/model/__init__.py

@@ -0,0 +1,12 @@
+from .generate_config import GenerateConfig
+from .model import Model, ModelAPI, get_model, get_model_with_task_config
+from .model_output import (
+    ChatCompletionChoice,
+    Logprob,
+    Logprobs,
+    ModelOutput,
+    ModelUsage,
+    StopReason,
+    TopLogprob,
+    as_stop_reason,
+)