evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/benchmark/benchmark.py

@@ -9,7 +9,7 @@ from evalscope.api.dataset import DatasetDict, Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.filter import FilterEnsemble, build_filter_ensemble
 from evalscope.api.metric import AggScore, SampleScore
-from evalscope.api.mixin import LLMJudgeMixin
+from evalscope.api.mixin import LLMJudgeMixin, SandboxMixin
 from evalscope.api.model import Model
 from evalscope.report import Report
 from evalscope.utils.logger import get_logger
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 logger = get_logger()
 
 
-class DataAdapter(LLMJudgeMixin, ABC):
+class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC):
     """
     Data Adapter for the benchmark.
     """
@@ -40,9 +40,18 @@ class DataAdapter(LLMJudgeMixin, ABC):
         self.shuffle_choices = False
         """Whether to shuffle the choices in the dataset"""
 
+        self.use_batch_scoring = False
+        """Whether to use batch scoring for metrics that support it, need to be enabled in the benchmark as well"""
+
         self.save_metadata = True
         """Whether to save metadata in the review result"""
 
+        self.add_aggregation_name = True
+        """Whether to add aggregation name in the report"""
+
+        self.add_overall_metric = True
+        """Whether to add overall metric in the report"""
+
         self.category_map = {}
         """Category map for the benchmark"""
 
@@ -75,6 +84,12 @@ class DataAdapter(LLMJudgeMixin, ABC):
     def calculate_metrics(self, task_state: TaskState) -> SampleScore:
         pass
 
+    @abstractmethod
+    def batch_calculate_metrics(self, task_states: List[TaskState],
+                                sample_scores: List[SampleScore]) -> List[SampleScore]:
+        """Batch calculate metrics for a list of task states. Need to update sample_scores in place."""
+        pass
+
     @abstractmethod
     def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
         pass
@@ -86,6 +101,11 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         pass
 
+    @abstractmethod
+    def finalize(self, *args, **kwargs) -> None:
+        """Finalize the evaluation process."""
+        pass
+
     @property
     def name(self) -> str:
         """
@@ -170,6 +190,13 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._benchmark_meta.default_subset
 
+    @default_subset.setter
+    def default_subset(self, value: str):
+        """
+        Set the default subset of the benchmark.
+        """
+        self._benchmark_meta.default_subset = value
+
     @property
     def few_shot_num(self) -> int:
         """
@@ -198,6 +225,13 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._benchmark_meta.train_split
 
+    @train_split.setter
+    def train_split(self, value: str):
+        """
+        Set the train split of the benchmark.
+        """
+        self._benchmark_meta.train_split = value
+
     @property
     def eval_split(self) -> Optional[str]:
         """
@@ -205,6 +239,13 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._benchmark_meta.eval_split
 
+    @eval_split.setter
+    def eval_split(self, value: str):
+        """
+        Set the eval split of the benchmark.
+        """
+        self._benchmark_meta.eval_split = value
+
     @property
     def prompt_template(self) -> Optional[str]:
         """
@@ -299,6 +340,48 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         return self._task_config.seed
 
+    @property
+    def shuffle(self) -> bool:
+        """
+        Return whether to shuffle the dataset before evaluation.
+        """
+        return self._benchmark_meta.shuffle
+
+    @shuffle.setter
+    def shuffle(self, value: bool):
+        """
+        Set whether to shuffle the dataset before evaluation.
+        """
+        self._benchmark_meta.shuffle = value
+
+    @property
+    def shuffle_choices(self) -> bool:
+        """
+        Return whether to shuffle the choices in multiple-choice datasets.
+        """
+        return self._benchmark_meta.shuffle_choices
+
+    @shuffle_choices.setter
+    def shuffle_choices(self, value: bool):
+        """
+        Set whether to shuffle the choices in multiple-choice datasets.
+        """
+        self._benchmark_meta.shuffle_choices = value
+
+    @property
+    def review_timeout(self) -> Optional[float]:
+        """
+        Return the timeout for the review process.
+        """
+        return self._benchmark_meta.review_timeout
+
+    @review_timeout.setter
+    def review_timeout(self, value: float):
+        """
+        Set the timeout for the review process.
+        """
+        self._benchmark_meta.review_timeout = value
+
     @contextlib.contextmanager
     def _temporary_attribute(self, attr_name: str, new_value):
         """

evalscope/api/benchmark/meta.py

@@ -73,8 +73,17 @@ class BenchmarkMeta:
     aggregation: str = 'mean'
     """ Aggregation function for the metrics. Default is 'mean'. Can be 'mean', 'pass@<k>' or a custom function name."""
 
+    shuffle: bool = False
+    """Whether to shuffle the dataset before evaluation."""
+
+    shuffle_choices: bool = False
+    """Whether to shuffle the choices in multiple-choice datasets."""
+
+    review_timeout: Optional[float] = None
+    """Timeout for review in seconds."""
+
     extra_params: Dict = field(default_factory=dict)
-    """ Additional parameters for the benchmark."""
+    """Additional parameters for the benchmark."""
 
     def __post_init__(self):
         """Validate fields after initialization."""

evalscope/api/dataset/dataset.py

@@ -5,9 +5,8 @@ from dataclasses import dataclass, field
 from pydantic import BaseModel, Field
 from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Union
 
-from evalscope.api.messages import ChatMessage, messages_pretty_str
+from evalscope.api.messages import ChatMessage, messages_to_markdown
 from evalscope.api.tool import ToolInfo
-from evalscope.utils.multi_choices import answer_character, answer_index
 
 
 class Sample(BaseModel):
@@ -31,9 +30,6 @@ class Sample(BaseModel):
     tools: Optional[List[ToolInfo]] = None
     """List of tools available to the model during inference (optional)."""
 
-    category: Optional[str] = None
-    """Category of the sample (optional)."""
-
     subset_key: Optional[str] = None
     """Key for the subset this sample belongs to, used for generating subsets (optional)."""
 
@@ -54,7 +50,7 @@ class Sample(BaseModel):
         if isinstance(self.input, str):
             input_text = self.input
         else:
-            input_text = messages_pretty_str(self.input)
+            input_text = messages_to_markdown(self.input, max_length=50)
         return f'Sample ID: {self.id}\nInput: {input_text}\nTarget: {self.target}'
 
 
@@ -230,6 +226,8 @@ class MemoryDataset(Dataset):
         self._shuffled = True
 
     def shuffle_choices(self, seed: Optional[int] = None) -> None:
+        from evalscope.utils.multi_choices import answer_character
+
         rand = random.Random(seed)
         for sample in self.samples:
             if not sample.choices:
@@ -249,6 +247,8 @@ class MemoryDataset(Dataset):
             sample.target = self._remap_target(sample.target, position_map=position_map)
 
     def _remap_target(self, target: Union[str, List[str]], position_map: Dict[int, str]) -> Union[str, List[str]]:
+        from evalscope.utils.multi_choices import answer_index
+
         if isinstance(target, list):
             return [position_map[answer_index(t)] for t in target]
         else:
@@ -347,3 +347,24 @@ class DatasetDict:
             cur_dataset.reindex(group_size=repeats)
             dataset_dict[key] = cur_dataset
         return cls(dataset_dict)
+
+    @classmethod
+    def from_dataset_dicts(cls, dataset_dicts: List['DatasetDict']) -> 'DatasetDict':
+        """
+        Create a DatasetDict by merging multiple DatasetDicts.
+
+        Args:
+            dataset_dicts (List[DatasetDict]): List of DatasetDicts to merge.
+
+        Returns:
+            DatasetDict: A new DatasetDict containing the merged datasets.
+        """
+        merged_dict = defaultdict(list)
+        for dataset_dict in dataset_dicts:
+            for key, dataset in dataset_dict.items():
+                merged_dict[key].extend(dataset.samples)
+        # Create a MemoryDataset for each subset key
+        final_dict = {}
+        for key, samples in merged_dict.items():
+            final_dict[key] = MemoryDataset(samples, name=key)
+        return cls(final_dict)

evalscope/api/dataset/loader.py

@@ -8,7 +8,7 @@ from typing import Callable, Dict, List, Optional, Union
 from evalscope.api.dataset.utils import record_to_sample_fn
 from evalscope.constants import DEFAULT_EVALSCOPE_CACHE_DIR, HubType
 from evalscope.utils import get_logger
-from evalscope.utils.io_utils import csv_to_list, gen_hash, jsonl_to_list, safe_filename
+from evalscope.utils.io_utils import csv_to_list, gen_hash, jsonl_to_list, safe_filename, tsv_to_list
 from .dataset import Dataset, FieldSpec, MemoryDataset, Sample
 from .utils import data_to_samples, shuffle_choices_if_requested
 
@@ -126,7 +126,8 @@ class RemoteDataLoader(DataLoader):
                 self.limit = int(len(dataset) * self.limit)
             elif isinstance(self.limit, int) and self.limit < 0:
                 raise ValueError('Limit must be a non-negative integer or a float between 0 and 1.')
-            dataset = dataset.select(range(self.limit))
+            if len(dataset) > self.limit:
+                dataset = dataset.select(range(self.limit))
 
         # convert to list
         dataset = dataset.to_list()
@@ -167,7 +168,11 @@ class LocalDataLoader(DataLoader):
         dataset = []
 
         # Check for JSONL or CSV files in the specified path
-        for ext, loader in [('.jsonl', jsonl_to_list), ('.csv', csv_to_list)]:
+        for ext, loader in [
+            ('.jsonl', jsonl_to_list),
+            ('.csv', csv_to_list),
+            ('.tsv', tsv_to_list),
+        ]:
             # Check if the file exists with the given extension
             if os.path.isfile(path) and path.endswith(ext):
                 file_paths = [path]

evalscope/api/evaluator/cache.py

@@ -68,6 +68,8 @@ class CacheManager:
             # Convert to task state for further processing
             cached_state = cached_model_result.to_task_state(dataset=dataset)
 
+            if cached_state is None:
+                continue
             cached_task_states.append(cached_state)
             cached_sample_ids.add(cached_state.sample_id)
 
@@ -283,9 +285,11 @@ class ModelResult(BaseModel):
         Raises:
             ValueError: If the sample index is not found in the dataset
         """
-        sample = dataset[self.index]
-        if not sample:
-            raise ValueError(f'Sample with index {self.index} not found in dataset')
+        try:
+            sample = dataset[self.index]
+        except IndexError:
+            logger.warning(f'Sample index {self.index} not found in dataset during cache restoration.')
+            return None
 
         # update metadata if exists
         if self.metadata:
@@ -299,6 +303,15 @@ class ModelResult(BaseModel):
             completed=True,  # Mark as completed since it was cached
         )
 
+    def pretty_print(self) -> str:
+        """
+        Generate a pretty-printed string representation of the model result.
+
+        Returns:
+            A string representation of the model result
+        """
+        return self.model_dump_json(indent=2)
+
 
 class ReviewResult(BaseModel):
     """
@@ -340,7 +353,7 @@ class ReviewResult(BaseModel):
 
         return cls(
             index=state.sample_id,
-            input=state.input_text,
+            input=state.input_markdown,
             target=state.target,
             sample_score=sample_score,
         )
@@ -353,3 +366,17 @@ class ReviewResult(BaseModel):
             The sample score object
         """
         return self.sample_score
+
+    def pretty_print(self) -> str:
+        """
+        Generate a pretty-printed string representation of the review result.
+
+        Returns:
+            A string representation of the review result
+        """
+        output = [
+            f'Review Result for Sample {self.index}:',
+            f'Target: {self.target}',
+            f'Score: {self.sample_score.model_dump_json(indent=2)}',
+        ]
+        return '\n'.join(output)

evalscope/api/evaluator/evaluator.py

@@ -54,3 +54,8 @@ class Evaluator(abc.ABC):
     def get_report(self, *args, **kwargs) -> Report:
         """Get the evaluation report."""
         pass
+
+    @abc.abstractmethod
+    def finalize(self, *args, **kwargs) -> None:
+        """Finalize the evaluation process."""
+        pass

evalscope/api/evaluator/state.py

@@ -3,7 +3,7 @@ from random import Random
 from typing import Any, Dict, List, Optional, Sequence, Union, overload
 
 from evalscope.api.dataset import Sample
-from evalscope.api.messages import ChatMessage, ChatMessageUser, messages_pretty_str
+from evalscope.api.messages import ChatMessage, ChatMessageUser, messages_pretty_str, messages_to_markdown
 from evalscope.api.model import ModelOutput
 
 
@@ -188,6 +188,17 @@ class TaskState:
         else:
             return messages_pretty_str(self._input)
 
+    @property
+    def input_markdown(self) -> str:
+        """Get the input text as markdown.
+
+        For multi-modal content, images will be represented in markdown format.
+        """
+        if isinstance(self._input, str):
+            return self._input
+        else:
+            return messages_to_markdown(self._input)
+
     @property
     def choices(self) -> Choices:
         """Choices for the sample, if applicable."""
@@ -262,3 +273,8 @@ class TaskState:
     def target(self) -> str:
         """The scoring target for this `Sample`."""
         return self._target.text
+
+    @target.setter
+    def target(self, text: str) -> None:
+        """Set the target for review purposes."""
+        self._target = Target(text)

evalscope/api/messages/__init__.py

@@ -6,6 +6,7 @@ from .chat_message import (
     ChatMessageUser,
     dict_to_chat_message,
     messages_pretty_str,
+    messages_to_markdown,
 )
 from .content import Content, ContentAudio, ContentData, ContentImage, ContentReasoning, ContentText, ContentVideo
 from .utils import parse_content_with_reasoning

evalscope/api/messages/chat_message.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, JsonValue, model_validator
 from typing import Any, Dict, List, Literal, Optional, Type, Union
 
 from evalscope.api.tool import ToolCall, ToolCallError
-from .content import Content, ContentReasoning, ContentText
+from .content import Content, ContentAudio, ContentImage, ContentReasoning, ContentText
 from .utils import parse_content_with_reasoning
 
 
@@ -184,7 +184,7 @@ def dict_to_chat_message(data: Dict[str, Any]) -> ChatMessage:
 
 
 def messages_pretty_str(messages: List[ChatMessage]) -> str:
-    """Pretty print a list of chat messages."""
+    """Pretty print a list of chat messages. Without images or other multi-modal contents."""
     output = []
     for message in messages:
         role = message.role.capitalize()
@@ -196,3 +196,53 @@ def messages_pretty_str(messages: List[ChatMessage]) -> str:
                 content += f'\nFunction: {message.function}'
         output.append(f'**{role}**: {content}')
     return '\n\n'.join(output)
+
+
+def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int] = None) -> str:
+    """Convert a list of chat messages to markdown format.
+
+    Args:
+        messages (List[ChatMessage]): The list of chat messages to convert.
+        max_length (Optional[int]): If provided, truncates the base64 string of images to this length.
+    """
+    output = []
+    for message in messages:
+        role = message.role.capitalize()
+
+        # Start with role header
+        content_parts = [f'**{role}**: ']
+
+        # Handle content based on type
+        if isinstance(message.content, str):
+            content_parts.append(message.content)
+        else:
+            for content_item in message.content:
+                if isinstance(content_item, ContentText):
+                    content_parts.append(content_item.text)
+                elif isinstance(content_item, ContentImage):
+                    # Use markdown image syntax
+                    image_base64 = content_item.image
+                    if max_length and len(image_base64) > max_length:
+                        image_base64 = image_base64[:max_length]
+                    content_parts.append(f'![image]({image_base64})')
+                elif isinstance(content_item, ContentAudio):
+                    audio_base64 = content_item.audio
+                    if max_length and len(audio_base64) > max_length:
+                        audio_base64 = audio_base64[:max_length]
+                    content_parts.append(f"<audio controls src='{audio_base64}'></audio>")
+                elif isinstance(content_item, ContentReasoning):
+                    content_parts.append(f'**Reasoning:** {content_item.reasoning}')
+
+        # Add tool-specific information
+        if isinstance(message, ChatMessageTool):
+            if message.error:
+                content_parts.append(f'**Error:** {message.error.message}')
+            if message.function:
+                content_parts.append(f'**Function:** {message.function}')
+        elif isinstance(message, ChatMessageAssistant) and message.tool_calls:
+            for tool_call in message.tool_calls:
+                content_parts.append(f'**Tool Call:** {tool_call.function}')
+
+        output.append('\n'.join(content_parts))
+
+    return '\n\n'.join(output)

evalscope/api/metric/__init__.py

@@ -1,2 +1,2 @@
-from .metric import Metric, T2IMetric
+from .metric import Metric, SingletonMetric, T2IMetric
 from .scorer import Aggregator, AggScore, SampleScore, Score, Value

evalscope/api/metric/metric.py

@@ -28,7 +28,8 @@ class Metric(ABC):
         return self.apply([prediction], [reference])[0]
 
 
-class T2IMetric(Metric):
+class SingletonMetric(Metric):
+    """Singleton base class for metrics."""
     _instance = None
 
     @thread_safe
@@ -48,6 +49,10 @@ class T2IMetric(Metric):
     def _init_once(self, *args, **kwargs):
         pass
 
+
+class T2IMetric(SingletonMetric):
+    """Singleton base class for T2I metrics."""
+
     def apply(self, images: List[str], texts: List[str], **kwargs) -> List[Union[float, dict]]:
         pass
 

evalscope/api/metric/scorer.py

@@ -35,20 +35,28 @@ class Score(BaseModel):
         """Main score value."""
         if self.main_score_name and self.main_score_name in self.value:
             return self.value[self.main_score_name]
-        return next(iter(self.value.values()), None)
+        elif self.value:
+            # If main_score_name is not set or not found, use the first value and update main_score_name
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            return self.value[first_key]
+        return None
 
     @main_value.setter
     def main_value(self, value: Union[int, float, bool]):
         """Set the main score value."""
         if self.main_score_name:
+            # If main_score_name is already set, use it
             self.value[self.main_score_name] = value
+        elif self.value:
+            # If no main_score_name but value dict exists, use the first key
+            first_key = next(iter(self.value))
+            self.main_score_name = first_key
+            self.value[first_key] = value
         else:
-            # If no main score name is set, just update the first value
-            if self.value:
-                first_key = next(iter(self.value))
-                self.value[first_key] = value
-            else:
-                self.value['default'] = value
+            # If neither main_score_name nor value dict exists, initialize both
+            self.main_score_name = 'default'
+            self.value[self.main_score_name] = value
 
 
 class SampleScore(BaseModel):

evalscope/api/mixin/__init__.py

@@ -1,2 +1,2 @@
-from .dataset_mixin import DatasetLoaderMixin
 from .llm_judge_mixin import LLMJudgeMixin
+from .sandbox_mixin import SandboxMixin

evalscope/api/mixin/llm_judge_mixin.py

@@ -24,6 +24,8 @@ class LLMJudgeMixin:
 
         self._llm_judge: Optional[LLMJudge] = None
 
+        super().__init__(task_config=task_config)
+
     @property
     def llm_judge(self) -> Optional[LLMJudge]:
         """Get LLM judge instance with lazy initialization."""