evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/api/benchmark/adapters/image_edit_adapter.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+
+from evalscope.constants import EvalType, FileConstants
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.io_utils import jsonl_to_list
+from .text2image_adapter import Text2ImageAdapter
+
+logger = get_logger()
+
+
+class ImageEditAdapter(Text2ImageAdapter):
+    """
+    Support two methods:
+    1. Inference using modelscope pipeline
+    2. Load local inference jsonl file with key to corresponding prompt
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.local_file = self.extra_params.get('local_file', None)
+        self.id_key = self.extra_params.get('id_key', FileConstants.ID)
+        self.image_key = self.extra_params.get('image_key', FileConstants.IMAGE_PATH)
+        self.local_data = self.load_local_file()
+
+    def load_local_file(self) -> Optional[dict]:
+        if not self.local_file:
+            return None
+
+        # Load file and check
+        data_list = jsonl_to_list(self.local_file)
+        data_dict = {}
+        for record in data_list:
+            if self.image_key not in record:
+                raise ValueError(f"Image key '{self.image_key}' not found in record: {record}, file {self.local_file}")
+            if self.id_key not in record:
+                raise ValueError(f"ID key '{self.id_key}' not found in record: {record}, file {self.local_file}")
+
+            image_path = record[self.image_key]
+            if not os.path.isabs(image_path):
+                image_path = os.path.join(os.path.dirname(self.local_file), image_path)
+            if not os.path.exists(image_path):
+                raise FileNotFoundError(f"Image file '{image_path}' not found.")
+
+            data_dict[record[self.id_key]] = record
+        return data_dict
+
+    def get_image_path_from_id(self, image_id) -> Optional[str]:
+        if not self.local_file:
+            return None
+
+        record = self.local_data.get(image_id)
+        if not record:
+            return None
+
+        return record[self.image_key]
+
+    def _post_process_samples(self):
+        super()._post_process_samples()
+
+        # Add local image path if exists
+        for subset in self.test_dataset.keys():
+            for sample in self.test_dataset[subset]:
+                local_image_path = self.get_image_path_from_id(sample.metadata.get(FileConstants.ID))
+                if local_image_path:
+                    sample.metadata[FileConstants.IMAGE_PATH] = local_image_path
+
+    def sample_filter(self, sample) -> bool:
+        """
+        Filter samples based on metadata availability.
+        If local file is not available, all samples are considered valid.
+        Otherwise, only samples with valid metadata and image path are kept.
+        """
+        if not self.local_data:
+            return True
+        else:
+            sample_id = sample.metadata.get(FileConstants.ID)
+            if (not sample_id) or (not self.get_image_path_from_id(sample_id)):
+                return False
+            return True

evalscope/api/benchmark/adapters/multi_choice_adapter.py

@@ -0,0 +1,83 @@
+from evalscope.api.dataset.dataset import Sample
+from evalscope.api.evaluator import Choices, Target, TaskState
+from evalscope.utils.multi_choices import (
+    FEW_SHOT_TEMPLATE,
+    MultipleChoiceTemplate,
+    format_example,
+    parse_answers,
+    parse_answers_zh,
+    prompt,
+    valid_template,
+)
+from .default_data_adapter import DefaultDataAdapter
+
+
+class MultiChoiceAdapter(DefaultDataAdapter):
+    """
+    Adapter for multi-choice benchmarks.
+    This adapter formats the input for multi-choice questions and handles few-shot examples.
+    """
+
+    multiple_correct: bool = False
+    """Whether the benchmark allows multiple correct answers."""
+
+    def format_prompt_template(self, sample: Sample) -> str:
+        """
+        Format the basic prompt template with the sample data.
+
+        Args:
+            sample (Sample): The sample object containing the prompt data
+
+        Returns:
+            str: The formatted prompt ready for model input
+        """
+        assert valid_template(self.prompt_template), 'Prompt template is not valid'
+
+        return prompt(
+            question=sample.input,
+            choices=Choices(sample.choices),
+            template=self.prompt_template,
+        )
+
+    def format_fewshot_template(self, fewshot: str, sample: Sample) -> str:
+        """
+        Format the few-shot template with demonstrations and the main prompt.
+
+        Args:
+            fewshot (str): The formatted few-shot demonstration examples
+            sample (Sample): The sample object containing the prompt data
+
+        Returns:
+            str: The complete formatted input with few-shot context
+        """
+
+        few_shot_prompt_template = self.few_shot_prompt_template or (FEW_SHOT_TEMPLATE + self.prompt_template)
+
+        assert valid_template(few_shot_prompt_template), 'Few-shot prompt template is not valid'
+
+        return prompt(
+            question=sample.input, choices=Choices(sample.choices), template=few_shot_prompt_template, fewshot=fewshot
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        """
+        Convert a sample to a few-shot formatted string.
+
+        Args:
+            sample (Sample): The sample object to format
+
+        Returns:
+            str: The formatted few-shot example string
+        """
+        return format_example(question=sample.input, choices=Choices(sample.choices), answer=Target(sample.target))
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        if self.prompt_template in [
+            MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT,
+            MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE
+        ]:
+            # For Chinese COT template, we use a different extraction method
+            answers = parse_answers_zh(task_state, multiple_correct=self.multiple_correct)
+        else:
+            answers = parse_answers(task_state, multiple_correct=self.multiple_correct)
+        return ''.join(sorted(list(answers)))

evalscope/api/benchmark/adapters/text2image_adapter.py

@@ -0,0 +1,156 @@
+import base64
+import os
+
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.messages.content import ContentImage
+from evalscope.api.metric import Score
+from evalscope.api.model import ChatCompletionChoice, Model, ModelOutput
+from evalscope.api.registry import get_metric
+from evalscope.constants import EvalType, FileConstants
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+from .default_data_adapter import DefaultDataAdapter
+
+logger = get_logger()
+
+
+class Text2ImageAdapter(DefaultDataAdapter):
+    """Text to Image Adapter for benchmarks."""
+
+    def load_from_disk(self, **kwargs):
+        return super().load_from_disk(use_local_loader=True)
+
+    def record_to_sample(self, record) -> Sample:
+        """Convert a record dictionary to a Sample object."""
+        return Sample(
+            input=[ChatMessageUser(content=record['prompt'])],
+            metadata={
+                'prompt': record['prompt'],
+                'category': record.get('category', ''),
+                'tags': record.get('tags', []),
+                FileConstants.ID: record[FileConstants.ID],
+                FileConstants.IMAGE_PATH: record.get(FileConstants.IMAGE_PATH,
+                                                     ''),  # Optional field for existing image path
+            }
+        )
+
+    def _on_inference(self, model: Model, sample: Sample) -> ModelOutput:
+        """
+        Hook method called during the actual inference process.
+
+        This method executes the model inference and can be overridden
+        to implement custom inference logic or model interaction patterns.
+
+        Args:
+            model (Model): The model to use for inference
+            sample (Sample): The sample to process
+
+        Returns:
+            ModelOutput: The raw output from the model
+        """
+        if self.eval_type == EvalType.MOCK_LLM:
+            return ModelOutput(
+                model=model.name,
+                choices=[ChatCompletionChoice.from_content('')],
+            )
+        else:
+            # Execute model inference with the processed input and any tools
+            model_output = model.generate(input=sample.input, tools=sample.tools)
+            return model_output
+
+    def _on_inference_end(
+        self, model: Model, sample: Sample, model_output: ModelOutput, output_dir: str, **kwargs
+    ) -> TaskState:
+        """
+        Hook method called after inference completes. Save generated images to output_dir.
+
+        Args:
+            model (Model): The model that performed inference
+            sample (Sample): The processed sample
+            model_output (ModelOutput): The raw model output
+            output_dir (str): The directory where the model output was saved
+
+        Returns:
+            TaskState: Complete state object for the inference task
+        """
+        if self.eval_type == EvalType.MOCK_LLM:
+            return TaskState(
+                model=model.name,
+                sample=sample,
+                messages=[model_output.message],
+                output=model_output,
+                completed=True,
+            )
+        else:
+            image_id = f'{sample.metadata.get(FileConstants.ID, sample.id)}_{sample.group_id}'
+            output_path = os.path.join(output_dir, 'images', f'{image_id}.png')
+            if not os.path.exists(os.path.dirname(output_path)):
+                os.makedirs(os.path.dirname(output_path))
+            # get base64 image from model_output
+            content = model_output.message.content[0]
+
+            assert isinstance(content, ContentImage), 'Expected ContentImage in model output'
+
+            image_base64 = content.image
+            with open(output_path, 'wb') as f:
+                f.write(base64.b64decode(image_base64))
+
+            sample.metadata[FileConstants.IMAGE_PATH] = output_path
+            return TaskState(
+                model=model.name,
+                sample=sample,
+                messages=[model_output.message],
+                output=model_output,
+                completed=True,
+            )
+
+    # NOTE: thread safe is needed, since we can't batch inference here.
+    @thread_safe
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        # Get prediction and prompt from task state
+        image_path = task_state.metadata.get(FileConstants.IMAGE_PATH, original_prediction)
+        prompt = task_state.input[0].content
+        meta = task_state.metadata
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=image_path,
+            prediction=image_path,
+        )
+
+        # Calculate scores for each configured metric
+        for metric in self.metric_list:
+            try:
+                if isinstance(metric, str):
+                    metric_name = metric
+                    metric_scorer = get_metric(metric)  # Get metric implementation from registry
+                    metric_func = metric_scorer()  # Instantiate the metric scorer
+                elif isinstance(metric, dict):
+                    metric_name = list(metric.keys())[0]
+                    metric_cls = get_metric(metric_name)
+                    metric_func = metric_cls(**metric[metric_name])  # Initialize with parameters
+                metric_score = metric_func(image_path, prompt)[0]
+
+                # fine-granular metrics
+                category = meta.get('category')
+                if category:
+                    metric_name = f'{metric_name}_{category}'
+                if isinstance(metric_score, dict):
+                    for k, v in metric_score.items():
+                        score.value[f'{metric_name}_{k}'] = v.cpu().item()
+                else:
+                    score.value[metric_name] = metric_score.cpu().item()
+            except Exception as e:
+                logger.error(f'Error calculating metric {metric}: {e}')
+                score.value[metric_name] = 0
+                score.metadata[metric_name] = f'error: {str(e)}'
+
+        return score
+
+    def _on_generate_report(self, scores, model_name, add_aggregation_name=True):
+        # Don't add aggregation name for needle haystack adapter
+        return super()._on_generate_report(scores, model_name, False)

evalscope/api/benchmark/adapters/vision_language_adapter.py

@@ -0,0 +1,6 @@
+from .default_data_adapter import DefaultDataAdapter
+
+
+class VisionLanguageAdapter(DefaultDataAdapter):
+    """Adapter for vision-language benchmarks. e.g., image captioning, visual question answering, etc."""
+    pass

evalscope/api/benchmark/benchmark.py

@@ -0,0 +1,356 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import contextlib
+from abc import ABC, abstractmethod
+from collections import OrderedDict
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+from evalscope.api.dataset import DatasetDict, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.filter import FilterEnsemble, build_filter_ensemble
+from evalscope.api.metric import AggScore, SampleScore
+from evalscope.api.mixin import LLMJudgeMixin
+from evalscope.api.model import Model
+from evalscope.report import Report
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.api.benchmark import BenchmarkMeta
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class DataAdapter(LLMJudgeMixin, ABC):
+    """
+    Data Adapter for the benchmark.
+    """
+
+    def __init__(self, benchmark_meta: 'BenchmarkMeta', task_config: Optional['TaskConfig'] = None):
+        self._benchmark_meta = benchmark_meta
+        self._task_config = task_config
+        super().__init__(task_config=task_config)
+
+        self.reformat_subset = False
+        """Whether to reformat the subset data with subset key"""
+
+        self.split_as_subset = False
+        """Whether to use the split name as the dataset subsets"""
+
+        self.shuffle_choices = False
+        """Whether to shuffle the choices in the dataset"""
+
+        self.save_metadata = True
+        """Whether to save metadata in the review result"""
+
+        self.category_map = {}
+        """Category map for the benchmark"""
+
+        self.current_subset_name = ''
+        """Subset name when loading datasets"""
+
+        # dataset
+        self.test_dataset: Optional[DatasetDict] = None
+        """Dataset to be evaluated"""
+
+        self.fewshot_dataset: Optional[DatasetDict] = None
+        """Dataset for few-shot evaluation"""
+
+        # filters
+        self._filter_ensemble: Optional[OrderedDict] = None
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the benchmark metadata to a dictionary."""
+        return self._benchmark_meta.to_string_dict()
+
+    @abstractmethod
+    def load_dataset(self) -> DatasetDict:
+        pass
+
+    @abstractmethod
+    def run_inference(self, model: Model, sample: Sample, output_dir: str, **kwargs) -> TaskState:
+        pass
+
+    @abstractmethod
+    def calculate_metrics(self, task_state: TaskState) -> SampleScore:
+        pass
+
+    @abstractmethod
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        pass
+
+    @abstractmethod
+    def generate_report(self, scores: Dict[str, List[AggScore]], model_name: str, output_dir: str, **kwargs) -> Report:
+        """
+        Generate a report based on the evaluation results.
+        """
+        pass
+
+    @property
+    def name(self) -> str:
+        """
+        Return the unique name of the benchmark.
+        """
+        return self._benchmark_meta.name
+
+    @property
+    def dataset_id(self) -> str:
+        """
+        Return the dataset ID or path to the benchmark.
+        """
+        return self._benchmark_meta.dataset_id
+
+    @property
+    def output_types(self) -> Optional[List[str]]:
+        """
+        Return the output types of the benchmark.
+        """
+        return self._benchmark_meta.output_types
+
+    @property
+    def limit(self) -> Optional[Union[int, float]]:
+        """
+        Return the limit for the benchmark.
+        """
+        return self._task_config.limit
+
+    @property
+    def repeats(self) -> int:
+        """
+        Return the number of repeats for each sample in the benchmark.
+        """
+        return self._task_config.repeats
+
+    @property
+    def dataset_hub(self) -> str:
+        """
+        Return the dataset hub type for the benchmark.
+        """
+        return self._task_config.dataset_hub
+
+    @dataset_hub.setter
+    def dataset_hub(self, value: str):
+        """
+        Set the dataset hub type for the benchmark.
+        """
+        self._task_config.dataset_hub = value
+
+    @property
+    def eval_type(self) -> str:
+        """
+        Return the evaluation type for the benchmark.
+        """
+        return self._task_config.eval_type
+
+    @property
+    def subset_list(self) -> List[str]:
+        """
+        Return the subset list of the benchmark.
+        """
+        return self._benchmark_meta.subset_list
+
+    @subset_list.setter
+    def subset_list(self, value: List[str]):
+        """
+        Set the subset list of the benchmark.
+        """
+        self._benchmark_meta.subset_list = value
+
+    @property
+    def metric_list(self) -> List[Union[str, Dict[str, Any]]]:
+        """
+        Return the metric list of the benchmark.
+        """
+        return self._benchmark_meta.metric_list
+
+    @property
+    def default_subset(self) -> str:
+        """
+        Return the default subset of the benchmark.
+        """
+        return self._benchmark_meta.default_subset
+
+    @default_subset.setter
+    def default_subset(self, value: str):
+        """
+        Set the default subset of the benchmark.
+        """
+        self._benchmark_meta.default_subset = value
+
+    @property
+    def few_shot_num(self) -> int:
+        """
+        Return the few shot number of the benchmark.
+        """
+        return self._benchmark_meta.few_shot_num
+
+    @few_shot_num.setter
+    def few_shot_num(self, value: int):
+        """
+        Set the few shot number of the benchmark.
+        """
+        self._benchmark_meta.few_shot_num = value
+
+    @property
+    def few_shot_random(self) -> bool:
+        """
+        Return whether few shot is random for the benchmark.
+        """
+        return self._benchmark_meta.few_shot_random
+
+    @property
+    def train_split(self) -> Optional[str]:
+        """
+        Return the train split of the benchmark.
+        """
+        return self._benchmark_meta.train_split
+
+    @property
+    def eval_split(self) -> Optional[str]:
+        """
+        Return the eval split of the benchmark.
+        """
+        return self._benchmark_meta.eval_split
+
+    @property
+    def prompt_template(self) -> Optional[str]:
+        """
+        Return the prompt template of the benchmark.
+        """
+        return self._benchmark_meta.prompt_template
+
+    @prompt_template.setter
+    def prompt_template(self, value: str):
+        """
+        Set the prompt template of the benchmark.
+        """
+        self._benchmark_meta.prompt_template = value
+
+    @property
+    def system_prompt(self) -> Optional[str]:
+        """
+        Return the system prompt of the benchmark.
+        """
+        return self._benchmark_meta.system_prompt
+
+    @property
+    def query_template(self) -> Optional[str]:
+        """
+        Return the query template of the benchmark.
+        """
+        return self._benchmark_meta.query_template
+
+    @property
+    def few_shot_prompt_template(self) -> Optional[str]:
+        """
+        Return the few-shot prompt template of the benchmark.
+        """
+        return self._benchmark_meta.few_shot_prompt_template
+
+    @property
+    def pretty_name(self) -> Optional[str]:
+        """
+        Return the pretty name of the benchmark.
+        """
+        return self._benchmark_meta.pretty_name
+
+    @property
+    def description(self) -> Optional[str]:
+        """
+        Return the description of the benchmark.
+        """
+        return self._benchmark_meta.description
+
+    @property
+    def tags(self) -> Optional[List[str]]:
+        """
+        Return the tags of the benchmark.
+        """
+        return self._benchmark_meta.tags
+
+    @property
+    def filters(self) -> Optional[OrderedDict]:
+        """
+        Return the filters of the benchmark.
+        """
+        return self._benchmark_meta.filters
+
+    @property
+    def filter_ensemble(self) -> Optional[FilterEnsemble]:
+        """
+        Return the filter ensemble of the benchmark.
+        """
+        if self._filter_ensemble is None:
+            if self.filters:
+                self._filter_ensemble = build_filter_ensemble(filters=self.filters)
+        return self._filter_ensemble
+
+    @property
+    def aggregation(self) -> str:
+        """
+        Return the aggregation function for the metrics.
+        """
+        return self._benchmark_meta.aggregation
+
+    @property
+    def extra_params(self) -> Optional[Dict]:
+        """
+        Return the extra parameters of the benchmark.
+        """
+        return self._benchmark_meta.extra_params
+
+    @property
+    def seed(self) -> Optional[int]:
+        """
+        Return the seed for the benchmark.
+        """
+        return self._task_config.seed
+
+    @property
+    def shuffle(self) -> bool:
+        """
+        Return whether to shuffle the dataset before evaluation.
+        """
+        return self._benchmark_meta.shuffle
+
+    @shuffle.setter
+    def shuffle(self, value: bool):
+        """
+        Set whether to shuffle the dataset before evaluation.
+        """
+        self._benchmark_meta.shuffle = value
+
+    @property
+    def shuffle_choices(self) -> bool:
+        """
+        Return whether to shuffle the choices in multiple-choice datasets.
+        """
+        return self._benchmark_meta.shuffle_choices
+
+    @shuffle_choices.setter
+    def shuffle_choices(self, value: bool):
+        """
+        Set whether to shuffle the choices in multiple-choice datasets.
+        """
+        self._benchmark_meta.shuffle_choices = value
+
+    @contextlib.contextmanager
+    def _temporary_attribute(self, attr_name: str, new_value):
+        """
+        Set a temporary value for an attribute and restore the original value after the context block.
+
+        Args:
+            attr_name: The name of the attribute to temporarily set.
+            new_value: The new value to set for the attribute.
+        """
+        had_attr = hasattr(self, attr_name)
+        original_value = getattr(self, attr_name, None) if had_attr else None
+
+        setattr(self, attr_name, new_value)
+        try:
+            yield
+        finally:
+            if had_attr:
+                setattr(self, attr_name, original_value)
+            else:
+                delattr(self, attr_name)