evalscope 1.0.0__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/backend/opencompass/backend_manager.py

@@ -47,7 +47,6 @@ class OpenCompassBackendManager(BackendManager):
                     datasets: list, the datasets.
                     models: list, the models.
                     work_dir (Optional): str, the working directory. Default to None, which means the current directory.
-                    dry_run (Optional): bool, the dry-run flag. Default to False.
                     debug (Optional): bool, the debug flag. Default to False.
                     reuse (Optional): str, reuse previous outputs & results. Default to None.
                     generation_kwargs (Optional): dict, the generation config. Default to {}.
@@ -140,7 +139,6 @@ class OpenCompassBackendManager(BackendManager):
             cmd_str = f'python -m run_oc ' \
                       f'--models {" ".join(self.args.models)} ' \
                       f'--datasets {" ".join(self.args.datasets)} ' \
-                      f'{self.get_restore_arg("dry-run", self.args.dry_run)} ' \
                       f'{self.get_arg_with_default("work-dir", self.args.work_dir)}'
 
         elif cmd_mode == CmdMode.SCRIPT:

evalscope/backend/rag_eval/utils/embedding.py

@@ -164,6 +164,13 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
             automodel_args=self.model_kwargs,
         )
+        self.tokenizer = self.model.tokenizer
+        # set pad token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        if ('pad_token_id' not in self.model.config) or (self.model.config.pad_token_id is None):
+            self.model.config.update({'pad_token_id': self.tokenizer.eos_token_id})
+
         self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
@@ -189,6 +196,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.check_embedding_ctx_length = kwargs.get('check_embedding_ctx_length', False)
         self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
@@ -196,7 +204,7 @@ class APIEmbeddingModel(BaseModel):
             openai_api_base=self.openai_api_base,
             openai_api_key=self.openai_api_key,
             dimensions=self.dimensions,
-            check_embedding_ctx_length=False
+            check_embedding_ctx_length=self.check_embedding_ctx_length,
         )
 
         super().__init__(model_name_or_path=self.model_name, **kwargs)

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -1,4 +1,3 @@
-import importlib
 import json
 import re
 import traceback
@@ -12,6 +11,7 @@ from evalscope.api.metric import Score
 from evalscope.api.model import Model, ModelOutput
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -67,11 +67,7 @@ class BFCLAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        spec = importlib.util.find_spec('bfcl_eval')
-        if spec is None:
-            raise ImportError(
-                '`bfcl_eval` not found, please install it with `pip install bfcl-eval==2025.6.16` before evaluating.'
-            )
+        check_import('bfcl_eval', package='bfcl-eval==2025.6.16', raise_error=True)
 
         self.category_map = SUBJECT_MAPPING
         self.reformat_subset = True

evalscope/benchmarks/bfcl/generation.py

@@ -78,7 +78,7 @@ def generate_turn(model: Model, row: dict[str, Any]):
             if isinstance(message, str):
                 result = message
             else:
-                result = message.content
+                result = message.text
 
             logger.debug(f'Turn:{turn_idx} Step:{n_steps} Result: {result}')
             current_responses.append(result)
@@ -186,7 +186,7 @@ def generate_turn_with_tools(model: Model, row: dict[str, Any]):
                     logger.error(f'Error converting tool calls to function call strings: {e}')
                     tool_call_strs = None
             else:
-                model_responses = [message.content]
+                model_responses = [message.text]
                 tool_call_strs = None
 
             current_responses.extend(model_responses)

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,10 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from functools import partial
 from typing import Any, Dict
 
 from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
-from evalscope.api.dataset import Dataset, RemoteDataLoader, Sample
+from evalscope.api.dataset import Sample
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -6,9 +6,7 @@ from typing import Any, Dict, List
 from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
 from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
 from evalscope.api.evaluator import TaskState
-from evalscope.api.metric import Score
 from evalscope.api.metric.scorer import AggScore, SampleScore
-from evalscope.api.model.model import Model
 from evalscope.api.registry import get_benchmark, register_benchmark
 from evalscope.config import TaskConfig
 from evalscope.constants import DataCollection, Tags
@@ -23,7 +21,11 @@ logger = get_logger()
     BenchmarkMeta(
         name=DataCollection.NAME,
         dataset_id='',  # dataset_id need to be set
-        description='Data collection',
+        description='Custom Data collection, mixing multiple evaluation datasets for '
+        'a unified evaluation, aiming to use less data to achieve a more comprehensive '
+        'assessment of the model\'s capabilities. '
+        '[Usage Reference](https://evalscope.readthedocs.io/zh-cn/latest/advanced_guides/collection/index.html)',
+        tags=[Tags.CUSTOM],
         metric_list=['acc'],
         eval_split='test',
         prompt_template='',
@@ -55,9 +57,10 @@ class DataCollectionAdapter(DefaultDataAdapter):
             data_id_or_path=dataset_path,
             split=self.eval_split,
             sample_fields=self.record_to_sample,
-            subset=self.default_subset,
+            subset='test',  # NOTE: using hardcoded test subset
             limit=self.limit,
-            repeats=self.repeats
+            repeats=self.repeats,
+            shuffle=self.shuffle,
         ).load()
 
         test_dataset = DatasetDict({self.default_subset: dataset})
@@ -95,7 +98,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
         # load dataset args
         dataset_args = copy.deepcopy(self._task_config.dataset_args)
-        common_args = dataset_args.get(DataCollection.NAME, {})
 
         # Iterate through each sample in the dataset
         dataset = self.test_dataset[self.default_subset]
@@ -108,7 +110,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
             # update dataset args
             cur_dataset_args = dataset_args.get(dataset_name, {})
-            cur_dataset_args.update(common_args)
 
             # Initialize dataset adapter
             if dataset_name not in self.dataset_adapters:
@@ -141,19 +142,22 @@ class DataCollectionAdapter(DefaultDataAdapter):
         data = []
         for sample_score in sample_scores:
             collection_info = sample_score.sample_metadata[DataCollection.INFO]
-            for metric_name, value in sample_score.score.value.items():
-                data.append(
-                    dict(
-                        task_type=collection_info['task_type'],
-                        categories=tuple(collection_info['categories']),
-                        dataset_name=collection_info['dataset_name'],
-                        subset_name=collection_info['subset_name'],
-                        tags=collection_info['tags'],
-                        sample_id=sample_score.sample_id,
-                        metric=metric_name,
-                        score=value
-                    )
+            main_score = sample_score.score.main_value
+            main_metric = sample_score.score.main_score_name
+
+            # use main score
+            data.append(
+                dict(
+                    task_type=collection_info['task_type'],
+                    categories=tuple(collection_info['categories']),
+                    dataset_name=collection_info['dataset_name'],
+                    subset_name=collection_info['subset_name'],
+                    tags=collection_info['tags'],
+                    sample_id=sample_score.sample_id,
+                    metric=main_metric,
+                    score=main_score
                 )
+            )
 
         df = pd.DataFrame(data)
 

evalscope/benchmarks/frames/frames_adapter.py

@@ -61,7 +61,8 @@ class FramesAdapter(DefaultDataAdapter):
             sample_fields=self.record_to_sample,
             subset='test',
             limit=self.limit,
-            repeats=self.repeats
+            repeats=self.repeats,
+            shuffle=self.shuffle,
         ).load()
 
         test_dataset = DatasetDict({'test': dataset})

evalscope/benchmarks/general_arena/general_arena_adapter.py

@@ -75,7 +75,11 @@ class GeneralArenaAdapter(DefaultDataAdapter):
         dataset_dict = {}
         for subset_name, samples in datasets.items():
             dataset = DictDataLoader(
-                dict_list=samples, limit=self.limit, repeats=self.repeats, sample_fields=self.record_to_sample
+                dict_list=samples,
+                limit=self.limit,
+                shuffle=self.shuffle,
+                repeats=self.repeats,
+                sample_fields=self.record_to_sample
             ).load()
             dataset_dict[subset_name] = dataset
 

evalscope/benchmarks/ifeval/instructions_util.py

@@ -14,7 +14,6 @@
 """Utility library of instructions."""
 
 import functools
-import immutabledict
 import nltk
 import os
 import random
@@ -1551,7 +1550,7 @@ WORD_LIST = [
 ]  # pylint: disable=line-too-long
 
 # ISO 639-1 codes to language names.
-LANGUAGE_CODES = immutabledict.immutabledict({
+LANGUAGE_CODES = {
     'en': 'English',
     'es': 'Spanish',
     'pt': 'Portuguese',
@@ -1582,7 +1581,7 @@ LANGUAGE_CODES = immutabledict.immutabledict({
     'pa': 'Punjabi',
     'ml': 'Malayalam',
     'fi': 'Finnish',
-})
+}
 
 _ALPHABETS = '([A-Za-z])'
 _PREFIXES = '(Mr|St|Mrs|Ms|Dr)[.]'

evalscope/benchmarks/image_edit/gedit/gedit_adapter.py

@@ -0,0 +1,138 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import copy
+import os
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, ImageEditAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import FileConstants, Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'background_change', 'color_alter', 'material_alter', 'motion_change', 'ps_human', 'style_change', 'subject-add',
+    'subject-remove', 'subject-replace', 'text_change', 'tone_transfer'
+]
+
+LANGUAGE_LIST = ['en', 'cn']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gedit',
+        pretty_name='GEdit-Bench',
+        dataset_id='stepfun-ai/GEdit-Bench',
+        description='GEdit-Bench Image Editing Benchmark, grounded in real-world '
+        'usages is developed to support more authentic and '
+        'comprehensive evaluation of image editing models.',
+        tags=[Tags.IMAGE_EDITING],
+        subset_list=SUBSET_LIST,
+        metric_list=['Semantic Consistency', 'Perceptual Similarity'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        extra_params={'language': f'# language of the instruction, choose from {LANGUAGE_LIST}, default to `en`'}
+    )
+)
+class GEditAdapter(ImageEditAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.language = self.extra_params.get('language', 'en')
+        if self.language not in LANGUAGE_LIST:
+            logger.warning(f"Invalid language '{self.language}', fallback to 'en'")
+            self.language = 'en'
+        self.reformat_subset = True
+        self._use_llm_judge = True
+
+        self.load_prompt()
+
+    def load_prompt(self):
+        from . import vie_prompts
+
+        self.context = vie_prompts._context_no_delimit
+        self.SC_prompt = '\n'.join([
+            self.context, vie_prompts._prompts_0shot_two_image_edit_rule, vie_prompts._prompts_0shot_tie_rule_SC
+        ])
+        self.PQ_prompt = '\n'.join([self.context, vie_prompts._prompts_0shot_rule_PQ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        record = copy.deepcopy(record)
+
+        # Process instruction and image
+        instruction = record['instruction']
+        image_bytes = record['input_image']['bytes']
+        input_image = bytes_to_base64(image_bytes, format='png', add_header=True)
+        record['input_image'] = input_image
+        record[FileConstants.ID] = record['key']
+        del record['input_image_raw']
+
+        text_content = ContentText(text=instruction)
+        image_content = ContentImage(image=input_image)
+
+        messages: List[ChatMessage] = [
+            ChatMessageUser(content=[text_content, image_content]),
+        ]
+
+        return Sample(input=messages, subset_key=record['task_type'], metadata=record)
+
+    def sample_filter(self, sample: Sample) -> bool:
+        language = sample.metadata.get('instruction_language', 'en')
+        return super().sample_filter(sample) and language == self.language
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state: TaskState) -> Score:
+        import math
+
+        from .utils import mllm_output_to_dict
+
+        metadata = task_state.metadata
+        text_prompt = metadata['instruction']
+        input_image = metadata['input_image']  # base64 image
+        edited_image = metadata[FileConstants.IMAGE_PATH]  # local image path
+        _SC_prompt = self.SC_prompt.replace('<instruction>', text_prompt)
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=edited_image,
+            prediction=edited_image,
+        )
+
+        # Build prompts
+        SC_prompt_final = [
+            ChatMessageUser(
+                content=[
+                    ContentImage(image=input_image),
+                    ContentImage(image=edited_image),
+                    ContentText(text=_SC_prompt)
+                ]
+            )
+        ]
+        PQ_prompt_final = [
+            ChatMessageUser(content=[ContentImage(image=edited_image),
+                                     ContentText(text=self.PQ_prompt)])
+        ]
+
+        guess_if_cannot_parse = True
+        result_SC = self.llm_judge.judge(messages=SC_prompt_final)
+        result_PQ = self.llm_judge.judge(messages=PQ_prompt_final)
+        SC_dict = mllm_output_to_dict(result_SC, give_up_parsing=guess_if_cannot_parse)
+        PQ_dict = mllm_output_to_dict(result_PQ, give_up_parsing=guess_if_cannot_parse)
+
+        SC_score = min(SC_dict['score'])
+        PQ_score = min(PQ_dict['score'])
+        O_score = math.sqrt(SC_score * PQ_score)
+
+        score.value = {'Semantic Consistency': SC_score, 'Perceptual Quality': PQ_score, 'Overall': O_score}
+        score.main_score_name = 'Overall'
+        score.metadata = {
+            'SC_dict': SC_dict,
+            'PQ_dict': PQ_dict,
+        }
+        return score