evalscope 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

evalscope/benchmarks/real_world_qa/real_world_qa_adapter.py

@@ -0,0 +1,64 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+OPEN_PROMPT = (
+    'Read the picture and solve the following problem step by step.'
+    'The last line of your response should be of the form'
+    ' "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.\n\n'
+    '{question}\n\n'
+    'Remember to put your answer on its own line at the end in the form'
+    ' "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem,'
+    ' and you do not need to use a \\boxed command.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='real_world_qa',
+        pretty_name='RealWorldQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'RealWorldQA is a benchmark designed to evaluate the real-world spatial understanding capabilities of multimodal AI models, contributed by XAI. It assesses how well these models comprehend physical environments. The benchmark consists of 700+ images, each accompanied by a question and a verifiable answer. These images are drawn from real-world scenarios, including those captured from vehicles. The goal is to advance AI models\' understanding of our physical world.',  # noqa: E501
+        dataset_id='lmms-lab/RealWorldQA',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class RealWorldQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        content_list: list[Content] = [ContentText(text=OPEN_PROMPT.format(question=record['question']))]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='webp', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            metadata={'image_path': record['image_path']}
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return ''

evalscope/benchmarks/tau_bench/tau_bench_adapter.py

@@ -47,7 +47,12 @@ class TauBenchAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        check_import('tau_bench', package='git+https://github.com/sierra-research/tau-bench', raise_error=True)
+        check_import(
+            'tau_bench',
+            package='git+https://github.com/sierra-research/tau-bench',
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
 
         # setup user model args
         self.user_model = self.extra_params.get('user_model', 'qwen-plus')

evalscope/config.py

@@ -18,6 +18,7 @@ from evalscope.constants import (
 )
 from evalscope.utils.argument_utils import BaseArgument, parse_int_or_float
 from evalscope.utils.deprecation_utils import deprecated_warning
+from evalscope.utils.import_utils import check_import
 from evalscope.utils.io_utils import dict_to_yaml, gen_hash, safe_filename
 from evalscope.utils.logger import get_logger
 
@@ -124,6 +125,19 @@ class TaskConfig(BaseArgument):
     analysis_report: bool = False
     """Whether to generate detailed analysis reports after evaluation."""
 
+    # Sandbox configuration arguments
+    use_sandbox: bool = False
+    """Whether to execute code in a sandboxed environment."""
+
+    sandbox_type: Optional[str] = 'docker'
+    """Type of sandbox environment for code execution (e.g., docker). Default is 'docker'."""
+
+    sandbox_manager_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for the sandbox manager. Default is local manager. If url is provided, it will use remote manager."""
+
+    sandbox_config: Optional[Dict] = field(default_factory=dict)
+    """Configuration for sandboxed code execution environments."""
+
     def __post_init__(self):
         self.__init_model_and_id()
 
@@ -132,6 +146,7 @@ class TaskConfig(BaseArgument):
         # Set default generation_config and model_args
         self.__init_default_generation_config()
         self.__init_default_model_args()
+        self.__init_default_sandbox_config()
 
     def __init_model_and_id(self):
         # Set model to DummyCustomModel if not provided
@@ -223,6 +238,14 @@ class TaskConfig(BaseArgument):
                     'precision': 'torch.float16',
                 }
 
+    def __init_default_sandbox_config(self):
+        if not self.use_sandbox:
+            return
+        check_import('ms_enclave', 'ms_enclave[docker]', raise_error=True)
+
+        if not self.sandbox_type:
+            self.sandbox_type = 'docker'
+
     def update(self, other: Union['TaskConfig', dict]):
         if isinstance(other, TaskConfig):
             other = other.to_dict()
@@ -238,7 +261,7 @@ class TaskConfig(BaseArgument):
             logger.warning(f'Failed to dump overall task config: {e}')
 
     def to_dict(self):
-        result = copy.deepcopy(self.__dict__)
+        result = copy.copy(self.__dict__)
         del result['api_key']  # Do not expose api_key in the config
 
         if isinstance(self.model, (Model, ModelAPI)):

evalscope/constants.py

@@ -15,6 +15,7 @@ DEFAULT_ROOT_CACHE_DIR = DEFAULT_DATASET_CACHE_DIR  # compatible with old versio
 DEFAULT_EVALSCOPE_CACHE_DIR = os.path.expanduser(
     os.getenv('EVALSCOPE_CACHE', '~/.cache/evalscope')
 )  # ~/.cache/evalscope
+IS_BUILD_DOC = os.getenv('BUILD_DOC', '0') == '1'  # To avoid some heavy dependencies when building doc
 
 
 class HubType:
@@ -130,6 +131,8 @@ class Tags:
     TEXT_TO_IMAGE = 'TextToImage'
     IMAGE_EDITING = 'ImageEditing'
     MULTI_MODAL = 'MultiModal'
+    MULTI_LINGUAL = 'MultiLingual'
+    MULTI_TURN = 'MultiTurn'
 
 
 class FileConstants:

evalscope/evaluator/evaluator.py

@@ -8,8 +8,9 @@ and report generation.
 """
 
 import os
+import traceback
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed
 from tqdm import tqdm
 from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
@@ -17,6 +18,7 @@ from evalscope.api.dataset import Dataset, DatasetDict, Sample
 from evalscope.api.evaluator import CacheManager, Evaluator, TaskState
 from evalscope.api.metric import AggScore, SampleScore
 from evalscope.report import Report, gen_table
+from evalscope.utils.logger import get_logger
 
 if TYPE_CHECKING:
     from evalscope.api.benchmark import DataAdapter
@@ -24,8 +26,6 @@ if TYPE_CHECKING:
     from evalscope.config import TaskConfig
     from evalscope.utils.io_utils import OutputsStructure
 
-from evalscope.utils.logger import get_logger
-
 logger = get_logger()
 
 
@@ -104,6 +104,9 @@ class DefaultEvaluator(Evaluator):
 
         # Generate the report based on aggregated scores
         report = self.get_report(agg_score_dict)
+
+        # Finalize the evaluation process
+        self.finalize()
         return report
 
     def evaluate_subset(self, subset: str, dataset: Dataset) -> List[AggScore]:
@@ -186,7 +189,10 @@ class DefaultEvaluator(Evaluator):
                         logger.debug(f'Model result: \n{model_result.pretty_print()}')
 
                     except Exception as exc:
-                        logger.error(f'{sample.model_dump_json(indent=2)} prediction failed: due to {exc}')
+                        tb_str = traceback.format_exc()
+                        logger.error(
+                            f'{sample.model_dump_json(indent=2)} prediction failed: due to {exc}\nTraceback:\n{tb_str}'
+                        )
                         if self.task_config.ignore_errors:
                             logger.warning('Error ignored, continuing with next sample.')
                         else:
@@ -253,7 +259,13 @@ class DefaultEvaluator(Evaluator):
                 for future in as_completed(future_to_task_state):
                     task_state = future_to_task_state[future]
                     try:
-                        sample_score = future.result()
+                        try:
+                            sample_score = future.result()
+                        except TimeoutError:
+                            logger.warning(
+                                f'Timeout when reviewing sample {task_state.sample_id}, setting score to zero.'
+                            )
+                            sample_score = SampleScore(sample_id=task_state.sample_id, scores={})
                         sample_score_list.append(sample_score)
 
                         # Save the review result to cache for future use
@@ -266,7 +278,10 @@ class DefaultEvaluator(Evaluator):
                         logger.debug(f'Review result: \n{review_result.pretty_print()}')
 
                     except Exception as exc:
-                        logger.error(f'Error when review sample {task_state.sample_id}: due to {exc}')
+                        tb_str = traceback.format_exc()
+                        logger.error(
+                            f'Error when review sample {task_state.sample_id}: due to {exc}\nTraceback:\n{tb_str}'
+                        )
                         if self.task_config.ignore_errors:
                             logger.warning('Error ignored, continuing with next sample.')
                         else:
@@ -319,7 +334,7 @@ class DefaultEvaluator(Evaluator):
 
         # Generate and display a summary table of results
         try:
-            report_table = gen_table(report_list=[report], add_overall_metric=True)
+            report_table = gen_table(report_list=[report], add_overall_metric=self.benchmark.add_overall_metric)
             logger.info(f'\n{self.benchmark_name} report table:'
                         f'\n{report_table} \n')
         except Exception:
@@ -337,3 +352,6 @@ class DefaultEvaluator(Evaluator):
         report.to_json(report_file)
         logger.info(f'Dump report to: {report_file} \n')
         return report
+
+    def finalize(self, *args, **kwargs):
+        self.benchmark.finalize(*args, **kwargs)

evalscope/metrics/metric.py

@@ -6,11 +6,19 @@ from evalscope.api.registry import register_aggregation, register_metric
 from .metrics import mean
 
 
+def normalize_text(text: str) -> str:
+    """Normalize text by lowering case and stripping whitespace."""
+    return text.strip().lower()
+
+
 @register_metric(name='exact_match')
 class ExactMatch(Metric):
 
     def apply(self, predictions, references):
-        return [float(prediction == reference) for prediction, reference in zip(predictions, references)]
+        return [
+            float(normalize_text(prediction) == normalize_text(reference))
+            for prediction, reference in zip(predictions, references)
+        ]
 
 
 @register_metric(name='acc')
@@ -202,6 +210,9 @@ class Mean(Aggregator):
 
     name = 'mean'
 
+    def agg_func(self, values: List[float]) -> float:
+        return mean(values)
+
     def __call__(self, scores: List[SampleScore]) -> List[AggScore]:
         """Aggregate scores by computing the mean for each metric.
 
@@ -230,7 +241,7 @@ class Mean(Aggregator):
             if values:  # Only process non-empty value lists
                 aggregated_scores.append(
                     AggScore(
-                        score=mean(values),
+                        score=self.agg_func(values),
                         metric_name=metric_name,
                         aggregation_name=self.name,
                         num=len(values),
@@ -241,6 +252,20 @@ class Mean(Aggregator):
         return aggregated_scores
 
 
+@register_aggregation(name='clipped_mean')
+class ClippedMean(Mean):
+
+    name = 'clipped_mean'
+
+    def __init__(self, clip_min: float = 0.0, clip_max: float = 1.0):
+        self.clip_min = clip_min
+        self.clip_max = clip_max
+
+    def agg_func(self, values: List[float]) -> float:
+        clipped_values = min(max(mean(values), self.clip_min), self.clip_max)
+        return clipped_values
+
+
 @register_aggregation(name='pass_at_k')
 class PassAtK(Aggregator):
 

evalscope/models/model_apis.py

@@ -28,7 +28,7 @@ def server() -> type[ModelAPI]:
 
 @register_model_api(name='llm_ckpt')
 def llm_ckpt() -> type[ModelAPI]:
-    check_import('torch', package='torch', raise_error=True)
+    check_import('torch', package='torch', raise_error=True, feature_name='llm_ckpt')
 
     from .modelscope import ModelScopeAPI
 
@@ -38,7 +38,7 @@ def llm_ckpt() -> type[ModelAPI]:
 @register_model_api(name='checkpoint')
 @deprecated(since='1.0.0', remove_in='1.1.0', alternative='llm_ckpt')
 def checkpoint() -> type[ModelAPI]:
-    check_import('torch', package='torch', raise_error=True)
+    check_import('torch', package='torch', raise_error=True, feature_name='llm_ckpt')
 
     from .modelscope import ModelScopeAPI
 
@@ -47,9 +47,10 @@ def checkpoint() -> type[ModelAPI]:
 
 @register_model_api(name='text2image')
 def text2image() -> type[ModelAPI]:
-    check_import('torch', package='evalscope[aigc]', raise_error=True)
-    check_import('torchvision', package='evalscope[aigc]', raise_error=True)
-    check_import('diffusers', package='evalscope[aigc]', raise_error=True)
+    check_import(['torch', 'torchvision', 'diffusers'],
+                 package='evalscope[aigc]',
+                 raise_error=True,
+                 feature_name='text2image')
 
     from .text2image_model import Text2ImageAPI
 
@@ -58,9 +59,10 @@ def text2image() -> type[ModelAPI]:
 
 @register_model_api(name='image_editing')
 def image_editing() -> type[ModelAPI]:
-    check_import('torch', package='evalscope[aigc]', raise_error=True)
-    check_import('torchvision', package='evalscope[aigc]', raise_error=True)
-    check_import('diffusers', package='evalscope[aigc]', raise_error=True)
+    check_import(['torch', 'torchvision', 'diffusers'],
+                 package='evalscope[aigc]',
+                 raise_error=True,
+                 feature_name='image_editing')
 
     from .image_edit_model import ImageEditAPI
 

evalscope/models/utils/openai.py

@@ -104,10 +104,9 @@ def openai_chat_completion_part(content: Content) -> ChatCompletionContentPartPa
         )
     elif content.type == 'audio':
         audio_data_uri = file_as_data_uri(content.audio)
-        audio_data = audio_data_uri.split('base64,')[1]
 
         return ChatCompletionContentPartInputAudioParam(
-            type='input_audio', input_audio=dict(data=audio_data, format=content.format)
+            type='input_audio', input_audio=dict(data=audio_data_uri, format=content.format)
         )
 
     else:

evalscope/perf/arguments.py

@@ -55,6 +55,7 @@ class Arguments(BaseArgument):
     image_height: int = 224  # Height of the image for random VL dataset
     image_format: str = 'RGB'  # Image format for random VL dataset
     image_num: int = 1  # Number of images for random VL dataset
+    image_patch_size: int = 28  # Patch size for image tokenizer, only for local image token calculation
 
     # Dataset settings
     dataset: str = 'openqa'  # Dataset type (default: 'line_by_line')
@@ -171,6 +172,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--image-height', type=int, default=224, help='Height of the image for random VL dataset')
     parser.add_argument('--image-format', type=str, default='RGB', help='Image format for random VL dataset')
     parser.add_argument('--image-num', type=int, default=1, help='Number of images for random VL dataset')
+    parser.add_argument('--image-patch-size', type=int, default=28, help='Patch size for image tokenizer, only for local image token calculation')  # noqa: E501
 
     # Output settings
     parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')

evalscope/perf/plugin/api/base.py

@@ -43,7 +43,7 @@ class ApiPluginBase:
 
     @abstractmethod
     async def process_request(self, client_session: aiohttp.ClientSession, url: str, headers: Dict,
-                              body: Dict) -> AsyncGenerator[Tuple[bool, int, str], None]:
+                              body: Dict) -> AsyncGenerator[Tuple[bool, int, Any], None]:
         """Process the HTTP request and handle the response.
 
         Args:
@@ -53,7 +53,7 @@ class ApiPluginBase:
             body: The request body
 
         Yields:
-            Tuple[bool, int, str]: (is_error, status_code, response_data)
+            Tuple[bool, int, Any]: (is_error, status_code, response_data)
         """
         raise NotImplementedError
 

evalscope/perf/plugin/api/default_api.py

@@ -18,7 +18,7 @@ class DefaultApiPlugin(ApiPluginBase):
         super().__init__(param)
 
     async def process_request(self, client_session: aiohttp.ClientSession, url: str, headers: Dict,
-                              body: Dict) -> AsyncGenerator[Tuple[bool, int, str], None]:
+                              body: Dict) -> AsyncGenerator[Tuple[bool, int, Any], None]:
         """Process the HTTP request and handle the response.
 
         Args:
@@ -28,7 +28,7 @@ class DefaultApiPlugin(ApiPluginBase):
             body: The request body
 
         Yields:
-            Tuple[bool, int, str]: (is_error, status_code, response_data)
+            Tuple[bool, int, Any]: (is_error, status_code, response_data)
         """
         try:
             headers = {'Content-Type': 'application/json', **headers}
@@ -40,7 +40,7 @@ class DefaultApiPlugin(ApiPluginBase):
             logger.error(f'Error in process_request: {e}')
             yield (True, None, str(e))
 
-    async def _handle_stream(self, response: aiohttp.ClientResponse) -> AsyncGenerator[Tuple[bool, int, str], None]:
+    async def _handle_stream(self, response: aiohttp.ClientResponse) -> AsyncGenerator[Tuple[bool, int, Any], None]:
         """Handle streaming response from server-sent events.
 
         Args:
@@ -71,14 +71,14 @@ class DefaultApiPlugin(ApiPluginBase):
             logger.error(f'Error in _handle_stream: {e}')
             yield True, response.status, str(e)
 
-    async def _handle_response(self, response: aiohttp.ClientResponse) -> AsyncGenerator[Tuple[bool, int, str], None]:
+    async def _handle_response(self, response: aiohttp.ClientResponse) -> AsyncGenerator[Tuple[bool, int, Any], None]:
         """Handle the HTTP response based on content type and status.
 
         Args:
             response: The aiohttp response object
 
         Yields:
-            Tuple[bool, int, str]: (is_error, status_code, response_data)
+            Tuple[bool, int, Any]: (is_error, status_code, response_data)
         """
         response_status = response.status
         response_content_type = response.content_type
@@ -94,7 +94,7 @@ class DefaultApiPlugin(ApiPluginBase):
             # Handle successful response with 'application/json' content type
             elif content_type_json in response_content_type:
                 content = await response.json()
-                yield (False, response_status, json.dumps(content, ensure_ascii=False))
+                yield (False, response_status, content)
             # Handle other successful responses
             else:
                 content = await response.read()
@@ -102,4 +102,4 @@ class DefaultApiPlugin(ApiPluginBase):
         else:
             # error is always in JSON format
             error = await response.json()
-            yield (True, response_status, json.dumps(error, ensure_ascii=False))
+            yield (True, response_status, error)

evalscope/perf/plugin/api/openai_api.py

@@ -1,10 +1,13 @@
 import json
+import math
 import os
+from collections import defaultdict
 from typing import Any, Dict, List, Tuple, Union
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.api.default_api import DefaultApiPlugin
 from evalscope.perf.plugin.registry import register_api
+from evalscope.utils.io_utils import base64_to_PIL
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -113,7 +116,7 @@ class OpenaiPlugin(DefaultApiPlugin):
             return input_tokens, output_tokens
 
         # no usage information in the response, parse the response to get the tokens
-        delta_contents = {}
+        delta_contents = defaultdict(list)
         for response in responses:
             if 'object' in response:
                 self.__process_response_object(response, delta_contents)
@@ -123,41 +126,46 @@ class OpenaiPlugin(DefaultApiPlugin):
         input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
         return input_tokens, output_tokens
 
-    def __process_response_object(self, js, delta_contents):
-        if js['object'] == 'chat.completion':
-            for choice in js['choices']:
+    def __process_response_object(self, response, delta_contents):
+        if not response.get('choices'):
+            return
+        if response['object'] == 'chat.completion':
+            for choice in response['choices']:
                 delta_contents[choice['index']] = [choice['message']['content']]
-        elif js['object'] == 'text_completion':
-            for choice in js['choices']:
-                delta_contents[choice['index']] = [choice['text']]
-        elif js['object'] == 'chat.completion.chunk':
-            for choice in js.get('choices', []):
+        elif response['object'] == 'text_completion':
+            for choice in response['choices']:
+                if 'text' in choice and 'index' in choice:
+                    delta_contents[choice['index']].append(choice['text'])
+        elif response['object'] == 'chat.completion.chunk':
+            for choice in response['choices']:
                 if 'delta' in choice and 'index' in choice:
                     delta = choice['delta']
                     idx = choice['index']
                     if 'content' in delta:
-                        delta_content = delta['content']
-                        delta_contents.setdefault(idx, []).append(delta_content)
+                        delta_contents[idx].append(delta['content'])
 
-    def __process_no_object(self, js, delta_contents):
+    def __process_no_object(self, response, delta_contents):
         #  assume the response is a single choice
-        for choice in js['choices']:
+        if not response.get('choices'):
+            return
+        for choice in response['choices']:
             if 'delta' in choice:
                 delta = choice['delta']
                 idx = choice['index']
                 if 'content' in delta:
-                    delta_content = delta['content']
-                    delta_contents.setdefault(idx, []).append(delta_content)
+                    delta_contents[idx].append(delta['content'])
             else:
                 delta_contents[choice['index']] = [choice['message']['content']]
 
-    def __calculate_tokens_from_content(self, request, delta_contents):
+    def __calculate_tokens_from_content(self, request, content):
         input_tokens = output_tokens = 0
         if self.tokenizer is not None:
-            for idx, choice_contents in delta_contents.items():
+            # Calculate input tokens
+            input_tokens += self._count_input_tokens(request)
+            for idx, choice_contents in content.items():
                 full_response_content = ''.join(choice_contents)
-                input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
-                output_tokens += len(self.tokenizer.encode(full_response_content))
+                # Calculate output tokens
+                output_tokens += self._count_output_tokens(full_response_content)
         else:
             raise ValueError(
                 'Error: Unable to retrieve usage information\n\n'
@@ -171,3 +179,59 @@ class OpenaiPlugin(DefaultApiPlugin):
                 'please open an issue on our GitHub repository https://github.com/modelscope/evalscope .'
             )
         return input_tokens, output_tokens
+
+    def _count_input_tokens(self, request: Dict) -> int:
+        """Count the number of input tokens in the request.
+
+        This method handles different types of requests and calculates tokens for:
+        - Text content in messages or prompts
+        - Images in multimodal messages (converted to patch tokens)
+
+        Args:
+            request (Dict): The request dictionary containing either 'messages' for chat
+                          completion or 'prompt' for text completion.
+
+        Returns:
+            int: The total number of input tokens including text and image tokens.
+        """
+        input_tokens = 0
+        if 'messages' in request:
+            input_content = self.tokenizer.apply_chat_template(
+                request['messages'], tokenize=True, add_generation_prompt=True
+            )
+            input_tokens += len(input_content)
+            # handle image tokens if any
+            for message in request['messages']:
+                content = message.get('content', '')
+                if isinstance(content, str):
+                    continue
+                for cont in content:
+                    if cont['type'] == 'image_url':
+                        try:
+                            # assuming image_url is base64 string
+                            image_base64 = cont['image_url']['url']
+                            image = base64_to_PIL(image_base64)
+                            # Use math.ceil for more accurate token count when image dimensions
+                            # aren't perfectly divisible by patch size
+                            n_patches = (
+                                math.ceil(image.height / self.param.image_patch_size)
+                                * math.ceil(image.width / self.param.image_patch_size)
+                            )
+                            input_tokens += n_patches
+                        except Exception as e:
+                            logger.warning(f'Failed to process image for token counting: {e}')
+                            # Continue processing other content without failing
+        elif 'prompt' in request:
+            input_tokens += len(self.tokenizer.encode(request['prompt'], add_special_tokens=False))
+        return input_tokens
+
+    def _count_output_tokens(self, response: str) -> int:
+        """Count the number of output tokens in the response. Only string response is supported.
+
+        Args:
+            response (str): The API response text.
+
+        Returns:
+            int: The number of output tokens.
+        """
+        return len(self.tokenizer.encode(response, add_special_tokens=False))

evalscope/perf/plugin/datasets/flickr8k.py

@@ -22,7 +22,7 @@ class FlickrDatasetPlugin(DatasetPluginBase):
         for item in dataset:
             pil_image = item['jpg']
             text = item['txt']
-            base64_image = PIL_to_base64(pil_image)
+            base64_image = PIL_to_base64(pil_image, add_header=True)
 
-            message = self.create_message(text=text, image_urls=f'data:image/jpeg;base64,{base64_image}')
+            message = self.create_message(text=text, image_urls=base64_image)
             yield [message]

evalscope/perf/plugin/datasets/kontext_bench.py

@@ -22,7 +22,7 @@ class KontextDatasetPlugin(DatasetPluginBase):
         for item in dataset:
             pil_image = item['image']
             text = item['instruction']
-            base64_image = PIL_to_base64(pil_image)
+            base64_image = PIL_to_base64(pil_image, add_header=True)
 
-            message = self.create_message(text=text, image_urls=f'data:image/jpeg;base64,{base64_image}')
+            message = self.create_message(text=text, image_urls=base64_image)
             yield [message]

evalscope/perf/plugin/datasets/random_vl_dataset.py

@@ -31,7 +31,7 @@ class RandomVLDatasetPlugin(RandomDatasetPlugin):
             # Generate random images based on image_num
             images_b64 = []
             for _ in range(self.image_num):
-                images_b64.append(f'data:image/png;base64,{self._generate_random_image_b64()}')
+                images_b64.append(self._generate_random_image_b64())
 
             message = self.create_message(text=prompt, image_urls=images_b64)
             yield [message]
@@ -77,4 +77,4 @@ class RandomVLDatasetPlugin(RandomDatasetPlugin):
                 draw.line(coords, fill=shape_color, width=random.randint(1, 5))
 
         # Convert to base64
-        return PIL_to_base64(image, format='PNG')
+        return PIL_to_base64(image, format='PNG', add_header=True)

evalscope/perf/utils/benchmark_util.py

@@ -44,8 +44,7 @@ class BenchmarkData:
             api_plugin.parse_responses(self.response_messages, request=self.request)
 
     def update_gpu_usage(self):
-        if check_import('torch'):
-
+        if check_import('torch', raise_warning=False):
             import torch
             total_memory = 0
             for i in range(torch.cuda.device_count()):