evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/perf/plugin/api/openai_api.py

@@ -1,35 +1,38 @@
 import json
+import math
 import os
-from typing import Any, Dict, Iterator, List, Union
+from collections import defaultdict
+from typing import Any, Dict, List, Tuple, Union
 
 from evalscope.perf.arguments import Arguments
-from evalscope.perf.plugin.api.base import ApiPluginBase
+from evalscope.perf.plugin.api.default_api import DefaultApiPlugin
 from evalscope.perf.plugin.registry import register_api
+from evalscope.utils.io_utils import base64_to_PIL
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
 @register_api(['openai', 'local_vllm', 'local'])
-class OpenaiPlugin(ApiPluginBase):
+class OpenaiPlugin(DefaultApiPlugin):
     """Base of openai interface."""
 
-    def __init__(self, mode_path: str):
-        """Init the plugin
+    def __init__(self, param: Arguments):
+        """Initialize the OpenaiPlugin.
 
         Args:
-            mode_path (str): The model path, we use the tokenizer
-                weight in the model to calculate the number of the
-                input and output tokens.
+            param (Arguments): Configuration object containing parameters
+                such as the tokenizer path and model details. If a tokenizer
+                path is provided, it is used to initialize the tokenizer.
         """
-        super().__init__(model_path=mode_path)
-        if mode_path is not None:
-            from transformers import AutoTokenizer
-            self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
+        super().__init__(param=param)
+        if param.tokenizer_path is not None:
+            from modelscope import AutoTokenizer
+            self.tokenizer = AutoTokenizer.from_pretrained(param.tokenizer_path)
         else:
             self.tokenizer = None
 
-    def build_request(self, messages: Union[List[Dict], str], param: Arguments) -> Dict:
+    def build_request(self, messages: Union[List[Dict], str], param: Arguments = None) -> Dict:
         """Build the openai format request based on prompt, dataset
 
         Args:
@@ -42,6 +45,7 @@ class OpenaiPlugin(ApiPluginBase):
         Returns:
             Dict: The request body. None if prompt format is error.
         """
+        param = param or self.param
         try:
             if param.query_template is not None:
                 if param.query_template.startswith('@'):
@@ -54,8 +58,6 @@ class OpenaiPlugin(ApiPluginBase):
                 else:
                     query = json.loads(param.query_template)
 
-                if 'stream' in query.keys():
-                    param.stream = query['stream']
                 # replace template messages with input messages.
                 query['messages'] = messages
             elif isinstance(messages, str):
@@ -75,6 +77,8 @@ class OpenaiPlugin(ApiPluginBase):
             payload['min_tokens'] = param.min_tokens
         if param.frequency_penalty is not None:
             payload['frequency_penalty'] = param.frequency_penalty
+        if param.repetition_penalty is not None:
+            payload['repetition_penalty'] = param.repetition_penalty
         if param.logprobs is not None:
             payload['logprobs'] = param.logprobs
         if param.n_choices is not None:
@@ -92,68 +96,143 @@ class OpenaiPlugin(ApiPluginBase):
             payload['temperature'] = param.temperature
         if param.top_p is not None:
             payload['top_p'] = param.top_p
+        if param.top_k is not None:
+            payload['top_k'] = param.top_k
+        if param.extra_args is not None:
+            payload.update(param.extra_args)
         return payload
 
-    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
+    def parse_responses(self, responses: List[Dict], request: str = None, **kwargs) -> tuple[int, int]:
         """Parser responses and return number of request and response tokens.
         Only one response for non-stream, multiple responses for stream.
         """
 
         # when stream, the last response is the full usage
         # when non-stream, the last response is the first response
-        last_response_js = json.loads(responses[-1])
+        last_response_js = responses[-1]
         if 'usage' in last_response_js and last_response_js['usage']:
             input_tokens = last_response_js['usage']['prompt_tokens']
             output_tokens = last_response_js['usage']['completion_tokens']
             return input_tokens, output_tokens
 
         # no usage information in the response, parse the response to get the tokens
-        delta_contents = {}
+        delta_contents = defaultdict(list)
         for response in responses:
-            js = json.loads(response)
-            if 'object' in js:
-                self.__process_response_object(js, delta_contents)
+            if 'object' in response:
+                self.__process_response_object(response, delta_contents)
             else:
-                self.__process_no_object(js, delta_contents)
+                self.__process_no_object(response, delta_contents)
 
         input_tokens, output_tokens = self.__calculate_tokens_from_content(request, delta_contents)
         return input_tokens, output_tokens
 
-    def __process_response_object(self, js, delta_contents):
-        if js['object'] == 'chat.completion':
-            for choice in js['choices']:
+    def __process_response_object(self, response, delta_contents):
+        if not response.get('choices'):
+            return
+        if response['object'] == 'chat.completion':
+            for choice in response['choices']:
                 delta_contents[choice['index']] = [choice['message']['content']]
-        elif js['object'] == 'text_completion':
-            for choice in js['choices']:
-                delta_contents[choice['index']] = [choice['text']]
-        elif js['object'] == 'chat.completion.chunk':
-            for choice in js.get('choices', []):
+        elif response['object'] == 'text_completion':
+            for choice in response['choices']:
+                if 'text' in choice and 'index' in choice:
+                    delta_contents[choice['index']].append(choice['text'])
+        elif response['object'] == 'chat.completion.chunk':
+            for choice in response['choices']:
                 if 'delta' in choice and 'index' in choice:
                     delta = choice['delta']
                     idx = choice['index']
                     if 'content' in delta:
-                        delta_content = delta['content']
-                        delta_contents.setdefault(idx, []).append(delta_content)
+                        delta_contents[idx].append(delta['content'])
 
-    def __process_no_object(self, js, delta_contents):
+    def __process_no_object(self, response, delta_contents):
         #  assume the response is a single choice
-        for choice in js['choices']:
+        if not response.get('choices'):
+            return
+        for choice in response['choices']:
             if 'delta' in choice:
                 delta = choice['delta']
                 idx = choice['index']
                 if 'content' in delta:
-                    delta_content = delta['content']
-                    delta_contents.setdefault(idx, []).append(delta_content)
+                    delta_contents[idx].append(delta['content'])
             else:
                 delta_contents[choice['index']] = [choice['message']['content']]
 
-    def __calculate_tokens_from_content(self, request, delta_contents):
+    def __calculate_tokens_from_content(self, request, content):
         input_tokens = output_tokens = 0
         if self.tokenizer is not None:
-            for idx, choice_contents in delta_contents.items():
+            # Calculate input tokens
+            input_tokens += self._count_input_tokens(request)
+            for idx, choice_contents in content.items():
                 full_response_content = ''.join(choice_contents)
-                input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
-                output_tokens += len(self.tokenizer.encode(full_response_content))
+                # Calculate output tokens
+                output_tokens += self._count_output_tokens(full_response_content)
         else:
-            logger.warning('No usage information found. Please specify `--tokenizer-path` to generate usage details.')
+            raise ValueError(
+                'Error: Unable to retrieve usage information\n\n'
+                'This error occurs when:\n'
+                '1. The API response does not contain usage data, AND\n'
+                '2. No tokenizer has been specified or found.\n\n'
+                'To resolve this issue, do ONE of the following:\n'
+                "a) Ensure that the API you're using supports and returns usage information, OR\n"
+                'b) Specify a tokenizer using the `--tokenizer-path` parameter.\n\n'
+                'If you continue to experience issues, '
+                'please open an issue on our GitHub repository https://github.com/modelscope/evalscope .'
+            )
         return input_tokens, output_tokens
+
+    def _count_input_tokens(self, request_str: str) -> int:
+        """Count the number of input tokens in the request.
+
+        This method handles different types of requests and calculates tokens for:
+        - Text content in messages or prompts
+        - Images in multimodal messages (converted to patch tokens)
+
+        Args:
+            request_str (str): The request json str containing either 'messages' for chat
+                          completion or 'prompt' for text completion.
+
+        Returns:
+            int: The total number of input tokens including text and image tokens.
+        """
+        input_tokens = 0
+        request = json.loads(request_str)
+        if 'messages' in request:
+            input_content = self.tokenizer.apply_chat_template(
+                request['messages'], tokenize=True, add_generation_prompt=True
+            )
+            input_tokens += len(input_content)
+            # handle image tokens if any
+            for message in request['messages']:
+                content = message.get('content', '')
+                if isinstance(content, str):
+                    continue
+                for cont in content:
+                    if cont['type'] == 'image_url':
+                        try:
+                            # assuming image_url is base64 string
+                            image_base64 = cont['image_url']['url']
+                            image = base64_to_PIL(image_base64)
+                            # Use math.ceil for more accurate token count when image dimensions
+                            # aren't perfectly divisible by patch size
+                            n_patches = (
+                                math.ceil(image.height / self.param.image_patch_size)
+                                * math.ceil(image.width / self.param.image_patch_size)
+                            )
+                            input_tokens += n_patches
+                        except Exception as e:
+                            logger.warning(f'Failed to process image for token counting: {e}')
+                            # Continue processing other content without failing
+        elif 'prompt' in request:
+            input_tokens += len(self.tokenizer.encode(request['prompt'], add_special_tokens=False))
+        return input_tokens
+
+    def _count_output_tokens(self, response: str) -> int:
+        """Count the number of output tokens in the response. Only string response is supported.
+
+        Args:
+            response (str): The API response text.
+
+        Returns:
+            int: The number of output tokens.
+        """
+        return len(self.tokenizer.encode(response, add_special_tokens=False))

evalscope/perf/plugin/datasets/__init__.py

@@ -1,6 +1,10 @@
-from evalscope.perf.plugin.datasets.custom import CustomDatasetPlugin
-from evalscope.perf.plugin.datasets.flickr8k import FlickrDatasetPlugin
-from evalscope.perf.plugin.datasets.line_by_line import LineByLineDatasetPlugin
-from evalscope.perf.plugin.datasets.longalpaca import LongAlpacaDatasetPlugin
-from evalscope.perf.plugin.datasets.openqa import OpenqaDatasetPlugin
-from evalscope.perf.plugin.datasets.speed_benchmark import SpeedBenchmarkDatasetPlugin, SpeedBenchmarkLongDatasetPlugin
+from .base import DatasetPluginBase
+from .custom import CustomDatasetPlugin
+from .flickr8k import FlickrDatasetPlugin
+from .kontext_bench import KontextDatasetPlugin
+from .line_by_line import LineByLineDatasetPlugin
+from .longalpaca import LongAlpacaDatasetPlugin
+from .openqa import OpenqaDatasetPlugin
+from .random_dataset import RandomDatasetPlugin
+from .random_vl_dataset import RandomVLDatasetPlugin
+from .speed_benchmark import SpeedBenchmarkDatasetPlugin, SpeedBenchmarkLongDatasetPlugin

evalscope/perf/plugin/datasets/base.py

@@ -1,7 +1,7 @@
 import json
 import sys
 from abc import abstractmethod
-from typing import Any, Dict, Iterator, List, Tuple
+from typing import Any, Dict, Iterator, List, Tuple, Union
 
 from evalscope.perf.arguments import Arguments
 
@@ -15,6 +15,11 @@ class DatasetPluginBase:
             dataset_path (str, optional): The input dataset path. Defaults to None.
         """
         self.query_parameters = query_parameters
+        if query_parameters.tokenizer_path:
+            from modelscope import AutoTokenizer
+            self.tokenizer = AutoTokenizer.from_pretrained(query_parameters.tokenizer_path, trust_remote_code=True)
+        else:
+            self.tokenizer = None
 
     def __next__(self):
         for item in self.build_messages():
@@ -64,3 +69,40 @@ class DatasetPluginBase:
         data = json.loads(content)
         for item in data:
             yield item
+
+    def create_message(self, text: str, image_urls: Union[List[str], str] = None, role: str = 'user') -> Dict:
+        """Create a message with text and optional image URLs.
+
+        Args:
+            text (str): The text content of the message.
+            image_urls (List[str], optional): List of image URLs. Defaults to None.
+            role (str, optional): The role of the message sender. Defaults to "user".
+
+        Returns:
+            Dict: A dictionary representing the message.
+        """
+        if image_urls is None:
+            message = {'role': role, 'content': text}
+        else:
+            message = {'role': role, 'content': [{'type': 'text', 'text': text}]}
+            if isinstance(image_urls, str):
+                image_urls = [image_urls]
+            for url in image_urls:
+                message['content'].append({'type': 'image_url', 'image_url': {'url': url}})
+        return message
+
+    def check_prompt_length(self, prompt: str) -> Tuple[bool, int]:
+        """Check if the prompt length is within the specified range.
+
+        Args:
+            prompt (str): The input prompt string.
+
+        Returns:
+            Tuple[bool, int]: A tuple containing a boolean indicating whether the prompt is valid and its length.
+        """
+        if self.tokenizer is None:
+            prompt_length = len(prompt)
+        else:
+            prompt_length = len(self.tokenizer.encode(prompt))
+        is_valid = self.query_parameters.min_prompt_length <= prompt_length <= self.query_parameters.max_prompt_length
+        return is_valid, prompt_length

evalscope/perf/plugin/datasets/custom.py

@@ -16,6 +16,25 @@ class CustomDatasetPlugin(DatasetPluginBase):
     def build_messages(self) -> Iterator[List[Dict]]:
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             prompt = item.strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+            is_valid, _ = self.check_prompt_length(prompt)
+            if is_valid:
+                if self.query_parameters.apply_chat_template:
+                    message = self.create_message(prompt)
+                    yield [message]
+                else:
+                    yield prompt
+
+
+if __name__ == '__main__':
+    from evalscope.perf.arguments import Arguments
+    from evalscope.perf.main import run_perf_benchmark
+
+    args = Arguments(
+        model='qwen2.5-7b-instruct',
+        url='https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions',
+        dataset_path='outputs/perf_data.txt',
+        api_key='EMPTY',
+        dataset='custom',
+    )
+
+    run_perf_benchmark(args)

evalscope/perf/plugin/datasets/flickr8k.py

@@ -1,18 +1,9 @@
-import base64
-from io import BytesIO
-from PIL import Image
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
 from evalscope.perf.plugin.datasets.base import DatasetPluginBase
 from evalscope.perf.plugin.registry import register_dataset
-
-
-def PIL_to_base64(image: Image.Image) -> str:
-    buffered = BytesIO()
-    image.save(buffered, format='JPEG')
-    img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
-    return img_str
+from evalscope.utils.io_utils import PIL_to_base64
 
 
 @register_dataset('flickr8k')
@@ -30,21 +21,8 @@ class FlickrDatasetPlugin(DatasetPluginBase):
 
         for item in dataset:
             pil_image = item['jpg']
-            base64_iamge = PIL_to_base64(pil_image)
+            text = item['txt']
+            base64_image = PIL_to_base64(pil_image, add_header=True)
 
-            yield [{
-                'role':
-                'user',
-                'content': [
-                    {
-                        'type': 'text',
-                        'text': 'Describe the image'
-                    },
-                    {
-                        'type': 'image_url',
-                        'image_url': {
-                            'url': f'data:image/jpeg;base64,{base64_iamge}',
-                        }
-                    },
-                ],
-            }]
+            message = self.create_message(text=text, image_urls=base64_image)
+            yield [message]

evalscope/perf/plugin/datasets/kontext_bench.py

@@ -0,0 +1,28 @@
+from typing import Any, Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.io_utils import PIL_to_base64
+
+
+@register_dataset('kontext_bench')
+class KontextDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    Datasets: https://modelscope.cn/datasets/black-forest-labs/kontext-bench/dataPeview
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        from modelscope.msdatasets import MsDataset
+        dataset = MsDataset.load('black-forest-labs/kontext-bench', subset_name='default', split='test')
+
+        for item in dataset:
+            pil_image = item['image']
+            text = item['instruction']
+            base64_image = PIL_to_base64(pil_image, add_header=True)
+
+            message = self.create_message(text=text, image_urls=base64_image)
+            yield [message]

evalscope/perf/plugin/datasets/line_by_line.py

@@ -17,6 +17,10 @@ class LineByLineDatasetPlugin(DatasetPluginBase):
     def build_messages(self) -> Iterator[List[Dict]]:
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             prompt = item.strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+            is_valid, _ = self.check_prompt_length(prompt)
+            if is_valid:
+                if self.query_parameters.apply_chat_template:
+                    message = self.create_message(prompt)
+                    yield [message]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/longalpaca.py

@@ -22,6 +22,10 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
             ds = self.dataset_json_list(self.query_parameters.dataset_path)
         for item in ds:
             prompt = item['instruction'].strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(
-                    prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]
+            is_valid, _ = self.check_prompt_length(prompt)
+            if is_valid:
+                if self.query_parameters.apply_chat_template:
+                    message = self.create_message(prompt)
+                    yield [message]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/openqa.py

@@ -1,5 +1,5 @@
 import json
-import subprocess
+import os
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -18,20 +18,19 @@ class OpenqaDatasetPlugin(DatasetPluginBase):
 
     def build_messages(self) -> Iterator[List[Dict]]:
         if not self.query_parameters.dataset_path:
-            subprocess.call([
-                'modelscope',
-                'download',
-                '--dataset',
-                'AI-ModelScope/HC3-Chinese',
-                'open_qa.jsonl',
-                '--local_dir',
-                './data',
-            ])
-            self.query_parameters.dataset_path = './data/open_qa.jsonl'
+            from modelscope import dataset_snapshot_download
+
+            file_name = 'open_qa.jsonl'
+            local_path = dataset_snapshot_download('AI-ModelScope/HC3-Chinese', allow_patterns=[file_name])
+            self.query_parameters.dataset_path = os.path.join(local_path, file_name)
 
         for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
             item = json.loads(item)
             prompt = item['question'].strip()
-            if (len(prompt) > self.query_parameters.min_prompt_length
-                    and len(prompt) < self.query_parameters.max_prompt_length):
-                yield [{'role': 'user', 'content': prompt}]
+            is_valid, _ = self.check_prompt_length(prompt)
+            if is_valid:
+                if self.query_parameters.apply_chat_template:
+                    message = self.create_message(prompt)
+                    yield [message]
+                else:
+                    yield prompt

evalscope/perf/plugin/datasets/random_dataset.py

@@ -0,0 +1,67 @@
+import numpy as np
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.base import DatasetPluginBase
+from evalscope.perf.plugin.registry import register_dataset
+
+
+@register_dataset('random')
+class RandomDatasetPlugin(DatasetPluginBase):
+    """Read dataset and return prompt.
+    """
+
+    def __init__(self, query_parameters: Arguments):
+        assert query_parameters.tokenizer_path, 'Tokenizer path is required for random data generation, please provide it with `--tokenizer-path`.'  # noqa: E501
+        super().__init__(query_parameters)
+
+        self.prefix_length = self.query_parameters.prefix_length
+        self.prefix_ids = self.get_random_inputs(self.prefix_length)
+        self.template_len = self.get_template_len()
+        self.number = self.query_parameters.number or 1
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        if self.query_parameters.apply_chat_template:
+            min_prompt_length = self.query_parameters.min_prompt_length - self.template_len
+            max_prompt_length = self.query_parameters.max_prompt_length - self.template_len + 1
+        else:
+            min_prompt_length = self.query_parameters.min_prompt_length
+            max_prompt_length = self.query_parameters.max_prompt_length + 1
+
+        assert min_prompt_length >= 0, f'min_prompt_length should be greater than or equal to the template length {self.template_len}.'  # noqa: E501
+        assert max_prompt_length >= min_prompt_length, 'max_prompt_length should be greater than or equal to min_prompt_length.'  # noqa: E501
+
+        # refer to https://github.com/vllm-project/vllm/blob/ed6e9075d31e32c8548b480a47d1ffb77da1f54c/benchmarks/benchmark_serving.py#L366C1-L399C1  # noqa: E501
+        input_lens = np.random.randint(min_prompt_length, max_prompt_length, size=self.number)
+        offsets = np.random.randint(0, self.tokenizer.vocab_size, size=self.number)
+
+        vocab_size = self.tokenizer.vocab_size
+
+        for i in range(self.number):
+            inner_seq = ((offsets[i] + i + np.arange(input_lens[i])) % vocab_size).tolist()
+            token_sequence = self.prefix_ids + inner_seq
+            prompt = self.tokenizer.decode(token_sequence)
+
+            # After decoding the prompt we have to encode and decode it again.
+            # This is done because in some cases N consecutive tokens
+            # give a string tokenized into != N number of tokens.
+            total_input_len = self.prefix_length + int(input_lens[i])
+            re_encoded_sequence = self.tokenizer.encode(prompt, add_special_tokens=False)[:total_input_len]
+            prompt = self.tokenizer.decode(re_encoded_sequence)
+
+            if self.query_parameters.apply_chat_template:
+                message = self.create_message(prompt)
+                yield [message]
+            else:
+                yield prompt
+
+    def get_random_inputs(self, length: int) -> List[int]:
+        if length <= 0:
+            return []
+        input_ids = np.random.randint(0, self.tokenizer.vocab_size, size=length).tolist()
+        return input_ids
+
+    def get_template_len(self):
+        empty_message = [self.create_message(text='')]
+        template = self.tokenizer.apply_chat_template(empty_message, tokenize=True, add_generation_prompt=True)
+        return len(template)

evalscope/perf/plugin/datasets/random_vl_dataset.py

@@ -0,0 +1,80 @@
+import random
+from PIL import Image, ImageDraw
+from typing import Dict, Iterator, List
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.plugin.datasets.random_dataset import RandomDatasetPlugin
+from evalscope.perf.plugin.registry import register_dataset
+from evalscope.utils.io_utils import PIL_to_base64
+
+
+@register_dataset('random_vl')
+class RandomVLDatasetPlugin(RandomDatasetPlugin):
+    """Random Vision-Language Dataset Plugin for multimodal model stress testing."""
+
+    def __init__(self, query_parameters: Arguments):
+        super().__init__(query_parameters)
+
+        # Vision-language specific parameters
+        self.image_width = query_parameters.image_width
+        self.image_height = query_parameters.image_height
+        self.image_format = query_parameters.image_format
+        self.image_num = query_parameters.image_num
+
+        assert self.image_num > 0, 'image_num must be greater than 0.'
+
+    def build_messages(self) -> Iterator[List[Dict]]:
+        # Reuse parent's message generation logic
+        for messages in super().build_messages():
+            prompt = messages[0]['content'] if isinstance(messages[0], dict) else messages[0]
+
+            # Generate random images based on image_num
+            images_b64 = []
+            for _ in range(self.image_num):
+                images_b64.append(self._generate_random_image_b64())
+
+            message = self.create_message(text=prompt, image_urls=images_b64)
+            yield [message]
+
+    def _generate_random_image_b64(self) -> str:
+        """Generate a random image and return as base64 string."""
+        # Create a random colored image
+        color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+        image = Image.new(self.image_format, (self.image_width, self.image_height), color)
+
+        # Add some random shapes for variety
+        draw = ImageDraw.Draw(image)
+        for _ in range(random.randint(1, 5)):
+            shape_type = random.choice(['rectangle', 'ellipse', 'line'])
+
+            # Generate two random points
+            x1 = random.randint(0, self.image_width - 1)
+            y1 = random.randint(0, self.image_height - 1)
+            x2 = random.randint(0, self.image_width - 1)
+            y2 = random.randint(0, self.image_height - 1)
+
+            # Ensure proper coordinate ordering (x1 <= x2, y1 <= y2)
+            if x1 > x2:
+                x1, x2 = x2, x1
+            if y1 > y2:
+                y1, y2 = y2, y1
+
+            # Ensure we have at least a 1-pixel difference
+            if x1 == x2:
+                x2 = min(x1 + 1, self.image_width - 1)
+            if y1 == y2:
+                y2 = min(y1 + 1, self.image_height - 1)
+
+            coords = [x1, y1, x2, y2]
+
+            shape_color = (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255))
+
+            if shape_type == 'rectangle':
+                draw.rectangle(coords, fill=shape_color)
+            elif shape_type == 'ellipse':
+                draw.ellipse(coords, fill=shape_color)
+            else:
+                draw.line(coords, fill=shape_color, width=random.randint(1, 5))
+
+        # Convert to base64
+        return PIL_to_base64(image, format='PNG', add_header=True)