evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/models/openai_compatible.py

@@ -0,0 +1,123 @@
+import os
+from openai import APIStatusError, BadRequestError, OpenAI, PermissionDeniedError, UnprocessableEntityError
+from openai._types import NOT_GIVEN
+from openai.types.chat import ChatCompletion
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from evalscope.api.messages import ChatMessage
+from evalscope.api.model import ChatCompletionChoice, GenerateConfig, ModelAPI, ModelOutput
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils import get_logger
+from .utils.openai import (
+    chat_choices_from_openai,
+    collect_stream_response,
+    model_output_from_openai,
+    openai_chat_messages,
+    openai_chat_tool_choice,
+    openai_chat_tools,
+    openai_completion_params,
+    openai_handle_bad_request,
+)
+
+logger = get_logger()
+
+
+class OpenAICompatibleAPI(ModelAPI):
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ) -> None:
+
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+        )
+
+        # use service prefix to lookup api_key
+        self.api_key = api_key or os.environ.get('EVALSCOPE_API_KEY', None)
+        assert self.api_key, f'API key for {model_name} not found'
+
+        # use service prefix to lookup base_url
+        self.base_url = base_url or os.environ.get('EVALSCOPE_BASE_URL', None)
+        assert self.base_url, f'Base URL for {model_name} not found'
+
+        # create http client
+        self.client = OpenAI(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            **model_args,
+        )
+
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+        # setup request and response for ModelCall
+        request: Dict[str, Any] = {}
+        response: Dict[str, Any] = {}
+
+        tools, tool_choice, config = self.resolve_tools(tools, tool_choice, config)
+
+        # get completion params (slice off service from model name)
+        completion_params = self.completion_params(
+            config=config,
+            tools=len(tools) > 0,
+        )
+
+        request = dict(
+            messages=openai_chat_messages(input),
+            tools=openai_chat_tools(tools) if len(tools) > 0 else NOT_GIVEN,
+            tool_choice=openai_chat_tool_choice(tool_choice) if len(tools) > 0 else NOT_GIVEN,
+            **completion_params,
+        )
+
+        try:
+            # generate completion and save response for model call
+            completion = self.client.chat.completions.create(**request)
+            # handle streaming response
+            if not isinstance(completion, ChatCompletion):
+                completion = collect_stream_response(completion)
+            response = completion.model_dump()
+            self.on_response(response)
+
+            # return output and call
+            choices = self.chat_choices_from_completion(completion, tools)
+            return model_output_from_openai(completion, choices)
+
+        except (BadRequestError, UnprocessableEntityError, PermissionDeniedError) as ex:
+            return self.handle_bad_request(ex)
+
+    def resolve_tools(self, tools: List[ToolInfo], tool_choice: ToolChoice,
+                      config: GenerateConfig) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
+        """Provides an opportunity for concrete classes to customize tool resolution."""
+        return tools, tool_choice, config
+
+    def completion_params(self, config: GenerateConfig, tools: bool) -> Dict[str, Any]:
+        return openai_completion_params(
+            model=self.model_name,
+            config=config,
+            tools=tools,
+        )
+
+    def on_response(self, response: Dict[str, Any]) -> None:
+        """Hook for subclasses to do custom response handling."""
+        pass
+
+    def chat_choices_from_completion(self, completion: ChatCompletion,
+                                     tools: List[ToolInfo]) -> List[ChatCompletionChoice]:
+        """Hook for subclasses to do custom chat choice processing."""
+        return chat_choices_from_openai(completion, tools)
+
+    def handle_bad_request(self, ex: APIStatusError) -> Union[ModelOutput, Exception]:
+        """Hook for subclasses to do bad request handling"""
+        return openai_handle_bad_request(self.model_name, ex)

evalscope/models/text2image_model.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import importlib
+import time
+import torch
+from logging import getLogger
+from typing import Any, Dict, List, Literal, Optional, Protocol, Tuple, Union, cast
+
+from evalscope.api.messages import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
+from evalscope.api.model import (
+    ChatCompletionChoice,
+    GenerateConfig,
+    Logprob,
+    Logprobs,
+    ModelAPI,
+    ModelOutput,
+    ModelUsage,
+    TopLogprob,
+)
+from evalscope.api.tool import ToolChoice, ToolInfo
+from evalscope.utils.io_utils import PIL_to_base64
+from evalscope.utils.model_utils import get_device
+
+logger = getLogger()
+
+
+class Text2ImageAPI(ModelAPI):
+
+    def __init__(
+        self,
+        model_name: str,
+        base_url: Optional[str] = None,
+        api_key: Optional[str] = None,
+        config: GenerateConfig = GenerateConfig(),
+        **model_args: Any,
+    ):
+        super().__init__(
+            model_name=model_name,
+            base_url=base_url,
+            api_key=api_key,
+            config=config,
+        )
+
+        # collect known model_args (then delete them so we can pass the rest on)
+        def collect_model_arg(name: str) -> Optional[Any]:
+            nonlocal model_args
+            value = model_args.get(name, None)
+            if value is not None:
+                model_args.pop(name)
+            return value
+
+        model_path = collect_model_arg('model_path')
+        torch_dtype = collect_model_arg('precision') or collect_model_arg('torch_dtype')
+        device_map = collect_model_arg('device_map')
+        # torch dtype
+        DTYPE_MAP = {'float16': torch.float16, 'float32': torch.float32, 'bfloat16': torch.bfloat16, 'auto': 'auto'}
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = DTYPE_MAP.get(torch_dtype, torch.float32)
+        self.torch_dtype = torch_dtype
+        self.device = device_map or get_device()
+
+        self.pipeline_cls = collect_model_arg('pipeline_cls')
+        # default to DiffusionPipeline if not specified
+        if self.pipeline_cls is None:
+            if 'flux' in model_name.lower():
+                self.pipeline_cls = 'FluxPipeline'
+            else:
+                self.pipeline_cls = 'DiffusionPipeline'
+
+        model_name_or_path = model_path or model_name
+
+        # from modelscope import pipeline_cls
+        module = getattr(importlib.import_module('modelscope'), self.pipeline_cls)
+        logger.info(f'Loading model {model_name_or_path} with {self.pipeline_cls} ...')
+
+        self.model = module.from_pretrained(
+            model_name_or_path,
+            torch_dtype=self.torch_dtype,
+            **model_args,
+        )
+
+        self.model.to(self.device)
+
+    def generate(
+        self,
+        input: List[ChatMessage],
+        tools: List[ToolInfo],
+        tool_choice: ToolChoice,
+        config: GenerateConfig,
+    ) -> ModelOutput:
+
+        # prepare generator
+        kwargs: Dict[str, Any] = {}
+        if config.height is not None:
+            kwargs['height'] = config.height
+        if config.width is not None:
+            kwargs['width'] = config.width
+        if config.num_inference_steps is not None:
+            kwargs['num_inference_steps'] = config.num_inference_steps
+        if config.guidance_scale is not None:
+            kwargs['guidance_scale'] = config.guidance_scale
+        if config.extra_body is not None:
+            kwargs.update(config.extra_body)
+
+        # assume the first text as prompt
+        prompt = input[0].text
+        # get the first image as output
+        image = self.model(prompt=prompt, **kwargs).images[0]
+
+        image_base64 = PIL_to_base64(image)
+
+        return ModelOutput(
+            model=self.model_name,
+            choices=[ChatCompletionChoice.from_content(content=[ContentImage(image=image_base64)])],
+            time=time.time(),
+        )