evalscope 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/models/local_model.py

@@ -0,0 +1,74 @@
+import torch
+from typing import TYPE_CHECKING, Optional
+
+from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LocalModel:
+
+    def __init__(self,
+                 model_id: str,
+                 model_revision: str = DEFAULT_MODEL_REVISION,
+                 device_map: str = 'auto',
+                 torch_dtype: str = 'auto',
+                 cache_dir: str = None,
+                 **kwargs):
+        from modelscope import AutoModelForCausalLM, AutoTokenizer
+
+        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = eval(torch_dtype)
+
+        self.model_id = model_id
+        self.model_revision = model_revision
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            trust_remote_code=True,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model = AutoModelForCausalLM.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            device_map=device_map,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model_cfg = {
+            'model_id': model_id,
+            'device_map': device_map,
+            'torch_dtype': str(torch_dtype),
+        }
+
+
+def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
+    """Get the base local model for the task. If the task is not checkpoint-based, return None.
+       Avoids loading model multiple times for different datasets.
+    """
+    if task_cfg.eval_type != EvalType.CHECKPOINT:
+        return None
+    else:
+        device_map = task_cfg.model_args.get('device_map', 'auto')
+        cache_dir = task_cfg.model_args.get('cache_dir', None)
+        model_precision = task_cfg.model_args.get('precision', 'torch.float16')
+        model_revision = task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION)
+
+        base_model = LocalModel(
+            model_id=task_cfg.model,
+            model_revision=model_revision,
+            device_map=device_map,
+            torch_dtype=model_precision,
+            cache_dir=cache_dir)
+        return base_model

evalscope/models/model.py

@@ -1,7 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import random
+import time
 from abc import ABC, abstractmethod
 from typing import Any
 
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
 
 class BaseModel(ABC):
 
@@ -86,3 +93,137 @@ class ChatBaseModel(BaseModel):
                 }
         """
         raise NotImplementedError
+
+
+class OpenAIModel(ChatBaseModel):
+    """
+    APIs of OpenAI models.
+    Available models: gpt-3.5-turbo, gpt-4
+    """
+
+    MAX_RETRIES = 3
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        super(OpenAIModel, self).__init__(model_cfg=model_cfg, **kwargs)
+
+        openai_api_key = os.environ.get('OPENAI_API_KEY', None)
+        self.api_key = self.model_cfg.get('api_key', openai_api_key)
+
+        if not self.api_key:
+            logger.error('OpenAI API key is not provided, please set it in environment variable OPENAI_API_KEY')
+            # raise ValueError(
+            #     'OpenAI API key is not provided, '
+            #     'please set it in environment variable OPENAI_API_KEY')
+
+    def predict(self, model_id: str, inputs: dict, **kwargs) -> dict:
+
+        sys_prompt: str = inputs.get('sys_prompt', '')
+        user_prompt: str = inputs.get('user_prompt', '')
+
+        # model_id: str = kwargs.get('model_id', '')
+        temperature: float = kwargs.pop('temperature', 0.2)
+        max_tokens: int = kwargs.pop('max_tokens', 1024)
+        mode: str = kwargs.pop('mode', 'chat.completion')
+
+        logger.info(f'Using OpenAI model_id: {model_id}')
+
+        res = self._predict(
+            model_id=model_id,
+            sys_prompt=sys_prompt,
+            user_prompt=user_prompt,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            mode=mode)
+
+        return res
+
+    def _predict(
+        self,
+        model_id,
+        sys_prompt,
+        user_prompt,
+        temperature,
+        max_tokens,
+        mode: str = 'chat.completion',
+    ) -> dict:
+        import openai
+
+        res = {}
+        openai.api_key = self.api_key
+
+        for i in range(self.MAX_RETRIES):
+            try:
+                if mode == 'chat.completion':
+                    resp = openai.ChatCompletion.create(
+                        model=model_id,
+                        messages=[{
+                            'role': 'system',
+                            'content': sys_prompt
+                        }, {
+                            'role': 'user',
+                            'content': user_prompt
+                        }],
+                        temperature=temperature,
+                        max_tokens=max_tokens)
+
+                    if resp:
+                        ans_text = resp['choices'][0]['message']['content']
+                        model_id = resp['model']
+                    else:
+                        logger.warning(f'OpenAI GPT API call failed: got empty response '
+                                       f'for input {sys_prompt} {user_prompt}')
+                        ans_text = ''
+                        model_id = ''
+
+                    res['ans_text'] = ans_text
+                    res['model_id'] = model_id
+                else:
+                    raise ValueError(f'Invalid mode: {mode}')
+
+                return res
+
+            except Exception as e:
+                logger.warning(f'OpenAI API call failed: {e}')
+                time.sleep(3)
+        logger.error(f'OpenAI API call failed after {self.MAX_RETRIES} retries')
+        return res
+
+
+class DummyChatModel(ChatBaseModel):
+
+    MODEL_ID = 'dummy_chat_model_0801'
+    REVISION = 'v1.0.0'
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        model_cfg['model_id'] = self.MODEL_ID
+        model_cfg['revision'] = self.REVISION
+        super(DummyChatModel, self).__init__(model_cfg=model_cfg)
+
+    def predict(self, inputs: dict, **kwargs) -> dict:
+
+        debug: bool = False
+        if debug:
+            messages = inputs['messages']
+            history = inputs['history']
+
+            logger.info(f'** messages: {messages}')
+            logger.info(f'** history: {history}')
+
+        choice = random.choice(['A', 'B', 'C', 'D'])
+
+        # Build response
+        res = {
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': choice,
+                    'role': 'assistant'
+                }
+            }],
+            'created': time.time(),
+            'model': self.MODEL_ID + '-' + self.REVISION,
+            'object': 'chat.completion',
+            'usage': {}
+        }
+
+        return res

evalscope/models/server_adapter.py

@@ -0,0 +1,111 @@
+import requests
+import time
+from typing import Optional, Union
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.utils.chat_service import ChatMessage
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class ServerModelAdapter(BaseModelAdapter):
+    """
+    Server model adapter to request remote API model and generate results.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        self.api_url = api_url
+        self.model_id = model_id
+        self.api_key = api_key
+        self.seed = kwargs.get('seed', None)
+        self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
+        super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
+
+    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = None) -> dict:
+        """
+        Model prediction func.
+
+        Args:
+            inputs (Union[str, dict, list]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (dict): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+
+        # Process inputs
+        if isinstance(inputs, str):
+            query = inputs
+            system_prompt = None
+        elif isinstance(inputs, dict):
+            data: list = inputs['data']
+            if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+                query = '\n'.join(''.join(item) for item in data)
+                system_prompt = inputs.get('system_prompt', None)
+            else:
+                query = data[0]
+                system_prompt = inputs.get('system_prompt', None)
+        elif isinstance(inputs, list):
+            query = '\n'.join(inputs)
+            system_prompt = None
+        else:
+            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+
+        content = self.make_request_content(query, system_prompt)
+        request_json = self.make_request(content, infer_cfg)
+        response = self.send_request(request_json)
+        return response
+
+    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> dict:
+        """
+        Make request content for API.
+        """
+        if system_prompt is not None:
+            messages = [
+                ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
+                ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
+            ]
+        else:
+            messages = [ChatMessage(role='user', content=query).model_dump(exclude_unset=True)]
+        return {'messages': messages}
+
+    def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
+        """Make request to remote API."""
+        # Format request JSON according to OpenAI API format
+        do_sample = infer_cfg.get('do_sample', False)
+        temperature = infer_cfg.get('temperature', 0.0) if do_sample else 0.0
+
+        request_json = {
+            **content, 'model': self.model_id,
+            'max_tokens': infer_cfg.get('max_tokens', 2048),
+            'temperature': temperature,
+            'top_p': infer_cfg.get('top_p', 1.0),
+            'n': infer_cfg.get('num_return_sequences', 1),
+            'stop': infer_cfg.get('stop', None)
+        }
+        if self.seed is not None:
+            request_json['seed'] = self.seed
+        logger.debug(f'Request to remote API: {request_json}')
+        return request_json
+
+    def send_request(self, request_json: dict, max_retries: int = 3) -> dict:
+        for attempt in range(max_retries):
+            response = requests.post(
+                self.api_url, json=request_json, headers={'Authorization': f'Bearer {self.api_key}'})
+            if response.status_code == 200:
+                response_data = response.json()
+                return response_data
+            logger.warning(f'Failed to request to remote API: {response.status_code} {response.text}')
+            if attempt < max_retries - 1:
+                time.sleep(5)  # Sleep for 5 seconds before retrying
+            else:
+                raise RuntimeError(f'Failed to request to remote API after {max_retries} attempts: '
+                                   f'{response.status_code} {response.text}')

evalscope/perf/__init__.py

@@ -0,0 +1 @@
+from evalscope.perf.main import run_perf_benchmark

evalscope/perf/main.py

@@ -1,5 +1,4 @@
 import asyncio
-import logging
 import os
 import platform
 from argparse import Namespace

evalscope/perf/plugin/api/custom_api.py

@@ -1,5 +1,4 @@
 import json
-from transformers import AutoTokenizer
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -25,6 +24,7 @@ class CustomPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
+            from transformers import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None

evalscope/perf/plugin/api/openai_api.py

@@ -1,6 +1,5 @@
 import json
 import os
-from transformers import AutoTokenizer
 from typing import Any, Dict, Iterator, List, Union
 
 from evalscope.perf.arguments import Arguments
@@ -25,6 +24,7 @@ class OpenaiPlugin(ApiPluginBase):
         """
         super().__init__(model_path=mode_path)
         if mode_path is not None:
+            from transformers import AutoTokenizer
             self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
         else:
             self.tokenizer = None

evalscope/perf/plugin/datasets/flickr8k.py

@@ -1,6 +1,5 @@
 import base64
 from io import BytesIO
-from modelscope.msdatasets import MsDataset
 from PIL import Image
 from typing import Any, Dict, Iterator, List
 
@@ -26,6 +25,7 @@ class FlickrDatasetPlugin(DatasetPluginBase):
         super().__init__(query_parameters)
 
     def build_messages(self) -> Iterator[List[Dict]]:
+        from modelscope.msdatasets import MsDataset
         dataset = MsDataset.load('clip-benchmark/wds_flickr8k', split='test')
 
         for item in dataset:

evalscope/perf/plugin/datasets/longalpaca.py

@@ -1,4 +1,3 @@
-from modelscope import MsDataset
 from typing import Any, Dict, Iterator, List
 
 from evalscope.perf.arguments import Arguments
@@ -17,6 +16,7 @@ class LongAlpacaDatasetPlugin(DatasetPluginBase):
 
     def build_messages(self) -> Iterator[List[Dict]]:
         if not self.query_parameters.dataset_path:
+            from modelscope import MsDataset
             ds = MsDataset.load('AI-ModelScope/LongAlpaca-12k', subset_name='default', split='train')
         else:
             ds = self.dataset_json_list(self.query_parameters.dataset_path)

evalscope/report/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.report.combinator import gen_table, get_data_frame, get_report_list
+from evalscope.report.generator import ReportGenerator
+from evalscope.report.utils import Category, Report, ReportKey, Subset