evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/models/api/openai_api.py

@@ -1,34 +1,36 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import json
+import requests
 import threading
 import time
 from asyncio import Queue
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+from typing import Dict, List, Optional, Union
 
-import requests
-from typing import Union, List, Optional, Dict
-from concurrent.futures import ThreadPoolExecutor
-from modelscope.utils.logger import get_logger
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
 class OpenaiApi:
 
-    def __init__(self,
-                 model: str,
-                 openai_api_key,
-                 openai_api_base,
-                 logprobs: Optional[bool] = False,
-                 top_logprobs: Optional[int] = None,
-                 max_new_tokens: int = 4096,
-                 temperature: Optional[float] = 0.0,
-                 repetition_penalty: Optional[float] = 1.0,
-                 is_chat: bool = True,
-                 verbose: bool = True,
-                 retry: int = 3,
-                 query_per_second: int = 10,     # TODO
-                 **kwargs):
+    def __init__(
+            self,
+            model: str,
+            openai_api_key,
+            openai_api_base,
+            logprobs: Optional[bool] = False,
+            top_logprobs: Optional[int] = None,
+            max_new_tokens: int = 4096,
+            temperature: Optional[float] = 0.0,
+            repetition_penalty: Optional[float] = 1.0,
+            is_chat: bool = True,
+            verbose: bool = True,
+            retry: int = 3,
+            query_per_second: int = 10,  # TODO
+            **kwargs):
 
         self.temperature = temperature
         self.repetition_penalty = repetition_penalty
@@ -45,14 +47,17 @@ class OpenaiApi:
 
         self.token_bucket = TokenBucket(query_per_second, verbose)
 
-    def generate_simple(self, inputs: Union[List[str]]):
+    def generate_simple(self, inputs: Union[List[str]], num_proc: int = 8):
 
         def process_one(in_data: str):
 
             if self.is_chat:
                 data = dict(
                     model=self.model,
-                    messages=[{'role': 'user', 'content': in_data}],
+                    messages=[{
+                        'role': 'user',
+                        'content': in_data
+                    }],
                     max_tokens=self.max_tokens,
                     n=1,
                     logprobs=self.logprobs,
@@ -72,7 +77,10 @@ class OpenaiApi:
 
             # todo
             openai_api_key = self.openai_api_key or ''
-            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            header = {
+                'Authorization': f'Bearer {openai_api_key}',
+                'content-type': 'application/json',
+            }
             data = json.dumps(data, ensure_ascii=False)
 
             if self.verbose:
@@ -91,14 +99,18 @@ class OpenaiApi:
                 else:
                     return resp['choices'][0]['text'].strip()
 
-        with ThreadPoolExecutor() as executor:
-            results = list(executor.map(process_one, inputs))
+        results = []
+        with ThreadPoolExecutor(max_workers=num_proc) as executor:
+            # Submit all tasks
+            future_to_task = {executor.submit(process_one, input_one): input_one for input_one in inputs}
+
+            # Show progress bar
+            for future in tqdm(as_completed(future_to_task), total=len(inputs)):
+                results.append(future.result())
 
         return results
 
-    def generate(self,
-                 inputs: Union[List[str], List[List]],
-                 **kwargs) -> List[str]:
+    def generate(self, inputs: Union[List[str], List[List]], **kwargs) -> List[str]:
         """
         Generate responses from OpenAI API.
 
@@ -160,13 +172,12 @@ class OpenaiApi:
 
                 def remove_none_val(input_d: dict):
                     return {k: v for k, v in input_d.items() if v is not None}
+
                 data = remove_none_val(data)
 
                 if self.verbose:
                     logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
-                raw_response = requests.post(self.url,
-                                             headers=header,
-                                             data=json.dumps(data, ensure_ascii=False))
+                raw_response = requests.post(self.url, headers=header, data=json.dumps(data, ensure_ascii=False))
 
                 response = raw_response.json()
                 if self.verbose:

evalscope/models/custom/__init__.py

@@ -1,4 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 from evalscope.models.custom.custom_model import *
-

evalscope/models/custom/custom_model.py

@@ -1,7 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from abc import ABC, abstractmethod
-from typing import Any, Union, Dict, List
 import torch
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Union
 
 
 class CustomModel(ABC):
@@ -11,7 +11,7 @@ class CustomModel(ABC):
         self.kwargs = kwargs
 
         if config.get('model_id', None) is None:
-            raise ValueError(f"**Error: model_id is required in config for CustomModel. Got config: {config}")
+            raise ValueError(f'**Error: model_id is required in config for CustomModel. Got config: {config}')
 
     @abstractmethod
     @torch.no_grad()

evalscope/models/dummy_chat_model.py

@@ -2,6 +2,7 @@
 
 import random
 import time
+
 from evalscope.models import ChatBaseModel
 from evalscope.utils.logger import get_logger
 
@@ -32,15 +33,13 @@ class DummyChatModel(ChatBaseModel):
 
         # Build response
         res = {
-            'choices': [
-                {
-                    'index': 0,
-                    'message': {
-                        'content': choice,
-                        'role': 'assistant'
-                    }
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': choice,
+                    'role': 'assistant'
                 }
-            ],
+            }],
             'created': time.time(),
             'model': self.MODEL_ID + '-' + self.REVISION,
             'object': 'chat.completion',

evalscope/models/model_adapter.py

@@ -1,35 +1,25 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 # Copyright (c) EleutherAI, Inc. and its affiliates.
 # flake8: noqa
+import numpy as np
 import os
 import sys
-from typing import List, Any, Union, Dict
-import numpy as np
 import time
+import torch
 from abc import ABC, abstractmethod
 from copy import deepcopy
-
-import torch
+from modelscope import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
 from torch import dtype
+from typing import Any, Dict, List, Union
 
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.constants import DEFAULT_MODEL_CACHE_DIR
 from evalscope.models.custom import CustomModel
-from evalscope.models.template import get_template, StopWordsCriteria
+from evalscope.utils.chat_service import ChatMessage
 from evalscope.utils.logger import get_logger
-from transformers import StoppingCriteriaList
+from evalscope.utils.model_utils import fix_do_sample_warning
 
 logger = get_logger()
 
-# Notes:
-# - modelscope>=1.9.5
-
-
-def get_model_cache_dir(root_cache_dir: str):
-    model_cache_dir = os.path.join(root_cache_dir, 'models')
-    model_cache_dir = os.path.expanduser(model_cache_dir)
-    os.makedirs(model_cache_dir, exist_ok=True)
-    return model_cache_dir
-
 
 class BaseModelAdapter(ABC):
     """
@@ -69,7 +59,7 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
                  torch_dtype: dtype = torch.bfloat16,
                  model_revision: str = None,
                  max_length: int = None,
-                 cache_dir: str = DEFAULT_ROOT_CACHE_DIR,
+                 cache_dir: str = None,
                  **kwargs):
         """
         Args:
@@ -80,11 +70,11 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
             max_length: The max length of input sequence. Default: None.
             **kwargs: Other args.
         """
-        model_cache_dir = get_model_cache_dir(cache_dir)
+        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
 
         self.model_id: str = model_id
         self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        logger.warning(f'**Device: {self.device}')
+        logger.warning(f'Device: {self.device}')
 
         torch_dtype = torch_dtype if torch_dtype is not None else 'auto'
 
@@ -93,31 +83,21 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         model_cfg['device_map'] = device_map
         model_cfg['torch_dtype'] = str(torch_dtype)
 
-        from modelscope.utils.hf_util import AutoModelForCausalLM, AutoTokenizer
-        # from modelscope import snapshot_download
-
-        # try:
-        #     model_dir = snapshot_download(self.model_id, cache_dir=model_cache_dir, local_files_only=True)
-        #     logger.warning('**Use local_files_only to load model **')
-        # except:
-        #     model_dir = snapshot_download(self.model_id,
-        #                                   revision=model_revision,
-        #                                   cache_dir=model_cache_dir, )
-        #     logger.warning('**Load model from ModelScope hub **')
-
-        tokenizer = AutoTokenizer.from_pretrained(self.model_id,    # self.model_id
-                                                  revision=model_revision,
-                                                  trust_remote_code=True,
-                                                  cache_dir=model_cache_dir,)
-
-        model = AutoModelForCausalLM.from_pretrained(self.model_id,  # self.model_id
-                                                     revision=model_revision,
-                                                     device_map=device_map,
-                                                     trust_remote_code=True,
-                                                     torch_dtype=torch_dtype,
-                                                     cache_dir=model_cache_dir,)
-
-        # model.generation_config = GenerationConfig.from_pretrained(model_id, trust_remote_code=True)
+        tokenizer = AutoTokenizer.from_pretrained(
+            self.model_id,  # self.model_id
+            revision=model_revision,
+            trust_remote_code=True,
+            cache_dir=model_cache_dir,
+        )
+
+        model = AutoModelForCausalLM.from_pretrained(
+            self.model_id,  # self.model_id
+            revision=model_revision,
+            device_map=device_map,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            cache_dir=model_cache_dir,
+        )
 
         super().__init__(model=model, tokenizer=tokenizer, model_cfg=model_cfg)
 
@@ -187,18 +167,16 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         if softval.dtype in {torch.bfloat16, torch.float16}:
             softval = softval.to(dtype=torch.float32)
         probs = softval.detach().cpu().numpy()
-        pred: str = multi_choices[int(np.argmax(probs))]        # Format: A or B or C or D
+        pred: str = multi_choices[int(np.argmax(probs))]  # Format: A or B or C or D
 
         res_d = {
-            'choices': [
-                {
-                    'index': 0,
-                    'message': {
-                        'content': pred,
-                        'role': 'assistant'
-                    }
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': pred,
+                    'role': 'assistant'
                 }
-            ],
+            }],
             'created': time.time(),
             'model': self.model_id,
             'object': 'chat.completion',
@@ -226,7 +204,7 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
                  device_map: str = 'auto',
                  torch_dtype: dtype = torch.bfloat16,
                  model_revision: str = None,
-                 cache_dir: str = DEFAULT_ROOT_CACHE_DIR,
+                 cache_dir: str = None,
                  **kwargs):
         """
         Continuation-logits model adapter.
@@ -239,12 +217,13 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
             **kwargs: Other args.
         """
 
-        super().__init__(model_id=model_id,
-                         device_map=device_map,
-                         torch_dtype=torch_dtype,
-                         model_revision=model_revision,
-                         cache_dir=cache_dir,
-                         **kwargs)
+        super().__init__(
+            model_id=model_id,
+            device_map=device_map,
+            torch_dtype=torch_dtype,
+            model_revision=model_revision,
+            cache_dir=cache_dir,
+            **kwargs)
 
     @torch.no_grad()
     def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
@@ -282,15 +261,13 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
         pred_list: list = self.loglikelihood(inputs=inputs['data'], infer_cfg=infer_cfg)
 
         res_d = {
-            'choices': [
-                {
-                    'index': 0,
-                    'message': {
-                        'content': pred_list,
-                        'role': 'assistant'
-                    }
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': pred_list,
+                    'role': 'assistant'
                 }
-            ],
+            }],
             'created': time.time(),
             'model': self.model_id,
             'object': 'chat.completion',
@@ -347,10 +324,10 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
 
     def __init__(self,
                  model_id: str,
-                 model_revision: str,
+                 model_revision: str = 'master',
                  device_map: str = 'auto',
-                 torch_dtype: dtype = torch.float16,
-                 cache_dir: str = DEFAULT_ROOT_CACHE_DIR,
+                 torch_dtype: dtype = 'auto',
+                 cache_dir: str = None,
                  **kwargs):
         """
         Chat completion model adapter. Tasks of chat and generation are supported.
@@ -359,17 +336,18 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
             model_id: The model id on ModelScope, or local model_dir.
             model_revision: The model revision on ModelScope. Default: None.
             device_map: The device map for model inference.
-            torch_dtype: The torch dtype for model inference. Default: torch.float16.
+            torch_dtype: The torch dtype for model inference. Default: 'auto'.
             **kwargs: Other args.
         """
 
         custom_generation_config = kwargs.pop('generation_config', None)
-        model_cache_dir = get_model_cache_dir(root_cache_dir=cache_dir)
+        custom_chat_template = kwargs.pop('chat_template', None)
+        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
 
         self.model_id: str = model_id
         self.model_revision: str = model_revision
         self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-        logger.warning(f'**Device: {self.device}')
+        logger.warning(f'Device: {self.device}')
 
         torch_dtype = torch_dtype if torch_dtype is not None else 'auto'
 
@@ -378,72 +356,47 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         model_cfg['device_map'] = device_map
         model_cfg['torch_dtype'] = str(torch_dtype)
 
-        self.template_type = kwargs.pop('template_type', None)
-        logger.warning(f'**Template type: {self.template_type}')
-
-        from evalscope.models.template import TemplateType
-        if isinstance(self.model_id, str) \
-                and os.path.isdir(os.path.expanduser(self.model_id)) \
-                and self.template_type is None:
-            raise ValueError(f'Please specify the --template-type for local model dir.\n'
-                             f'Available template types: {TemplateType.get_template_name_list()}\n'
-                             f'Refer to `https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md` for more details.')
-
-        from modelscope.utils.hf_util import AutoModelForCausalLM, AutoTokenizer
-        # from modelscope import snapshot_download
-
-        # try:
-        #     model_dir = snapshot_download(self.model_id, cache_dir=model_cache_dir, local_files_only=True)
-        #     logger.warning('**Use local_files_only to load model **')
-        # except:
-        #     model_dir = snapshot_download(self.model_id,
-        #                                   revision=model_revision,
-        #                                   cache_dir=model_cache_dir, )
-        #     logger.warning('**Load model from ModelScope hub **')
-
-        tokenizer = AutoTokenizer.from_pretrained(self.model_id,
-                                                  revision=model_revision,
-                                                  trust_remote_code=True,
-                                                  cache_dir=model_cache_dir,)
-
-        model = AutoModelForCausalLM.from_pretrained(self.model_id,
-                                                     revision=model_revision,
-                                                     device_map=device_map,
-                                                     trust_remote_code=True,
-                                                     torch_dtype=torch_dtype,
-                                                     cache_dir=model_cache_dir,)
-
-        self.origin_tokenizer = deepcopy(tokenizer)
-
-        self.generation_config, self.generation_template = self._parse_generation_config(tokenizer, model)
+        tokenizer = AutoTokenizer.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            trust_remote_code=True,
+            cache_dir=model_cache_dir,
+        )
+
+        model = AutoModelForCausalLM.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            device_map=device_map,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            cache_dir=model_cache_dir,
+        )
+
+        self.generation_config = self._parse_generation_config(tokenizer, model)
 
         if custom_generation_config:
-            logger.info('**Updating generation config ...')
-            self.generation_config.update(**custom_generation_config.to_dict())
-        logger.info(f'**Generation config init: {self.generation_config.to_dict()}')
+            logger.info('Updating generation config ...')
+            self.generation_config.update(**custom_generation_config)
 
-        super().__init__(model=model, tokenizer=self.generation_template.tokenizer, model_cfg=model_cfg)
+        if custom_chat_template:
+            tokenizer.chat_template = custom_chat_template
+            logger.info(f'Using custom chat template: {custom_chat_template}')
 
-    def _parse_generation_config(self, tokenizer, model):
-        from modelscope.utils.hf_util import GenerationConfig
+        super().__init__(model=model, tokenizer=tokenizer, model_cfg=model_cfg)
 
-        generation_config = getattr(model, 'generation_config', GenerationConfig())
+    def _parse_generation_config(self, tokenizer, model):
+        generation_config = getattr(model, 'generation_config', GenerationConfig(do_sample=False))
 
         try:
             remote_config = GenerationConfig.from_pretrained(
-                self.model_id,
-                revision=self.model_revision,
-                trust_remote_code=True)
+                self.model_id, revision=self.model_revision, trust_remote_code=True)
             generation_config.update(**remote_config.to_dict())
         except:
             logger.warning(f'Failed to get generation config of {self.model_id} from model hub, use default.')
 
-        # Parse templates for chat-completion
         if isinstance(self.model_id, str) and os.path.exists(self.model_id):
             logger.warning(f'Got local model dir: {self.model_id}')
 
-        generation_template = get_template(template_type=self.template_type, tokenizer=tokenizer)
-
         if tokenizer.eos_token_id is not None:
             generation_config.eos_token_id = tokenizer.eos_token_id
         if tokenizer.pad_token_id is not None:
@@ -451,24 +404,19 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         if generation_config.max_new_tokens is None:
             generation_config.max_new_tokens = 2048
 
-        return generation_config, generation_template
+        return generation_config
 
     def _model_generate(self, query: str, infer_cfg: dict) -> str:
-        example = dict(query=query,
-                       history=[],
-                       system=None)
-
-        inputs, _ = self.generation_template.encode(example)
+        messages = [ChatMessage(role='user', content=query)]
+        formatted_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
         input_ids = inputs['input_ids']
-        input_ids = torch.tensor(input_ids)[None].to(self.device)
-        attention_mask = torch.ones_like(input_ids).to(self.device)
 
         # Process infer_cfg
-        infer_cfg = infer_cfg or {}
         if isinstance(infer_cfg.get('num_return_sequences'), int) and infer_cfg['num_return_sequences'] > 1:
             infer_cfg['do_sample'] = True
 
-        # TODO: stop settings
+        # stop settings
         stop = infer_cfg.get('stop', None)
         eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0] \
             if stop else self.tokenizer.eos_token_id
@@ -478,25 +426,16 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
             infer_cfg['pad_token_id'] = eos_token_id  # setting eos_token_id as pad token
 
         self.generation_config.update(**infer_cfg)
-
-        # stopping
-        stop_words = [self.generation_template.suffix[-1]]
-        decode_kwargs = {}
-        stopping_criteria = StoppingCriteriaList(
-            [StopWordsCriteria(self.tokenizer, stop_words, **decode_kwargs)])
+        fix_do_sample_warning(self.generation_config)
 
         # Run inference
-        output_ids = self.model.generate(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            generation_config=self.generation_config,
-            stopping_criteria=stopping_criteria, )
+        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
 
-        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], True, **decode_kwargs)
+        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
         return response
 
     @torch.no_grad()
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = dict({})) -> dict:
+    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = {}) -> dict:
 
         # Process inputs
         if isinstance(inputs, str):
@@ -510,12 +449,7 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
 
         response = self._model_generate(query, infer_cfg)
 
-        choices_list = [
-            {'index': 0,
-             'message': {'content': response,
-                         'role': 'assistant'}
-             }
-        ]
+        choices_list = [{'index': 0, 'message': {'content': response, 'role': 'assistant'}}]
 
         res_d = {
             'choices': choices_list,
@@ -589,4 +523,3 @@ class CustomModelAdapter(BaseModelAdapter):
                 raise TypeError(f'Unsupported inputs type: {type(input_prompt)}')
 
         return self.custom_model.predict(prompts=in_prompts, **kwargs)
-

evalscope/models/openai_model.py

@@ -1,10 +1,9 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import openai
 import os
 import time
 
-import openai
-
 from evalscope.models import ChatBaseModel
 from evalscope.utils.logger import get_logger
 
@@ -43,22 +42,25 @@ class OpenAIModel(ChatBaseModel):
 
         logger.info(f'Using OpenAI model_id: {model_id}')
 
-        res = self._predict(model_id=model_id,
-                            sys_prompt=sys_prompt,
-                            user_prompt=user_prompt,
-                            temperature=temperature,
-                            max_tokens=max_tokens,
-                            mode=mode)
+        res = self._predict(
+            model_id=model_id,
+            sys_prompt=sys_prompt,
+            user_prompt=user_prompt,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            mode=mode)
 
         return res
 
-    def _predict(self,
-                 model_id,
-                 sys_prompt,
-                 user_prompt,
-                 temperature,
-                 max_tokens,
-                 mode: str = 'chat.completion',) -> dict:
+    def _predict(
+        self,
+        model_id,
+        sys_prompt,
+        user_prompt,
+        temperature,
+        max_tokens,
+        mode: str = 'chat.completion',
+    ) -> dict:
 
         res = {}
         openai.api_key = self.api_key
@@ -82,9 +84,8 @@ class OpenAIModel(ChatBaseModel):
                         ans_text = resp['choices'][0]['message']['content']
                         model_id = resp['model']
                     else:
-                        logger.warning(
-                            f'OpenAI GPT API call failed: got empty response '
-                            f'for input {sys_prompt} {user_prompt}')
+                        logger.warning(f'OpenAI GPT API call failed: got empty response '
+                                       f'for input {sys_prompt} {user_prompt}')
                         ans_text = ''
                         model_id = ''
 
@@ -98,6 +99,5 @@ class OpenAIModel(ChatBaseModel):
             except Exception as e:
                 logger.warning(f'OpenAI API call failed: {e}')
                 time.sleep(3)
-        logger.error(
-            f'OpenAI API call failed after {self.MAX_RETRIES} retries')
+        logger.error(f'OpenAI API call failed after {self.MAX_RETRIES} retries')
         return res