evalscope 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/models/choice_adapter.py

@@ -0,0 +1,211 @@
+import numpy as np
+import time
+import torch
+from typing import List
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.models.local_model import LocalModel
+from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
+
+
+class MultiChoiceModelAdapter(BaseModelAdapter):
+    """ The multi-choice model adapter. """
+
+    _DEFAULT_MAX_LENGTH = 2048
+
+    def __init__(self, model: LocalModel, **kwargs):
+        super().__init__(model)
+
+        self._max_length = kwargs.get('max_length')
+
+    @property
+    def max_length(self):
+        if self._max_length:
+            return self._max_length
+        seqlen_config_attrs = ('n_positions', 'max_position_embeddings', 'n_ctx')
+        for attr in seqlen_config_attrs:
+            if hasattr(self.model.config, attr):
+                return getattr(self.model.config, attr)
+        if hasattr(self.tokenizer, 'model_max_length'):
+            if self.tokenizer.model_max_length == 1000000000000000019884624838656:
+                return self._DEFAULT_MAX_LENGTH
+            return self.tokenizer.model_max_length
+        return self._DEFAULT_MAX_LENGTH
+
+    @torch.no_grad()
+    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+        """
+        Multi-choice model prediction func.
+
+        Args:
+            inputs (dict): The inputs for a doc. Format:
+                {'data': [full_prompt], 'multi_choices': ['A', 'B', 'C', 'D']}
+
+            infer_cfg (dict): inference configuration.
+
+        Returns:
+            res (dict): The model prediction results. Format:
+            {
+              'choices': [
+                {
+                  'index': 0,
+                  'message': {
+                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
+                    'role': 'assistant'
+                  }
+                }
+              ],
+              'created': 1677664795,
+              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
+              'model': 'gpt-3.5-turbo-0613',
+              'object': 'chat.completion',
+              'usage': {
+                'completion_tokens': 17,
+                'prompt_tokens': 57,
+                'total_tokens': 74
+              }
+            }
+        """
+        infer_cfg = infer_cfg or {}
+        self.model.generation_config.update(**infer_cfg)
+
+        input_data = inputs['data']
+        multi_choices = inputs['multi_choices']
+
+        output, input_info = self._get_logits(self.tokenizer, self.model, input_data)
+        assert output.shape[0] == 1
+        logits = output.flatten()
+
+        choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in multi_choices]
+        softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
+
+        if softval.dtype in {torch.bfloat16, torch.float16}:
+            softval = softval.to(dtype=torch.float32)
+        probs = softval.detach().cpu().numpy()
+        pred: str = multi_choices[int(np.argmax(probs))]  # Format: A or B or C or D
+
+        res_d = ChatCompletionResponse(
+            model=self.model_id,
+            choices=[
+                ChatCompletionResponseChoice(
+                    index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
+            ],
+            object='chat.completion',
+            created=int(time.time()),
+            usage=None).model_dump(exclude_unset=True)
+
+        return res_d
+
+    @staticmethod
+    def _get_logits(tokenizer, model, inputs: List[str]):
+        input_ids = tokenizer(inputs, padding=False)['input_ids']
+        input_ids = torch.tensor(input_ids, device=model.device)
+        tokens = {'input_ids': input_ids}
+
+        outputs = model(input_ids)['logits']
+        logits = outputs[:, -1, :]
+        log_probs = torch.nn.functional.softmax(logits, dim=-1)
+        return log_probs, {'tokens': tokens}
+
+
+class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
+    """
+    Continuation-logits model adapter.
+    """
+
+    def __init__(self, model: LocalModel, **kwargs):
+        super().__init__(model, **kwargs)
+
+    @torch.no_grad()
+    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+        """
+        Multi-choice model prediction func.
+        Args:
+            inputs (dict): The inputs for a doc. Format:
+                {'data': [(context, continuation), ...]}
+            infer_cfg (dict): inference configuration.
+        Returns:
+            res (dict): The model prediction results. Format:
+            {
+              'choices': [
+                {
+                  'index': 0,
+                  'message': {
+                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
+                    'role': 'assistant'
+                  }
+                }
+              ],
+              'created': 1677664795,
+              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
+              'model': 'gpt-3.5-turbo-0613',
+              'object': 'chat.completion',
+              'usage': {
+                'completion_tokens': 17,
+                'prompt_tokens': 57,
+                'total_tokens': 74
+              }
+            }
+        """
+        infer_cfg = infer_cfg or {}
+
+        pred_list: list = self.loglikelihood(inputs=inputs['data'], infer_cfg=infer_cfg)
+
+        res_d = ChatCompletionResponse(
+            model=self.model_id,
+            choices=[{
+                'index': 0,
+                'message': {
+                    'content': pred_list,
+                    'role': 'assistant'
+                }
+            }],
+            object='chat.completion',
+            created=int(time.time()),
+            usage=None).model_dump(exclude_unset=True)
+
+        return res_d
+
+    def loglikelihood(self, inputs: list, infer_cfg: dict = None) -> list:
+        self.model.generation_config.update(**infer_cfg)
+        # To predict one doc
+        doc_ele_pred = []
+        for ctx, continuation in inputs:
+
+            # ctx_enc shape: [context_tok_len]  cont_enc shape: [continuation_tok_len]
+            ctx_enc, cont_enc = self._encode_pair(ctx, continuation)
+
+            inputs_tokens = torch.tensor(
+                (ctx_enc.tolist() + cont_enc.tolist())[-(self.max_length + 1):][:-1],
+                dtype=torch.long,
+                device=self.model.device).unsqueeze(0)
+
+            logits = self.model(inputs_tokens)[0]
+            logits = torch.nn.functional.log_softmax(logits.float(), dim=-1)
+
+            logits = logits[:, -len(cont_enc):, :]
+            cont_enc = cont_enc.unsqueeze(0).unsqueeze(-1)
+            logits = torch.gather(logits.cpu(), 2, cont_enc.cpu()).squeeze(-1)
+
+            choice_score = float(logits.sum())
+            doc_ele_pred.append(choice_score)
+
+        # e.g. [-2.3, -9.2, -12.9, 1.1], length=len(choices)
+        return doc_ele_pred
+
+    def _encode_pair(self, context, continuation):
+        n_spaces = len(context) - len(context.rstrip())
+        if n_spaces > 0:
+            continuation = context[-n_spaces:] + continuation
+            context = context[:-n_spaces]
+
+        whole_enc = self.tokenizer(context + continuation, padding=False)['input_ids']
+        whole_enc = torch.tensor(whole_enc, device=self.device)
+
+        context_enc = self.tokenizer(context, padding=False)['input_ids']
+        context_enc = torch.tensor(context_enc, device=self.device)
+
+        context_enc_len = len(context_enc)
+        continuation_enc = whole_enc[context_enc_len:]
+
+        return context_enc, continuation_enc

evalscope/models/custom_adapter.py

@@ -0,0 +1,67 @@
+from typing import Any, Dict, List, Union
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.models.custom import CustomModel
+
+
+class CustomModelAdapter(BaseModelAdapter):
+
+    def __init__(self, custom_model: CustomModel, **kwargs):
+        """
+        Custom model adapter.
+
+        Args:
+            custom_model: The custom model instance.
+            **kwargs: Other args.
+        """
+        self.custom_model = custom_model
+        super(CustomModelAdapter, self).__init__(model=custom_model)
+
+    def predict(self, inputs: Union[str, dict, list], **kwargs) -> List[Dict[str, Any]]:
+        """
+        Model prediction func.
+
+        Args:
+            inputs (Union[str, dict, list]): The input data. Depending on the specific model.
+                str: 'xxx'
+                dict: {'data': [full_prompt]}
+                list: ['xxx', 'yyy', 'zzz']
+            **kwargs: kwargs
+
+        Returns:
+            res (dict): The model prediction results. Format:
+            {
+              'choices': [
+                {
+                  'index': 0,
+                  'message': {
+                    'content': 'xxx',
+                    'role': 'assistant'
+                  }
+                }
+              ],
+              'created': 1677664795,
+              'model': 'gpt-3.5-turbo-0613',   # should be model_id
+              'object': 'chat.completion',
+              'usage': {
+                'completion_tokens': 17,
+                'prompt_tokens': 57,
+                'total_tokens': 74
+              }
+            }
+        """
+        in_prompts = []
+
+        # Note: here we assume the inputs are all prompts for the benchmark.
+        for input_prompt in inputs:
+            if isinstance(input_prompt, str):
+                in_prompts.append(input_prompt)
+            elif isinstance(input_prompt, dict):
+                # TODO: to be supported for continuation list like truthful_qa
+                in_prompts.append(input_prompt['data'][0])
+            elif isinstance(input_prompt, list):
+                in_prompts.append('\n'.join(input_prompt))
+            else:
+                raise TypeError(f'Unsupported inputs type: {type(input_prompt)}')
+
+        return self.custom_model.predict(prompts=in_prompts, **kwargs)

evalscope/models/local_model.py

@@ -0,0 +1,74 @@
+import torch
+from typing import TYPE_CHECKING, Optional
+
+from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class LocalModel:
+
+    def __init__(self,
+                 model_id: str,
+                 model_revision: str = DEFAULT_MODEL_REVISION,
+                 device_map: str = 'auto',
+                 torch_dtype: str = 'auto',
+                 cache_dir: str = None,
+                 **kwargs):
+        from modelscope import AutoModelForCausalLM, AutoTokenizer
+
+        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
+
+        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
+            torch_dtype = eval(torch_dtype)
+
+        self.model_id = model_id
+        self.model_revision = model_revision
+        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            trust_remote_code=True,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model = AutoModelForCausalLM.from_pretrained(
+            self.model_id,
+            revision=model_revision,
+            device_map=device_map,
+            trust_remote_code=True,
+            torch_dtype=torch_dtype,
+            cache_dir=model_cache_dir,
+        )
+
+        self.model_cfg = {
+            'model_id': model_id,
+            'device_map': device_map,
+            'torch_dtype': str(torch_dtype),
+        }
+
+
+def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
+    """Get the base local model for the task. If the task is not checkpoint-based, return None.
+       Avoids loading model multiple times for different datasets.
+    """
+    if task_cfg.eval_type != EvalType.CHECKPOINT:
+        return None
+    else:
+        device_map = task_cfg.model_args.get('device_map', 'auto')
+        cache_dir = task_cfg.model_args.get('cache_dir', None)
+        model_precision = task_cfg.model_args.get('precision', 'torch.float16')
+        model_revision = task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION)
+
+        base_model = LocalModel(
+            model_id=task_cfg.model,
+            model_revision=model_revision,
+            device_map=device_map,
+            torch_dtype=model_precision,
+            cache_dir=cache_dir)
+        return base_model

evalscope/models/model.py

@@ -1,7 +1,14 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import random
+import time
 from abc import ABC, abstractmethod
 from typing import Any
 
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
 
 class BaseModel(ABC):
 
@@ -86,3 +93,137 @@ class ChatBaseModel(BaseModel):
                 }
         """
         raise NotImplementedError
+
+
+class OpenAIModel(ChatBaseModel):
+    """
+    APIs of OpenAI models.
+    Available models: gpt-3.5-turbo, gpt-4
+    """
+
+    MAX_RETRIES = 3
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        super(OpenAIModel, self).__init__(model_cfg=model_cfg, **kwargs)
+
+        openai_api_key = os.environ.get('OPENAI_API_KEY', None)
+        self.api_key = self.model_cfg.get('api_key', openai_api_key)
+
+        if not self.api_key:
+            logger.error('OpenAI API key is not provided, please set it in environment variable OPENAI_API_KEY')
+            # raise ValueError(
+            #     'OpenAI API key is not provided, '
+            #     'please set it in environment variable OPENAI_API_KEY')
+
+    def predict(self, model_id: str, inputs: dict, **kwargs) -> dict:
+
+        sys_prompt: str = inputs.get('sys_prompt', '')
+        user_prompt: str = inputs.get('user_prompt', '')
+
+        # model_id: str = kwargs.get('model_id', '')
+        temperature: float = kwargs.pop('temperature', 0.2)
+        max_tokens: int = kwargs.pop('max_tokens', 1024)
+        mode: str = kwargs.pop('mode', 'chat.completion')
+
+        logger.info(f'Using OpenAI model_id: {model_id}')
+
+        res = self._predict(
+            model_id=model_id,
+            sys_prompt=sys_prompt,
+            user_prompt=user_prompt,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            mode=mode)
+
+        return res
+
+    def _predict(
+        self,
+        model_id,
+        sys_prompt,
+        user_prompt,
+        temperature,
+        max_tokens,
+        mode: str = 'chat.completion',
+    ) -> dict:
+        import openai
+
+        res = {}
+        openai.api_key = self.api_key
+
+        for i in range(self.MAX_RETRIES):
+            try:
+                if mode == 'chat.completion':
+                    resp = openai.ChatCompletion.create(
+                        model=model_id,
+                        messages=[{
+                            'role': 'system',
+                            'content': sys_prompt
+                        }, {
+                            'role': 'user',
+                            'content': user_prompt
+                        }],
+                        temperature=temperature,
+                        max_tokens=max_tokens)
+
+                    if resp:
+                        ans_text = resp['choices'][0]['message']['content']
+                        model_id = resp['model']
+                    else:
+                        logger.warning(f'OpenAI GPT API call failed: got empty response '
+                                       f'for input {sys_prompt} {user_prompt}')
+                        ans_text = ''
+                        model_id = ''
+
+                    res['ans_text'] = ans_text
+                    res['model_id'] = model_id
+                else:
+                    raise ValueError(f'Invalid mode: {mode}')
+
+                return res
+
+            except Exception as e:
+                logger.warning(f'OpenAI API call failed: {e}')
+                time.sleep(3)
+        logger.error(f'OpenAI API call failed after {self.MAX_RETRIES} retries')
+        return res
+
+
+class DummyChatModel(ChatBaseModel):
+
+    MODEL_ID = 'dummy_chat_model_0801'
+    REVISION = 'v1.0.0'
+
+    def __init__(self, model_cfg: dict, **kwargs):
+        model_cfg['model_id'] = self.MODEL_ID
+        model_cfg['revision'] = self.REVISION
+        super(DummyChatModel, self).__init__(model_cfg=model_cfg)
+
+    def predict(self, inputs: dict, **kwargs) -> dict:
+
+        debug: bool = False
+        if debug:
+            messages = inputs['messages']
+            history = inputs['history']
+
+            logger.info(f'** messages: {messages}')
+            logger.info(f'** history: {history}')
+
+        choice = random.choice(['A', 'B', 'C', 'D'])
+
+        # Build response
+        res = {
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': choice,
+                    'role': 'assistant'
+                }
+            }],
+            'created': time.time(),
+            'model': self.MODEL_ID + '-' + self.REVISION,
+            'object': 'chat.completion',
+            'usage': {}
+        }
+
+        return res

evalscope/models/server_adapter.py

@@ -0,0 +1,104 @@
+import requests
+import time
+from typing import Optional, Union
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.utils.chat_service import ChatMessage
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class ServerModelAdapter(BaseModelAdapter):
+    """
+    Server model adapter to request remote API model and generate results.
+    """
+
+    def __init__(self, api_url: str, model_id: str, api_key: str = 'EMPTY', **kwargs):
+        """
+        Args:
+            api_url: The URL of the remote API model.
+            model_id: The ID of the remote API model.
+            api_key: The API key of the remote API model.
+        """
+        self.api_url = api_url
+        self.model_id = model_id
+        self.api_key = api_key
+        self.seed = kwargs.get('seed', None)
+        self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
+        super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
+
+    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = None) -> dict:
+        """
+        Model prediction func.
+
+        Args:
+            inputs (Union[str, dict, list]): The input data.
+            infer_cfg (dict): Inference configuration.
+
+        Returns:
+            res (dict): The model prediction results.
+        """
+        infer_cfg = infer_cfg or {}
+
+        # Process inputs
+        if isinstance(inputs, str):
+            query = inputs
+            system_prompt = None
+        elif isinstance(inputs, dict):
+            # TODO: to be supported for continuation list like truthful_qa
+            query = inputs['data'][0]
+            system_prompt = inputs.get('system_prompt', None)
+        elif isinstance(inputs, list):
+            query = '\n'.join(inputs)
+            system_prompt = None
+        else:
+            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+
+        content = self.make_request_content(query, system_prompt)
+        request_json = self.make_request(content, infer_cfg)
+        response = self.send_request(request_json)
+        return response
+
+    def make_request_content(self, query: str, system_prompt: Optional[str] = None) -> dict:
+        """
+        Make request content for API.
+        """
+        if system_prompt is not None:
+            messages = [
+                ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
+                ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
+            ]
+        else:
+            messages = [ChatMessage(role='user', content=query).model_dump(exclude_unset=True)]
+        return {'messages': messages}
+
+    def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
+        """Make request to remote API."""
+        # Format request JSON according to OpenAI API format
+        request_json = {
+            **content, 'model': self.model_id,
+            'max_tokens': infer_cfg.get('max_tokens', 2048),
+            'temperature': infer_cfg.get('temperature', 0.0),
+            'top_p': infer_cfg.get('top_p', 1.0),
+            'n': infer_cfg.get('num_return_sequences', 1),
+            'stop': infer_cfg.get('stop', None)
+        }
+        if self.seed is not None:
+            request_json['seed'] = self.seed
+        logger.debug(f'Request to remote API: {request_json}')
+        return request_json
+
+    def send_request(self, request_json: dict, max_retries: int = 3) -> dict:
+        for attempt in range(max_retries):
+            response = requests.post(
+                self.api_url, json=request_json, headers={'Authorization': f'Bearer {self.api_key}'})
+            if response.status_code == 200:
+                response_data = response.json()
+                return response_data
+            logger.warning(f'Failed to request to remote API: {response.status_code} {response.text}')
+            if attempt < max_retries - 1:
+                time.sleep(5)  # Sleep for 5 seconds before retrying
+            else:
+                raise RuntimeError(f'Failed to request to remote API after {max_retries} attempts: '
+                                   f'{response.status_code} {response.text}')

evalscope/perf/arguments.py

@@ -68,6 +68,7 @@ class Arguments:
             model=args.model,
             attn_implementation=args.attn_implementation,
             url=args.url,
+            port=args.port,
             api_key=args.api_key,
             connect_timeout=args.connect_timeout,
             read_timeout=args.read_timeout,

evalscope/perf/benchmark.py

@@ -157,7 +157,7 @@ async def statistic_benchmark_metric_worker(benchmark_data_queue: asyncio.Queue,
             while not (data_process_completed_event.is_set() and benchmark_data_queue.empty()):
                 try:
                     # Attempt to get benchmark data from the queue with a timeout
-                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=1)
+                    benchmark_data = await asyncio.wait_for(benchmark_data_queue.get(), timeout=0.01)
                     benchmark_data_queue.task_done()
                 except asyncio.TimeoutError:
                     # If timeout, continue to the next iteration

evalscope/perf/main.py

@@ -19,7 +19,9 @@ def run_perf_benchmark(args):
         args = Arguments(**args)
     elif isinstance(args, Namespace):
         args = Arguments.from_args(args)
-    seed_everything(args.seed)
+
+    if args.seed is not None:
+        seed_everything(args.seed)
 
     # Setup logger and output
     args.outputs_dir = get_output_path(args)