evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/models/adapters/chat_adapter.py

@@ -1,207 +0,0 @@
-import os
-import time
-import torch
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from evalscope.constants import OutputType
-from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage, Usage
-from evalscope.utils.logger import get_logger
-from evalscope.utils.model_utils import fix_do_sample_warning
-from ..local_model import LocalModel
-from ..register import register_model_adapter
-from .base_adapter import BaseModelAdapter
-
-logger = get_logger()
-
-
-@register_model_adapter(name=OutputType.GENERATION)
-class ChatGenerationModelAdapter(BaseModelAdapter):
-    """
-    Chat generation model adapter.
-    """
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model)
-
-        self.generation_config = self._parse_generation_config(self.tokenizer, self.model)
-
-        custom_generation_config = kwargs.pop('generation_config', None)
-        custom_chat_template = kwargs.pop('chat_template', None)
-
-        if custom_generation_config:
-            logger.info('Updating generation config ...')
-            self.generation_config.update(**custom_generation_config)
-
-        if custom_chat_template:
-            self.tokenizer.chat_template = custom_chat_template
-            logger.info(f'Using custom chat template: {custom_chat_template}')
-
-    def _parse_generation_config(self, tokenizer, model):
-        from modelscope import GenerationConfig
-
-        generation_config = getattr(model, 'generation_config', GenerationConfig(do_sample=False))
-
-        try:
-            remote_config = GenerationConfig.from_pretrained(
-                self.model_id, revision=self.model_revision, trust_remote_code=True)
-            generation_config.update(**remote_config.to_dict())
-        except Exception:
-            logger.warning(f'Failed to get generation config of {self.model_id} from model hub, use default.')
-
-        if isinstance(self.model_id, str) and os.path.exists(self.model_id):
-            logger.warning(f'Got local model dir: {self.model_id}')
-
-        if tokenizer.eos_token_id is not None:
-            generation_config.eos_token_id = tokenizer.eos_token_id
-        if tokenizer.pad_token_id is not None:
-            generation_config.pad_token_id = tokenizer.pad_token_id
-        if generation_config.max_new_tokens is None:
-            generation_config.max_new_tokens = 2048
-
-        return generation_config
-
-    def _model_generate(self,
-                        formatted_prompts: List[str],
-                        infer_cfg: Dict[str, Any] = None) -> Tuple[List[List[str]], List[int]]:
-        """
-        Args:
-            formatted_prompts: The formatted prompts.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prediction results.
-        """
-        if infer_cfg is None:
-            infer_cfg = {}
-
-        # Process infer_cfg
-        num_return_sequences = infer_cfg.get('num_return_sequences', 1)
-        if num_return_sequences > 1:
-            infer_cfg['do_sample'] = True
-
-        # stop settings
-        stop = infer_cfg.get('stop', [])
-        if stop:
-            eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0]
-        else:
-            eos_token_id = self.tokenizer.eos_token_id
-
-        if eos_token_id is not None:
-            infer_cfg['eos_token_id'] = eos_token_id
-
-        self.generation_config.update(**infer_cfg)
-        fix_do_sample_warning(self.generation_config)
-
-        # Get input ids
-        inputs = self.tokenizer(
-            formatted_prompts, return_tensors='pt', padding=True, truncation=True,
-            padding_side='left').to(self.model.device)  # padding_side='left' is important for chat model
-        input_ids = inputs['input_ids']
-
-        # Run inference
-        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
-
-        # Decode output
-        responses = []
-        input_lengths = [len(self.tokenizer.encode(prompt)) for prompt in formatted_prompts]
-        for i in range(0, len(output_ids), num_return_sequences):
-            query_responses = []
-            for j in range(num_return_sequences):
-                output = output_ids[i + j]
-                response = self.tokenizer.decode(
-                    output[len(input_ids[i // num_return_sequences]):], skip_special_tokens=True)
-                query_responses.append(response)
-            responses.append(query_responses)
-
-        return responses, input_lengths
-
-    def _prepare_inputs(self, inputs: List[dict], infer_cfg: dict = {}) -> List[str]:
-        """
-        Prepare the inputs for the model.
-        Args:
-            inputs: The input data.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prepared inputs and system prompts.
-        """
-        queries = []
-        system_prompts = []
-        message_list = []
-
-        for input_item in inputs:
-            queries.append(input_item['data'][0])
-            system_prompts.append(input_item.get('system_prompt', None))
-            if input_item.get('messages', None):
-                message_list.append(input_item.get('messages', None))
-
-        # For non chat model, use the original queries as the input
-        if self.tokenizer.chat_template is None:
-            return queries
-
-        # For chat model, use the messages as the input
-        # if message_list is None, use the queries as the input
-        if len(message_list) == 0:
-            for i, query in enumerate(queries):
-                messages = [ChatMessage(role='user', content=query)]
-                if i < len(system_prompts) and system_prompts[i]:
-                    messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
-                message_list.append(messages)
-
-        # Format the messages
-        formatted_prompts = []
-        for messages in message_list:
-            # apply chat template
-            chat_template_kwargs = infer_cfg.get('chat_template_kwargs', None)
-            if chat_template_kwargs is not None:
-                prompts = self.tokenizer.apply_chat_template(
-                    messages, tokenize=False, add_generation_prompt=True, **chat_template_kwargs)
-            else:
-                prompts = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-            formatted_prompts.append(prompts)
-
-        logger.debug(f'formatted_prompts: {formatted_prompts}')
-        return formatted_prompts
-
-    @torch.no_grad()
-    def predict(self, inputs: List[dict], infer_cfg: Optional[dict] = {}) -> List[dict]:
-        """
-        Args:
-            inputs: The input data.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prediction results.
-        """
-
-        # Process inputs
-        formatted_prompts = self._prepare_inputs(inputs, infer_cfg)
-
-        # Run inference
-        responses, input_lengths = self._model_generate(formatted_prompts, infer_cfg)
-
-        # Process outputs
-        results = []
-        for response, input_length in zip(responses, input_lengths):
-            choices_list = []
-            completion_tokens = 0
-
-            for index, one_response in enumerate(response):
-                choice = ChatCompletionResponseChoice(
-                    index=index, message=ChatMessage(content=one_response, role='assistant'), finish_reason='stop')
-                choices_list.append(choice)
-
-                completion_tokens += len(self.tokenizer.encode(one_response))
-
-            usage = Usage(
-                prompt_tokens=input_length,
-                completion_tokens=completion_tokens,
-                total_tokens=input_length + completion_tokens)
-
-            res_d = ChatCompletionResponse(
-                model=self.model_id,
-                choices=choices_list,
-                object='chat.completion',
-                created=int(time.time()),
-                usage=usage).model_dump(exclude_unset=True)
-
-            results.append(res_d)
-
-        return results

evalscope/models/adapters/choice_adapter.py

@@ -1,222 +0,0 @@
-import numpy as np
-import time
-import torch
-from typing import List
-
-from evalscope.constants import OutputType
-from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
-from ..local_model import LocalModel
-from ..register import register_model_adapter
-from .base_adapter import BaseModelAdapter
-
-
-@register_model_adapter(name=OutputType.MULTIPLE_CHOICE)
-class MultiChoiceModelAdapter(BaseModelAdapter):
-    """ The multi-choice model adapter. """
-
-    _DEFAULT_MAX_LENGTH = 2048
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model)
-
-        self._max_length = kwargs.get('max_length')
-
-    @property
-    def max_length(self):
-        if self._max_length:
-            return self._max_length
-        seqlen_config_attrs = ('n_positions', 'max_position_embeddings', 'n_ctx')
-        for attr in seqlen_config_attrs:
-            if hasattr(self.model.config, attr):
-                return getattr(self.model.config, attr)
-        if hasattr(self.tokenizer, 'model_max_length'):
-            if self.tokenizer.model_max_length == 1000000000000000019884624838656:
-                return self._DEFAULT_MAX_LENGTH
-            return self.tokenizer.model_max_length
-        return self._DEFAULT_MAX_LENGTH
-
-    @torch.no_grad()
-    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
-        """
-        Multi-choice model prediction func.
-
-        Args:
-            inputs (List[dict]): The inputs for a doc. Format:
-                {'data': [full_prompt], 'multi_choices': ['A', 'B', 'C', 'D']}
-
-            infer_cfg (dict): inference configuration.
-
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
-              'model': 'gpt-3.5-turbo-0613',
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        infer_cfg = infer_cfg or {}
-        self.model.generation_config.update(**infer_cfg)
-
-        input_data = [inp['data'][0] for inp in inputs]
-        multi_choices = [inp['multi_choices'] for inp in inputs]
-
-        outputs, input_info = self._get_logits(self.tokenizer, self.model, input_data)
-
-        results = []
-        for i, (logits, choices) in enumerate(zip(outputs, multi_choices)):
-            choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in choices]
-            softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
-
-            if softval.dtype in {torch.bfloat16, torch.float16}:
-                softval = softval.to(dtype=torch.float32)
-            probs = softval.detach().cpu().numpy()
-            pred: str = choices[int(np.argmax(probs))]  # Format: A or B or C or D
-
-            res_d = ChatCompletionResponse(
-                model=self.model_id,
-                choices=[
-                    ChatCompletionResponseChoice(
-                        index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
-                ],
-                object='chat.completion',
-                created=int(time.time()),
-                usage=None).model_dump(exclude_unset=True)
-
-            results.append(res_d)
-
-        return results
-
-    @staticmethod
-    def _get_logits(tokenizer, model, inputs: List[str]):
-        input_ids = tokenizer(
-            inputs, padding=True, return_tensors='pt', padding_side='left')['input_ids'].to(model.device)
-        tokens = {'input_ids': input_ids}
-
-        outputs = model(input_ids)['logits']
-        logits = outputs[:, -1, :]
-        log_probs = torch.nn.functional.softmax(logits, dim=-1)
-        return log_probs, {'tokens': tokens}
-
-
-@register_model_adapter(name=OutputType.CONTINUOUS)
-class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
-    """
-    Continuation-logits model adapter.
-    """
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model, **kwargs)
-
-    @torch.no_grad()
-    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
-        """
-        Multi-choice model prediction func.
-        Args:
-            inputs (List[dict]): The inputs for a doc. Format:
-                {'data': [(context, continuation), ...]}
-            infer_cfg (dict): inference configuration.
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
-              'model': 'gpt-3.5-turbo-0613',
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        infer_cfg = infer_cfg or {}
-
-        pred_list: list = []
-        for inp in inputs:
-            pred_list.append(self.loglikelihood(inputs=inp['data'], infer_cfg=infer_cfg))
-
-        results = []
-        for pred in pred_list:
-            res_d = ChatCompletionResponse(
-                model=self.model_id,
-                choices=[{
-                    'index': 0,
-                    'message': {
-                        'content': pred,
-                        'role': 'assistant'
-                    }
-                }],
-                object='chat.completion',
-                created=int(time.time()),
-                usage=None).model_dump(exclude_unset=True)
-            results.append(res_d)
-
-        return results
-
-    def loglikelihood(self, inputs: List[tuple], infer_cfg: dict = None) -> list:
-        self.model.generation_config.update(**infer_cfg)
-        # To predict one doc
-        doc_ele_pred = []
-        for ctx, continuation in inputs:
-
-            # ctx_enc shape: [context_tok_len]  cont_enc shape: [continuation_tok_len]
-            ctx_enc, cont_enc = self._encode_pair(ctx, continuation)
-
-            inputs_tokens = torch.tensor(
-                (ctx_enc.tolist() + cont_enc.tolist())[-(self.max_length + 1):][:-1],
-                dtype=torch.long,
-                device=self.model.device).unsqueeze(0)
-
-            logits = self.model(inputs_tokens)[0]
-            logits = torch.nn.functional.log_softmax(logits.float(), dim=-1)
-
-            logits = logits[:, -len(cont_enc):, :]
-            cont_enc = cont_enc.unsqueeze(0).unsqueeze(-1)
-            logits = torch.gather(logits.cpu(), 2, cont_enc.cpu()).squeeze(-1)
-
-            choice_score = float(logits.sum())
-            doc_ele_pred.append(choice_score)
-
-        # e.g. [-2.3, -9.2, -12.9, 1.1], length=len(choices)
-        return doc_ele_pred
-
-    def _encode_pair(self, context, continuation):
-        n_spaces = len(context) - len(context.rstrip())
-        if n_spaces > 0:
-            continuation = context[-n_spaces:] + continuation
-            context = context[:-n_spaces]
-
-        whole_enc = self.tokenizer(context + continuation, padding=False)['input_ids']
-        whole_enc = torch.tensor(whole_enc, device=self.device)
-
-        context_enc = self.tokenizer(context, padding=False)['input_ids']
-        context_enc = torch.tensor(context_enc, device=self.device)
-
-        context_enc_len = len(context_enc)
-        continuation_enc = whole_enc[context_enc_len:]
-
-        return context_enc, continuation_enc

evalscope/models/adapters/custom_adapter.py

@@ -1,71 +0,0 @@
-from typing import TYPE_CHECKING, Any, Dict, List, Union
-
-from ..register import register_model_adapter
-from .base_adapter import BaseModelAdapter
-
-if TYPE_CHECKING:
-    from ..custom import CustomModel
-
-
-@register_model_adapter(name='custom')
-class CustomModelAdapter(BaseModelAdapter):
-
-    def __init__(self, custom_model: 'CustomModel', **kwargs):
-        """
-        Custom model adapter.
-
-        Args:
-            custom_model: The custom model instance.
-            **kwargs: Other args.
-        """
-        self.custom_model = custom_model
-        super(CustomModelAdapter, self).__init__(model=custom_model)
-
-    def predict(self, inputs: List[Union[str, dict, list]], **kwargs) -> List[Dict[str, Any]]:
-        """
-        Model prediction func.
-
-        Args:
-            inputs (List[Union[str, dict, list]]): The input data. Depending on the specific model.
-                str: 'xxx'
-                dict: {'data': [full_prompt]}
-                list: ['xxx', 'yyy', 'zzz']
-            **kwargs: kwargs
-
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': 'xxx',
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              'model': 'gpt-3.5-turbo-0613',   # should be model_id
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        in_prompts = []
-
-        # Note: here we assume the inputs are all prompts for the benchmark.
-        for input_prompt in inputs:
-            if isinstance(input_prompt, str):
-                in_prompts.append(input_prompt)
-            elif isinstance(input_prompt, dict):
-                # TODO: to be supported for continuation list like truthful_qa
-                in_prompts.append(input_prompt['data'][0])
-            elif isinstance(input_prompt, list):
-                in_prompts.append('\n'.join(input_prompt))
-            else:
-                raise TypeError(f'Unsupported inputs type: {type(input_prompt)}')
-
-        return self.custom_model.predict(prompts=in_prompts, origin_inputs=inputs, **kwargs)