evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/models/base_adapter.py

@@ -1,52 +0,0 @@
-import torch
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Optional, Union
-
-from evalscope.constants import EvalType
-from evalscope.models.custom import CustomModel
-from evalscope.models.local_model import LocalModel
-
-if TYPE_CHECKING:
-    from evalscope.config import TaskConfig
-
-
-class BaseModelAdapter(ABC):
-
-    def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
-        if model is None:
-            self.model_cfg = kwargs.get('model_cfg', None)
-        elif isinstance(model, LocalModel):
-            self.model = model.model
-            self.model_id = model.model_id
-            self.model_revision = model.model_revision
-            self.device = model.device
-            self.tokenizer = model.tokenizer
-            self.model_cfg = model.model_cfg
-        elif isinstance(model, CustomModel):
-            self.model_cfg = model.config
-        else:
-            raise ValueError(f'Unsupported model type: {type(model)}')
-
-    @abstractmethod
-    @torch.no_grad()
-    def predict(self, *args, **kwargs) -> Any:
-        raise NotImplementedError
-
-
-def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseModelAdapter', base_model: 'LocalModel'):
-    """Initialize the model adapter based on the task configuration."""
-    if task_cfg.dry_run:
-        from evalscope.models.model import DummyChatModel
-        return DummyChatModel(model_cfg=dict())
-    elif task_cfg.eval_type == EvalType.CUSTOM:
-        if not isinstance(task_cfg.model, CustomModel):
-            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
-        from evalscope.models import CustomModelAdapter
-        return CustomModelAdapter(custom_model=task_cfg.model)
-    elif task_cfg.eval_type == EvalType.SERVICE:
-        from evalscope.models import ServerModelAdapter
-        return ServerModelAdapter(
-            api_url=task_cfg.api_url, model_id=task_cfg.model, api_key=task_cfg.api_key, seed=task_cfg.seed)
-    else:
-        return model_adapter_cls(
-            model=base_model, generation_config=task_cfg.generation_config, chat_template=task_cfg.chat_template)

evalscope/models/chat_adapter.py

@@ -1,138 +0,0 @@
-import os
-import time
-import torch
-from typing import Union
-
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.local_model import LocalModel
-from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
-from evalscope.utils.logger import get_logger
-from evalscope.utils.model_utils import fix_do_sample_warning
-
-logger = get_logger()
-
-
-class ChatGenerationModelAdapter(BaseModelAdapter):
-    """
-    Chat generation model adapter.
-    """
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model)
-
-        self.generation_config = self._parse_generation_config(self.tokenizer, self.model)
-
-        custom_generation_config = kwargs.pop('generation_config', None)
-        custom_chat_template = kwargs.pop('chat_template', None)
-
-        if custom_generation_config:
-            logger.info('Updating generation config ...')
-            self.generation_config.update(**custom_generation_config)
-
-        if custom_chat_template:
-            self.tokenizer.chat_template = custom_chat_template
-            logger.info(f'Using custom chat template: {custom_chat_template}')
-
-    def _parse_generation_config(self, tokenizer, model):
-        from modelscope import GenerationConfig
-
-        generation_config = getattr(model, 'generation_config', GenerationConfig(do_sample=False))
-
-        try:
-            remote_config = GenerationConfig.from_pretrained(
-                self.model_id, revision=self.model_revision, trust_remote_code=True)
-            generation_config.update(**remote_config.to_dict())
-        except Exception:
-            logger.warning(f'Failed to get generation config of {self.model_id} from model hub, use default.')
-
-        if isinstance(self.model_id, str) and os.path.exists(self.model_id):
-            logger.warning(f'Got local model dir: {self.model_id}')
-
-        if tokenizer.eos_token_id is not None:
-            generation_config.eos_token_id = tokenizer.eos_token_id
-        if tokenizer.pad_token_id is not None:
-            generation_config.pad_token_id = tokenizer.pad_token_id
-        if generation_config.max_new_tokens is None:
-            generation_config.max_new_tokens = 2048
-
-        return generation_config
-
-    def _model_generate(self, query: str, system_prompt: str = None, infer_cfg: dict = {}) -> str:
-        """
-        Args:
-            query: The input query.
-            system_prompt: The system prompt.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prediction result.
-        """
-        # For chat model, use the chat template to format the input
-        if self.tokenizer.chat_template is not None:
-            messages = [ChatMessage(role='user', content=query)]
-            if system_prompt:
-                messages = [ChatMessage(role='system', content=system_prompt)] + messages
-            formatted_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        else:
-            # For base model, use the query as the input
-            formatted_prompt = query
-
-        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
-        input_ids = inputs['input_ids']
-
-        # Process infer_cfg
-        if isinstance(infer_cfg.get('num_return_sequences'), int) and infer_cfg['num_return_sequences'] > 1:
-            infer_cfg['do_sample'] = True
-
-        # stop settings
-        stop = infer_cfg.get('stop', None)
-        eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0] \
-            if stop else self.tokenizer.eos_token_id
-
-        if eos_token_id is not None:
-            infer_cfg['eos_token_id'] = eos_token_id
-            infer_cfg['pad_token_id'] = eos_token_id  # setting eos_token_id as pad token
-
-        self.generation_config.update(**infer_cfg)
-        fix_do_sample_warning(self.generation_config)
-
-        # Run inference
-        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
-
-        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
-        return response
-
-    @torch.no_grad()
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = {}) -> dict:
-        """
-        Args:
-            inputs: The input data.
-            infer_cfg: The inference configuration.
-        Returns:
-            The prediction result.
-        """
-
-        # Process inputs
-        if isinstance(inputs, str):
-            query = inputs
-            system_prompt = None
-        elif isinstance(inputs, dict):
-            query = inputs['data'][0]
-            system_prompt = inputs.get('system_prompt', None)
-        elif isinstance(inputs, list):
-            query = '\n'.join(inputs)
-            system_prompt = None
-        else:
-            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
-
-        response = self._model_generate(query, system_prompt, infer_cfg)
-
-        choices_list = [
-            ChatCompletionResponseChoice(
-                index=0, message=ChatMessage(content=response, role='assistant'), finish_reason='stop')
-        ]
-
-        res_d = ChatCompletionResponse(
-            model=self.model_id, choices=choices_list, object='chat.completion', created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
-
-        return res_d

evalscope/models/choice_adapter.py

@@ -1,211 +0,0 @@
-import numpy as np
-import time
-import torch
-from typing import List
-
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.local_model import LocalModel
-from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
-
-
-class MultiChoiceModelAdapter(BaseModelAdapter):
-    """ The multi-choice model adapter. """
-
-    _DEFAULT_MAX_LENGTH = 2048
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model)
-
-        self._max_length = kwargs.get('max_length')
-
-    @property
-    def max_length(self):
-        if self._max_length:
-            return self._max_length
-        seqlen_config_attrs = ('n_positions', 'max_position_embeddings', 'n_ctx')
-        for attr in seqlen_config_attrs:
-            if hasattr(self.model.config, attr):
-                return getattr(self.model.config, attr)
-        if hasattr(self.tokenizer, 'model_max_length'):
-            if self.tokenizer.model_max_length == 1000000000000000019884624838656:
-                return self._DEFAULT_MAX_LENGTH
-            return self.tokenizer.model_max_length
-        return self._DEFAULT_MAX_LENGTH
-
-    @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
-        """
-        Multi-choice model prediction func.
-
-        Args:
-            inputs (dict): The inputs for a doc. Format:
-                {'data': [full_prompt], 'multi_choices': ['A', 'B', 'C', 'D']}
-
-            infer_cfg (dict): inference configuration.
-
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
-              'model': 'gpt-3.5-turbo-0613',
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        infer_cfg = infer_cfg or {}
-        self.model.generation_config.update(**infer_cfg)
-
-        input_data = inputs['data']
-        multi_choices = inputs['multi_choices']
-
-        output, input_info = self._get_logits(self.tokenizer, self.model, input_data)
-        assert output.shape[0] == 1
-        logits = output.flatten()
-
-        choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in multi_choices]
-        softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
-
-        if softval.dtype in {torch.bfloat16, torch.float16}:
-            softval = softval.to(dtype=torch.float32)
-        probs = softval.detach().cpu().numpy()
-        pred: str = multi_choices[int(np.argmax(probs))]  # Format: A or B or C or D
-
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[
-                ChatCompletionResponseChoice(
-                    index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
-            ],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
-
-        return res_d
-
-    @staticmethod
-    def _get_logits(tokenizer, model, inputs: List[str]):
-        input_ids = tokenizer(inputs, padding=False)['input_ids']
-        input_ids = torch.tensor(input_ids, device=model.device)
-        tokens = {'input_ids': input_ids}
-
-        outputs = model(input_ids)['logits']
-        logits = outputs[:, -1, :]
-        log_probs = torch.nn.functional.softmax(logits, dim=-1)
-        return log_probs, {'tokens': tokens}
-
-
-class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
-    """
-    Continuation-logits model adapter.
-    """
-
-    def __init__(self, model: LocalModel, **kwargs):
-        super().__init__(model, **kwargs)
-
-    @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
-        """
-        Multi-choice model prediction func.
-        Args:
-            inputs (dict): The inputs for a doc. Format:
-                {'data': [(context, continuation), ...]}
-            infer_cfg (dict): inference configuration.
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': [-14.9609, -13.6015, ...],  # loglikelihood values for inputs context-continuation pairs.
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              # For models on the ModelScope or HuggingFace, concat model_id and revision with "-".
-              'model': 'gpt-3.5-turbo-0613',
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        infer_cfg = infer_cfg or {}
-
-        pred_list: list = self.loglikelihood(inputs=inputs['data'], infer_cfg=infer_cfg)
-
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[{
-                'index': 0,
-                'message': {
-                    'content': pred_list,
-                    'role': 'assistant'
-                }
-            }],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
-
-        return res_d
-
-    def loglikelihood(self, inputs: list, infer_cfg: dict = None) -> list:
-        self.model.generation_config.update(**infer_cfg)
-        # To predict one doc
-        doc_ele_pred = []
-        for ctx, continuation in inputs:
-
-            # ctx_enc shape: [context_tok_len]  cont_enc shape: [continuation_tok_len]
-            ctx_enc, cont_enc = self._encode_pair(ctx, continuation)
-
-            inputs_tokens = torch.tensor(
-                (ctx_enc.tolist() + cont_enc.tolist())[-(self.max_length + 1):][:-1],
-                dtype=torch.long,
-                device=self.model.device).unsqueeze(0)
-
-            logits = self.model(inputs_tokens)[0]
-            logits = torch.nn.functional.log_softmax(logits.float(), dim=-1)
-
-            logits = logits[:, -len(cont_enc):, :]
-            cont_enc = cont_enc.unsqueeze(0).unsqueeze(-1)
-            logits = torch.gather(logits.cpu(), 2, cont_enc.cpu()).squeeze(-1)
-
-            choice_score = float(logits.sum())
-            doc_ele_pred.append(choice_score)
-
-        # e.g. [-2.3, -9.2, -12.9, 1.1], length=len(choices)
-        return doc_ele_pred
-
-    def _encode_pair(self, context, continuation):
-        n_spaces = len(context) - len(context.rstrip())
-        if n_spaces > 0:
-            continuation = context[-n_spaces:] + continuation
-            context = context[:-n_spaces]
-
-        whole_enc = self.tokenizer(context + continuation, padding=False)['input_ids']
-        whole_enc = torch.tensor(whole_enc, device=self.device)
-
-        context_enc = self.tokenizer(context, padding=False)['input_ids']
-        context_enc = torch.tensor(context_enc, device=self.device)
-
-        context_enc_len = len(context_enc)
-        continuation_enc = whole_enc[context_enc_len:]
-
-        return context_enc, continuation_enc

evalscope/models/custom/__init__.py

@@ -1,3 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-
-from evalscope.models.custom.custom_model import *

evalscope/models/custom/custom_model.py

@@ -1,53 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import torch
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Union
-
-
-class CustomModel(ABC):
-
-    def __init__(self, config: dict, **kwargs):
-        self.config = config
-        self.kwargs = kwargs
-
-        if config.get('model_id', None) is None:
-            raise ValueError(f'**Error: model_id is required in config for CustomModel. Got config: {config}')
-
-    @abstractmethod
-    @torch.no_grad()
-    def predict(self, prompts: List[str], **kwargs) -> List[Dict[str, Any]]:
-        """
-        Model prediction function for batch inputs.
-
-        Args:
-            prompts (str): The input batch of prompts to predict.
-
-            **kwargs: kwargs
-
-        Returns:
-            res (dict): The model prediction results (batch). Format:
-            [
-                {
-                  'choices': [
-                    {
-                      'index': 0,
-                      'message': {
-                        'content': 'xxx',
-                        'role': 'assistant'
-                      }
-                    }
-                  ],
-                  'created': 1677664795,
-                  'model': 'gpt-3.5-turbo-0613',   # should be model_id
-                  'object': 'chat.completion',
-                  'usage': {
-                    'completion_tokens': 17,
-                    'prompt_tokens': 57,
-                    'total_tokens': 74
-                  }
-                }
-            ,
-            ...
-            ]
-        """
-        raise NotImplementedError

evalscope/models/custom/dummy_model.py

@@ -1,63 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-import os
-import time
-
-from evalscope.models.custom import CustomModel
-from evalscope.run import run_task
-from evalscope.utils.io_utils import yaml_to_dict
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-"""
-This script is used to rewrite the evaluation results without re-running the model predictions.
-"""
-
-
-class DummyCustomModel(CustomModel):
-
-    def __init__(self, config: dict, **kwargs):
-        super(DummyCustomModel, self).__init__(config=config, **kwargs)
-
-    def predict(self, prompts: str, **kwargs):
-        # ONLY FOR DUMMY IMPLEMENTATION, DO NOT EDIT OR USE IN PRODUCTION.
-
-        response = 'The answer is C. NOTE: ONLY FOR TEST'
-
-        res_d: dict = {
-            'choices': [{
-                'index': 0,
-                'message': {
-                    # 'content': f'The answer is B. Raw prompt: {prompt}',
-                    'content': response,
-                    'role': 'assistant'
-                }
-            }],
-            'created':
-            time.time(),
-            'model':
-            self.config.get('model_id'),  # should be model_id
-            'object':
-            'chat.completion',
-            'usage': {
-                'completion_tokens': 0,
-                'prompt_tokens': 0,
-                'total_tokens': 0
-            }
-        }
-
-        return [res_d for _ in prompts]
-
-
-if __name__ == '__main__':
-    # step1: 如果outputs做了迁移，需要修改outputs/eval_xxx 中的configs/task_output_config.yaml中的路径配置
-    # step2: 执行此脚本，默认使用use_cache=True，实现免推理对eval结果进行刷新
-
-    swift_model = DummyCustomModel(config={'model_id': 'swift-model-dummy'})
-
-    task_cfg_file = '/path/to/eval_your_model_results/configs/task_output_config.yaml'
-
-    task_cfg_d = yaml_to_dict(task_cfg_file)
-    task_cfg_d.update({'model': swift_model})
-
-    eval_results: dict = run_task(task_cfg=task_cfg_d)
-    print('** Evaluation results finished !\n')

evalscope/models/custom_adapter.py

@@ -1,67 +0,0 @@
-from typing import Any, Dict, List, Union
-
-from evalscope.models.base_adapter import BaseModelAdapter
-from evalscope.models.custom import CustomModel
-
-
-class CustomModelAdapter(BaseModelAdapter):
-
-    def __init__(self, custom_model: CustomModel, **kwargs):
-        """
-        Custom model adapter.
-
-        Args:
-            custom_model: The custom model instance.
-            **kwargs: Other args.
-        """
-        self.custom_model = custom_model
-        super(CustomModelAdapter, self).__init__(model=custom_model)
-
-    def predict(self, inputs: Union[str, dict, list], **kwargs) -> List[Dict[str, Any]]:
-        """
-        Model prediction func.
-
-        Args:
-            inputs (Union[str, dict, list]): The input data. Depending on the specific model.
-                str: 'xxx'
-                dict: {'data': [full_prompt]}
-                list: ['xxx', 'yyy', 'zzz']
-            **kwargs: kwargs
-
-        Returns:
-            res (dict): The model prediction results. Format:
-            {
-              'choices': [
-                {
-                  'index': 0,
-                  'message': {
-                    'content': 'xxx',
-                    'role': 'assistant'
-                  }
-                }
-              ],
-              'created': 1677664795,
-              'model': 'gpt-3.5-turbo-0613',   # should be model_id
-              'object': 'chat.completion',
-              'usage': {
-                'completion_tokens': 17,
-                'prompt_tokens': 57,
-                'total_tokens': 74
-              }
-            }
-        """
-        in_prompts = []
-
-        # Note: here we assume the inputs are all prompts for the benchmark.
-        for input_prompt in inputs:
-            if isinstance(input_prompt, str):
-                in_prompts.append(input_prompt)
-            elif isinstance(input_prompt, dict):
-                # TODO: to be supported for continuation list like truthful_qa
-                in_prompts.append(input_prompt['data'][0])
-            elif isinstance(input_prompt, list):
-                in_prompts.append('\n'.join(input_prompt))
-            else:
-                raise TypeError(f'Unsupported inputs type: {type(input_prompt)}')
-
-        return self.custom_model.predict(prompts=in_prompts, **kwargs)

evalscope/models/local_model.py

@@ -1,74 +0,0 @@
-import torch
-from typing import TYPE_CHECKING, Optional
-
-from evalscope.constants import DEFAULT_MODEL_CACHE_DIR, DEFAULT_MODEL_REVISION, EvalType
-from evalscope.utils.logger import get_logger
-
-if TYPE_CHECKING:
-    from evalscope.config import TaskConfig
-
-logger = get_logger()
-
-
-class LocalModel:
-
-    def __init__(self,
-                 model_id: str,
-                 model_revision: str = DEFAULT_MODEL_REVISION,
-                 device_map: str = 'auto',
-                 torch_dtype: str = 'auto',
-                 cache_dir: str = None,
-                 **kwargs):
-        from modelscope import AutoModelForCausalLM, AutoTokenizer
-
-        model_cache_dir = cache_dir or DEFAULT_MODEL_CACHE_DIR
-
-        if isinstance(torch_dtype, str) and torch_dtype != 'auto':
-            torch_dtype = eval(torch_dtype)
-
-        self.model_id = model_id
-        self.model_revision = model_revision
-        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            self.model_id,
-            revision=model_revision,
-            trust_remote_code=True,
-            cache_dir=model_cache_dir,
-        )
-
-        self.model = AutoModelForCausalLM.from_pretrained(
-            self.model_id,
-            revision=model_revision,
-            device_map=device_map,
-            trust_remote_code=True,
-            torch_dtype=torch_dtype,
-            cache_dir=model_cache_dir,
-        )
-
-        self.model_cfg = {
-            'model_id': model_id,
-            'device_map': device_map,
-            'torch_dtype': str(torch_dtype),
-        }
-
-
-def get_local_model(task_cfg: 'TaskConfig') -> Optional[LocalModel]:
-    """Get the base local model for the task. If the task is not checkpoint-based, return None.
-       Avoids loading model multiple times for different datasets.
-    """
-    if task_cfg.eval_type != EvalType.CHECKPOINT:
-        return None
-    else:
-        device_map = task_cfg.model_args.get('device_map', 'auto')
-        cache_dir = task_cfg.model_args.get('cache_dir', None)
-        model_precision = task_cfg.model_args.get('precision', 'torch.float16')
-        model_revision = task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION)
-
-        base_model = LocalModel(
-            model_id=task_cfg.model,
-            model_revision=model_revision,
-            device_map=device_map,
-            torch_dtype=model_precision,
-            cache_dir=cache_dir)
-        return base_model