evalscope 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

evalscope/models/chat_adapter.py

@@ -1,7 +1,7 @@
 import os
 import time
 import torch
-from typing import Union
+from typing import List, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.models.local_model import LocalModel
@@ -57,82 +57,104 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
 
         return generation_config
 
-    def _model_generate(self, query: str, system_prompt: str = None, infer_cfg: dict = {}) -> str:
+    def _model_generate(self, queries: List[str], system_prompts: List[str] = None, infer_cfg: dict = {}) -> List[str]:
         """
         Args:
-            query: The input query.
-            system_prompt: The system prompt.
+            queries: The input queries.
+            system_prompts: The system prompts.
             infer_cfg: The inference configuration.
         Returns:
-            The prediction result.
+            The prediction results.
         """
-        # For chat model, use the chat template to format the input
-        if self.tokenizer.chat_template is not None:
-            messages = [ChatMessage(role='user', content=query)]
-            if system_prompt:
-                messages = [ChatMessage(role='system', content=system_prompt)] + messages
-            formatted_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
-        else:
-            # For base model, use the query as the input
-            formatted_prompt = query
-
-        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
-        input_ids = inputs['input_ids']
-
         # Process infer_cfg
-        if isinstance(infer_cfg.get('num_return_sequences'), int) and infer_cfg['num_return_sequences'] > 1:
+        num_return_sequences = infer_cfg.get('num_return_sequences', 1)
+        if num_return_sequences > 1:
             infer_cfg['do_sample'] = True
 
         # stop settings
-        stop = infer_cfg.get('stop', None)
-        eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0] \
-            if stop else self.tokenizer.eos_token_id
+        stop = infer_cfg.get('stop', [])
+        if stop:
+            eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0]
+        else:
+            eos_token_id = self.tokenizer.eos_token_id
 
         if eos_token_id is not None:
             infer_cfg['eos_token_id'] = eos_token_id
-            infer_cfg['pad_token_id'] = eos_token_id  # setting eos_token_id as pad token
 
         self.generation_config.update(**infer_cfg)
         fix_do_sample_warning(self.generation_config)
 
+        # For chat model, use the chat template to format the input
+        if self.tokenizer.chat_template is not None:
+            formatted_prompts = []
+            for i, query in enumerate(queries):
+                messages = [ChatMessage(role='user', content=query)]
+                if i < len(system_prompts) and system_prompts[i]:
+                    messages = [ChatMessage(role='system', content=system_prompts[i])] + messages
+                formatted_prompts.append(
+                    self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True))
+        else:
+            # For base model, use the queries as the input
+            formatted_prompts = queries
+
+        logger.debug(f'formatted_prompts: {formatted_prompts}')
+
+        # Get input ids
+        inputs = self.tokenizer(
+            formatted_prompts, return_tensors='pt', padding=True, truncation=True,
+            padding_side='left').to(self.device)  # padding_side='left' is important for chat model
+        input_ids = inputs['input_ids']
+
         # Run inference
         output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
 
-        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
-        return response
+        responses = []
+        for i in range(0, len(output_ids), num_return_sequences):
+            query_responses = []
+            for j in range(num_return_sequences):
+                output = output_ids[i + j]
+                response = self.tokenizer.decode(
+                    output[len(input_ids[i // num_return_sequences]):], skip_special_tokens=True)
+                query_responses.append(response)
+            responses.append(query_responses)
+
+        return responses
 
     @torch.no_grad()
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = {}) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = {}) -> List[dict]:
         """
         Args:
             inputs: The input data.
             infer_cfg: The inference configuration.
         Returns:
-            The prediction result.
+            The prediction results.
         """
 
         # Process inputs
-        if isinstance(inputs, str):
-            query = inputs
-            system_prompt = None
-        elif isinstance(inputs, dict):
-            query = inputs['data'][0]
-            system_prompt = inputs.get('system_prompt', None)
-        elif isinstance(inputs, list):
-            query = '\n'.join(inputs)
-            system_prompt = None
-        else:
-            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+        queries = []
+        system_prompts = []
+
+        for input_item in inputs:
+            queries.append(input_item['data'][0])
+            system_prompts.append(input_item.get('system_prompt', None))
+
+        responses = self._model_generate(queries, system_prompts, infer_cfg)
 
-        response = self._model_generate(query, system_prompt, infer_cfg)
+        results = []
+        for response in responses:
+            choices_list = [
+                ChatCompletionResponseChoice(
+                    index=index, message=ChatMessage(content=one_response, role='assistant'), finish_reason='stop')
+                for index, one_response in enumerate(response)
+            ]
 
-        choices_list = [
-            ChatCompletionResponseChoice(
-                index=0, message=ChatMessage(content=response, role='assistant'), finish_reason='stop')
-        ]
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=choices_list,
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
 
-        res_d = ChatCompletionResponse(
-            model=self.model_id, choices=choices_list, object='chat.completion', created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
+            results.append(res_d)
 
-        return res_d
+        return results

evalscope/models/choice_adapter.py

@@ -33,12 +33,12 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         return self._DEFAULT_MAX_LENGTH
 
     @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
         """
         Multi-choice model prediction func.
 
         Args:
-            inputs (dict): The inputs for a doc. Format:
+            inputs (List[dict]): The inputs for a doc. Format:
                 {'data': [full_prompt], 'multi_choices': ['A', 'B', 'C', 'D']}
 
             infer_cfg (dict): inference configuration.
@@ -69,37 +69,39 @@ class MultiChoiceModelAdapter(BaseModelAdapter):
         infer_cfg = infer_cfg or {}
         self.model.generation_config.update(**infer_cfg)
 
-        input_data = inputs['data']
-        multi_choices = inputs['multi_choices']
+        input_data = [inp['data'][0] for inp in inputs]
+        multi_choices = [inp['multi_choices'] for inp in inputs]
 
-        output, input_info = self._get_logits(self.tokenizer, self.model, input_data)
-        assert output.shape[0] == 1
-        logits = output.flatten()
+        outputs, input_info = self._get_logits(self.tokenizer, self.model, input_data)
 
-        choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in multi_choices]
-        softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
+        results = []
+        for i, (logits, choices) in enumerate(zip(outputs, multi_choices)):
+            choice_logits = [logits[self.tokenizer(ch)['input_ids'][-1:]] for ch in choices]
+            softval = torch.nn.functional.softmax(torch.tensor(choice_logits).float(), dim=0)
 
-        if softval.dtype in {torch.bfloat16, torch.float16}:
-            softval = softval.to(dtype=torch.float32)
-        probs = softval.detach().cpu().numpy()
-        pred: str = multi_choices[int(np.argmax(probs))]  # Format: A or B or C or D
+            if softval.dtype in {torch.bfloat16, torch.float16}:
+                softval = softval.to(dtype=torch.float32)
+            probs = softval.detach().cpu().numpy()
+            pred: str = choices[int(np.argmax(probs))]  # Format: A or B or C or D
 
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[
-                ChatCompletionResponseChoice(
-                    index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
-            ],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=[
+                    ChatCompletionResponseChoice(
+                        index=0, message=ChatMessage(content=pred, role='assistant'), finish_reason='stop')
+                ],
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
 
-        return res_d
+            results.append(res_d)
+
+        return results
 
     @staticmethod
     def _get_logits(tokenizer, model, inputs: List[str]):
-        input_ids = tokenizer(inputs, padding=False)['input_ids']
-        input_ids = torch.tensor(input_ids, device=model.device)
+        input_ids = tokenizer(
+            inputs, padding=True, return_tensors='pt', padding_side='left')['input_ids'].to(model.device)
         tokens = {'input_ids': input_ids}
 
         outputs = model(input_ids)['logits']
@@ -117,11 +119,11 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
         super().__init__(model, **kwargs)
 
     @torch.no_grad()
-    def predict(self, inputs: dict, infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[dict], infer_cfg: dict = None) -> dict:
         """
         Multi-choice model prediction func.
         Args:
-            inputs (dict): The inputs for a doc. Format:
+            inputs (List[dict]): The inputs for a doc. Format:
                 {'data': [(context, continuation), ...]}
             infer_cfg (dict): inference configuration.
         Returns:
@@ -149,24 +151,29 @@ class ContinuationLogitsModelAdapter(MultiChoiceModelAdapter):
         """
         infer_cfg = infer_cfg or {}
 
-        pred_list: list = self.loglikelihood(inputs=inputs['data'], infer_cfg=infer_cfg)
-
-        res_d = ChatCompletionResponse(
-            model=self.model_id,
-            choices=[{
-                'index': 0,
-                'message': {
-                    'content': pred_list,
-                    'role': 'assistant'
-                }
-            }],
-            object='chat.completion',
-            created=int(time.time()),
-            usage=None).model_dump(exclude_unset=True)
-
-        return res_d
-
-    def loglikelihood(self, inputs: list, infer_cfg: dict = None) -> list:
+        pred_list: list = []
+        for inp in inputs:
+            pred_list.append(self.loglikelihood(inputs=inp['data'], infer_cfg=infer_cfg))
+
+        results = []
+        for pred in pred_list:
+            res_d = ChatCompletionResponse(
+                model=self.model_id,
+                choices=[{
+                    'index': 0,
+                    'message': {
+                        'content': pred,
+                        'role': 'assistant'
+                    }
+                }],
+                object='chat.completion',
+                created=int(time.time()),
+                usage=None).model_dump(exclude_unset=True)
+            results.append(res_d)
+
+        return results
+
+    def loglikelihood(self, inputs: List[tuple], infer_cfg: dict = None) -> list:
         self.model.generation_config.update(**infer_cfg)
         # To predict one doc
         doc_ele_pred = []

evalscope/models/custom_adapter.py

@@ -17,12 +17,12 @@ class CustomModelAdapter(BaseModelAdapter):
         self.custom_model = custom_model
         super(CustomModelAdapter, self).__init__(model=custom_model)
 
-    def predict(self, inputs: Union[str, dict, list], **kwargs) -> List[Dict[str, Any]]:
+    def predict(self, inputs: List[Union[str, dict, list]], **kwargs) -> List[Dict[str, Any]]:
         """
         Model prediction func.
 
         Args:
-            inputs (Union[str, dict, list]): The input data. Depending on the specific model.
+            inputs (List[Union[str, dict, list]]): The input data. Depending on the specific model.
                 str: 'xxx'
                 dict: {'data': [full_prompt]}
                 list: ['xxx', 'yyy', 'zzz']

evalscope/models/local_model.py

@@ -37,6 +37,10 @@ class LocalModel:
             cache_dir=model_cache_dir,
         )
 
+        # Fix no padding
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+
         self.model = AutoModelForCausalLM.from_pretrained(
             self.model_id,
             revision=model_revision,

evalscope/models/server_adapter.py

@@ -1,6 +1,6 @@
 import requests
 import time
-from typing import Optional, Union
+from typing import List, Optional, Union
 
 from evalscope.models.base_adapter import BaseModelAdapter
 from evalscope.utils.chat_service import ChatMessage
@@ -28,36 +28,35 @@ class ServerModelAdapter(BaseModelAdapter):
         self.model_cfg = {'api_url': api_url, 'model_id': model_id, 'api_key': api_key}
         super().__init__(model=None, model_cfg=self.model_cfg, **kwargs)
 
-    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = None) -> dict:
+    def predict(self, inputs: List[Union[str, dict, list]], infer_cfg: dict = None) -> List[dict]:
         """
         Model prediction func.
 
         Args:
-            inputs (Union[str, dict, list]): The input data.
+            inputs (List[Union[str, dict, list]]): The input data.
             infer_cfg (dict): Inference configuration.
 
         Returns:
-            res (dict): The model prediction results.
+            res (List[dict]): The model prediction results.
         """
         infer_cfg = infer_cfg or {}
+        results = []
 
-        # Process inputs
-        if isinstance(inputs, str):
-            query = inputs
-            system_prompt = None
-        elif isinstance(inputs, dict):
-            data: list = inputs['data']
-            if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
-                query = '\n'.join(''.join(item) for item in data)
-                system_prompt = inputs.get('system_prompt', None)
-            else:
-                query = data[0]
-                system_prompt = inputs.get('system_prompt', None)
-        elif isinstance(inputs, list):
-            query = '\n'.join(inputs)
-            system_prompt = None
+        for input_item in inputs:
+            response = self.process_single_input(input_item, infer_cfg)
+            results.append(response)
+
+        return results
+
+    def process_single_input(self, input_item: dict, infer_cfg: dict) -> dict:
+        """Process a single input item."""
+        data: list = input_item['data']
+        if isinstance(data[0], tuple):  # for truthful_qa and hellaswag
+            query = '\n'.join(''.join(item) for item in data)
+            system_prompt = input_item.get('system_prompt', None)
         else:
-            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+            query = data[0]
+            system_prompt = input_item.get('system_prompt', None)
 
         content = self.make_request_content(query, system_prompt)
         request_json = self.make_request(content, infer_cfg)
@@ -68,7 +67,7 @@ class ServerModelAdapter(BaseModelAdapter):
         """
         Make request content for API.
         """
-        if system_prompt is not None:
+        if system_prompt:
             messages = [
                 ChatMessage(role='system', content=system_prompt).model_dump(exclude_unset=True),
                 ChatMessage(role='user', content=query).model_dump(exclude_unset=True)
@@ -80,19 +79,14 @@ class ServerModelAdapter(BaseModelAdapter):
     def make_request(self, content: dict, infer_cfg: dict = {}) -> dict:
         """Make request to remote API."""
         # Format request JSON according to OpenAI API format
-        do_sample = infer_cfg.get('do_sample', False)
-        temperature = infer_cfg.get('temperature', 0.0) if do_sample else 0.0
-
-        request_json = {
-            **content, 'model': self.model_id,
-            'max_tokens': infer_cfg.get('max_tokens', 2048),
-            'temperature': temperature,
-            'top_p': infer_cfg.get('top_p', 1.0),
-            'n': infer_cfg.get('num_return_sequences', 1),
-            'stop': infer_cfg.get('stop', None)
-        }
-        if self.seed is not None:
-            request_json['seed'] = self.seed
+        from evalscope.config import DEFAULT_GENERATION_CONFIG
+        if infer_cfg == DEFAULT_GENERATION_CONFIG:
+            infer_cfg = {
+                'max_tokens': 2048,
+                'temperature': 0.0,
+            }
+
+        request_json = {'model': self.model_id, **content, **infer_cfg}
         logger.debug(f'Request to remote API: {request_json}')
         return request_json