evalscope 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

evalscope/backend/opencompass/backend_manager.py

@@ -242,4 +242,6 @@ if __name__ == '__main__':
                 'limit': 5
                 }
     )
+    all_datasets = OpenCompassBackendManager.list_datasets()
+    print(f'all_datasets: {all_datasets}')
     oc_backend_manager.run()

evalscope/backend/opencompass/tasks/eval_datasets.py

@@ -49,6 +49,7 @@ with read_base():
     from opencompass.configs.datasets.obqa.obqa_gen_9069e4 import obqa_datasets
     from opencompass.configs.datasets.nq.nq_gen_c788f6 import nq_datasets
     from opencompass.configs.datasets.triviaqa.triviaqa_gen_2121ce import triviaqa_datasets
+    from opencompass.configs.datasets.cmb.cmb_gen_dfb5c4 import cmb_datasets
     from opencompass.configs.datasets.bbh.bbh_gen_5b92b0 import bbh_datasets
 
     # Note: to be supported

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,5 +1,5 @@
 from typing import Optional, Union
-from evalscope.utils import is_module_installed, get_module_path, get_valid_list, yaml_to_dict, json_to_dict
+from evalscope.utils import is_module_installed, get_valid_list
 from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial
@@ -37,6 +37,7 @@ class VLMEvalKitBackendManager(BackendManager):
 
         self._check_valid()
 
+    
     def _check_valid(self):
         # Ensure not both model and datasets are empty
         if not self.args.data or not self.args.model:
@@ -44,9 +45,9 @@ class VLMEvalKitBackendManager(BackendManager):
 
         # Check datasets
         valid_datasets, invalid_datasets = get_valid_list(self.args.data, self.valid_datasets)
-        assert len(invalid_datasets) == 0, f'Invalid datasets: {invalid_datasets}, ' \
-            f'refer to the following list to get proper dataset name: {self.valid_datasets}'
-
+        if len(invalid_datasets) != 0:
+            logger.warning(f"Using custom dataset: {invalid_datasets}, ")
+            
         # Check model
         if isinstance(self.args.model[0], dict):
             model_names = [model['name'] for model in self.args.model]
@@ -61,10 +62,14 @@ class VLMEvalKitBackendManager(BackendManager):
                 model_class = self.valid_models[model_name]
                 if model_name == 'CustomAPIModel':
                     model_type = model_cfg['type']
+                    remain_cfg = copy.deepcopy(model_cfg)
+                    del remain_cfg['name'] # remove not used args
+                    del remain_cfg['type'] # remove not used args
+
                     self.valid_models.update({
                                 model_type: partial(model_class, 
                                                    model=model_type,
-                                                   **model_cfg)
+                                                   **remain_cfg)
                                 })
                     new_model_names.append(model_type)
                 else:
@@ -78,8 +83,8 @@ class VLMEvalKitBackendManager(BackendManager):
 
         elif isinstance(self.args.model[0], str):
             valid_model_names, invalid_model_names = get_valid_list(self.args.model, self.valid_model_names)
-            assert len(invalid_model_names) == 0, f'Invalid models: {invalid_model_names}, ' \
-                f'refer to the following list to get proper model name: {self.valid_model_names}'
+            if len(invalid_datasets) != 0:
+                logger.warning(f"Using custom dataset: {invalid_datasets}, ")
 
     @property
     def cmd(self):

evalscope/backend/vlm_eval_kit/custom_dataset.py

@@ -0,0 +1,47 @@
+import os
+import numpy as np
+from vlmeval.dataset.image_base import ImageBaseDataset
+from vlmeval.dataset.image_vqa import CustomVQADataset
+from vlmeval.smp import load, dump, d2df
+
+class CustomDataset:
+
+    def load_data(self, dataset):
+        # customize the loading of the dataset
+        data_path = os.path.join("~/LMUData", f'{dataset}.tsv')
+        return load(data_path)
+
+
+    def build_prompt(self, line):
+        msgs = ImageBaseDataset.build_prompt(self, line)
+        # add a hint or custom instruction here
+        msgs[-1]['value'] += '\nAnswer the question using a single word or phrase.'
+        return msgs
+    
+    
+    def evaluate(self, eval_file, **judge_kwargs):
+        data = load(eval_file)
+        assert 'answer' in data and 'prediction' in data
+        data['prediction'] = [str(x) for x in data['prediction']]
+        data['answer'] = [str(x).lower() for x in data['answer']]
+        
+        print(data)
+        
+        # ========compute the evaluation metrics as you need =========
+        # exact match
+        result = np.mean(data['answer'] == data['prediction'])
+        ret = {'Overall': result}
+        ret = d2df(ret).round(2)
+
+        # save the result
+        suffix = eval_file.split('.')[-1]
+        result_file = eval_file.replace(f'.{suffix}', '_acc.csv')
+        dump(ret, result_file)
+        return ret
+        # ============================================================
+
+
+# override the default dataset class
+CustomVQADataset.load_data = CustomDataset.load_data
+CustomVQADataset.build_prompt = CustomDataset.build_prompt
+CustomVQADataset.evaluate = CustomDataset.evaluate

evalscope/benchmarks/benchmark.py

@@ -46,7 +46,7 @@ class Benchmark(object):
 
             dataset.dataset_name = dataset_name.split('/')[-1]
             dataset.subset_name = subset
-            dataset.split = split
+            # dataset.split = split
             return dataset
         elif hub == 'HuggingFace':
             # TODO: implement this by xingjun.wxj@alibaba-inc.com

evalscope/config.py

@@ -33,6 +33,7 @@ registry_tasks = {
 @dataclass
 class TaskConfig:
     model_args: Optional[dict] = field(default_factory=dict)
+    template_type: Optional[str] = 'default-generation'
     generation_config: Optional[dict] = field(default_factory=dict)
     dataset_args: Optional[dict] = field(default_factory=dict)
     dry_run: bool = False

evalscope/evaluator/evaluator.py

@@ -244,8 +244,8 @@ class Evaluator(object):
                 answer_d[AnswerKeys.ORIGIN_PROMPT] = input_prompt
 
                 if debug:
-                    logger.debug(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.debug(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
+                    logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
+                    logger.info(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
 
                 answers_list.append(answer_d)
 
@@ -349,7 +349,7 @@ class Evaluator(object):
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
             if debug:
-                logger.debug(review_d)
+                logger.info(review_d)
 
             reviews_list.append(review_d)
 

evalscope/models/api/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.models.api.openai_api import OpenaiApi

evalscope/models/api/openai_api.py

@@ -0,0 +1,228 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+import threading
+import time
+from asyncio import Queue
+
+import requests
+from typing import Union, List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OpenaiApi:
+
+    def __init__(self,
+                 model: str,
+                 openai_api_key,
+                 openai_api_base,
+                 logprobs: Optional[bool] = False,
+                 top_logprobs: Optional[int] = None,
+                 max_new_tokens: int = 4096,
+                 temperature: Optional[float] = 0.0,
+                 repetition_penalty: Optional[float] = 1.0,
+                 is_chat: bool = True,
+                 verbose: bool = True,
+                 retry: int = 3,
+                 query_per_second: int = 10,     # TODO
+                 **kwargs):
+
+        self.temperature = temperature
+        self.repetition_penalty = repetition_penalty
+        self.max_tokens = max_new_tokens
+        self.logprobs = logprobs
+        self.top_logprobs = top_logprobs
+
+        self.openai_api_key = openai_api_key
+        self.url = openai_api_base
+        self.model = model
+        self.is_chat = is_chat
+        self.retry = retry
+        self.verbose = verbose
+
+        self.token_bucket = TokenBucket(query_per_second, verbose)
+
+    def generate_simple(self, inputs: Union[List[str]]):
+
+        def process_one(in_data: str):
+
+            if self.is_chat:
+                data = dict(
+                    model=self.model,
+                    messages=[{'role': 'user', 'content': in_data}],
+                    max_tokens=self.max_tokens,
+                    n=1,
+                    logprobs=self.logprobs,
+                    top_logprobs=self.top_logprobs,
+                    stop=None,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+            else:
+                data = dict(
+                    model=self.model,
+                    prompt=in_data,
+                    max_tokens=self.max_tokens,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+
+            # todo
+            openai_api_key = self.openai_api_key or ''
+            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            data = json.dumps(data, ensure_ascii=False)
+
+            if self.verbose:
+                print(f'>>data in generate_simple: {data}')
+
+            resp = requests.post(self.url, headers=header, data=data)
+            resp = resp.json()
+            if self.verbose:
+                print(f'>>resp in generate_simple: {resp}')
+
+            if self.logprobs:
+                return resp['choices']
+            else:
+                if self.is_chat:
+                    return resp['choices'][0]['message']['content'].strip()
+                else:
+                    return resp['choices'][0]['text'].strip()
+
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(process_one, inputs))
+
+        return results
+
+    def generate(self,
+                 inputs: Union[List[str], List[List]],
+                 **kwargs) -> List[str]:
+        """
+        Generate responses from OpenAI API.
+
+        Args:
+            inputs: The input messages for the model. It can be a string or a list of messages.
+                e.g. ['who are you ?', 'what is your name ?']
+                e.g. [[{'role': 'user', 'content': 'who are you ?'}], ...]
+            kwargs: The optional arguments for the model.
+        """
+        results = []
+        # with ThreadPoolExecutor() as executor:
+        #     results = list(executor.map(self._generate, inputs))
+
+        for input in inputs:
+            results.append(self._generate(input))
+
+        return results
+
+    def _generate(self, messages: Union[str, List[Dict]]) -> str:
+
+        if isinstance(messages, str):
+            messages = [{'role': 'user', 'content': messages}]
+
+        max_num_retries = 0
+        while max_num_retries < self.retry:
+            # self.wait()
+
+            header = {
+                'Authorization': f'Bearer {self.openai_api_key}',
+                'content-type': 'application/json',
+            }
+
+            try:
+                if self.is_chat:
+                    data = dict(
+                        model=self.model,
+                        messages=messages,
+                        max_tokens=self.max_tokens,
+                        n=1,
+                        logprobs=self.logprobs,
+                        top_logprobs=self.top_logprobs,
+                        stop=None,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+                else:
+                    # TODO: This is a temporary solution for non-chat models.
+                    input_prompts = []
+                    for msg in messages:
+                        input_prompts.append(msg['content'])
+
+                    data = dict(
+                        model=self.model,
+                        prompt='\n'.join(input_prompts),
+                        max_tokens=self.max_tokens,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+
+                def remove_none_val(input_d: dict):
+                    return {k: v for k, v in input_d.items() if v is not None}
+                data = remove_none_val(data)
+
+                if self.verbose:
+                    logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
+                raw_response = requests.post(self.url,
+                                             headers=header,
+                                             data=json.dumps(data, ensure_ascii=False))
+
+                response = raw_response.json()
+                if self.verbose:
+                    logger.info(f'>> response: {response}')
+
+                if self.logprobs:
+                    return response['choices']
+                else:
+                    if self.is_chat:
+                        return response['choices'][0]['message']['content'].strip()
+                    else:
+                        return response['choices'][0]['text'].strip()
+
+            except Exception as e:
+                logger.error(f'Error occurs: {str(e)}')
+                max_num_retries += 1
+                continue
+
+    def wait(self):
+        return self.token_bucket.get_token()
+
+
+class TokenBucket:
+    """A token bucket for rate limiting.
+
+    Args:
+        query_per_second (float): The rate of the token bucket.
+    """
+
+    def __init__(self, rate, verbose=False):
+        self._rate = rate
+        self._tokens = threading.Semaphore(0)
+        self.started = False
+        self._request_queue = Queue()
+        self.logger = get_logger()
+        self.verbose = verbose
+
+    def _add_tokens(self):
+        """Add tokens to the bucket."""
+        while True:
+            if self._tokens._value < self._rate:
+                self._tokens.release()
+            time.sleep(1 / self._rate)
+
+    def get_token(self):
+        """Get a token from the bucket."""
+        if not self.started:
+            self.started = True
+            threading.Thread(target=self._add_tokens, daemon=True).start()
+        self._tokens.acquire()
+        if self.verbose:
+            cur_time = time.time()
+            while not self._request_queue.empty():
+                if cur_time - self._request_queue.queue[0] > 60:
+                    self._request_queue.get()
+                else:
+                    break
+            self._request_queue.put(cur_time)
+            self.logger.info(f'Current RPM {self._request_queue.qsize()}.')

evalscope/models/model_adapter.py

@@ -362,6 +362,8 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
             torch_dtype: The torch dtype for model inference. Default: torch.float16.
             **kwargs: Other args.
         """
+
+        custom_generation_config = kwargs.pop('generation_config', None)
         model_cache_dir = get_model_cache_dir(root_cache_dir=cache_dir)
 
         self.model_id: str = model_id
@@ -414,6 +416,10 @@ class ChatGenerationModelAdapter(BaseModelAdapter):
         self.origin_tokenizer = deepcopy(tokenizer)
 
         self.generation_config, self.generation_template = self._parse_generation_config(tokenizer, model)
+
+        if custom_generation_config:
+            logger.info('**Updating generation config ...')
+            self.generation_config.update(**custom_generation_config.to_dict())
         logger.info(f'**Generation config init: {self.generation_config.to_dict()}')
 
         super().__init__(model=model, tokenizer=self.generation_template.tokenizer, model_cfg=model_cfg)

evalscope/perf/http_client.py

@@ -51,15 +51,15 @@ UNLIMITED_RATE = -1
 
 
 async def on_request_start(session, context, params):
-    logger.debug(f'Starting request: <{params}>')
+    logger.info(f'Starting request: <{params}>')
 
 
 async def on_request_chunk_sent(session, context, params):
-    logger.debug(f'Request body: {params}')
+    logger.info(f'Request body: {params}')
 
 
 async def on_response_chunk_received(session, context, params):
-    logger.debug(f'Response info: <{params}>')
+    logger.info(f'Response info: <{params}>')
 
 
 class AioHttpClient:
@@ -116,7 +116,7 @@ class AioHttpClient:
                 line = line.decode("utf8")
                 line = line.rstrip("\n").rstrip("\r")
                 if self.debug:
-                    logger.debug(line)
+                    logger.info(line)
                 sse_msg = ServerSentEvent.decode(line)
                 if not sse_msg:
                     continue
@@ -567,7 +567,7 @@ async def send_requests_worker(task_id, request_queue: asyncio.Queue, benchmark_
                     else:
                         if response_data:
                             collected_messages.append(response_data)  # save the message
-                            logger.debug(response_data)
+                            logger.info(response_data)
                             benchmark_data["chunk_times"].append(time.perf_counter())
 
                 benchmark_data["response_messages"] = collected_messages

evalscope/run_arena.py

@@ -100,17 +100,18 @@ class ArenaWorkflow:
             model_revision = cfg_d.get(EvalConfigKeys.MODEL_REVISION, None)
             precision = cfg_d.get(EvalConfigKeys.PRECISION, torch.float16)
             precision = eval(precision) if isinstance(precision, str) else precision
-            generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
-            generation_config = GenerationConfig(**generation_config)
+            custom_generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
+            custom_generation_config = GenerationConfig(**custom_generation_config)
             ans_output_file = os.path.join(WORK_DIR, cfg_d.get(EvalConfigKeys.OUTPUT_FILE))
             template_type = cfg_d.get(EvalConfigKeys.TEMPLATE_TYPE)
 
             answers_list = self._predict_answers(model_id_or_path=model_id_or_path,
                                                  model_revision=model_revision,
                                                  precision=precision,
-                                                 generation_config=generation_config,
+                                                 generation_config=custom_generation_config,
                                                  template_type=template_type)
 
+            os.makedirs(os.path.dirname(ans_output_file), exist_ok=True)
             dump_jsonl_data(answers_list, ans_output_file)
             logger.info(f'Answers generated by model {model_name} and saved to {ans_output_file}')
 
@@ -168,6 +169,7 @@ class ArenaWorkflow:
             res_list = ae.run(self.review_file)
             rating_df = res_list[0]
             logger.info(f'Rating results:\n{rating_df.to_csv()}')
+            os.makedirs(os.path.dirname(report_file), exist_ok=True)
             rating_df.to_csv(report_file, index=True)
             logger.info(f'Rating results are saved to {report_file}')
         else:

evalscope/summarizer.py

@@ -99,19 +99,25 @@ class Summarizer:
             elif eval_backend == EvalBackend.VLM_EVAL_KIT.value:
                 eval_config = Summarizer.parse_eval_config(candidate_task)
 
-                work_dir = eval_config.get('work_dir') or 'outputs/default'
+                work_dir = eval_config.get('work_dir') or 'outputs'
                 if not os.path.exists(work_dir):
                     raise ValueError(f'work_dir {work_dir} does not exist.')
                 
-                # TODO: parse summary files: acc.csv, score.csv, score.json for different models
                 for model in eval_config['model']:
                     if model['name'] == 'CustomAPIModel':
                         model_name = model['type']
                     else:
                         model_name = model['name']
-                    summary_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
+                    
+                    csv_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
+                    json_files = glob.glob(os.path.join(work_dir, model_name, '*.json'))
+                    
+                    summary_files = csv_files + json_files
                     for summary_file_path in summary_files:
-                        summary_res: dict = csv_to_list(file_path=summary_file_path)[0]
+                        if summary_file_path.endswith('csv'):
+                            summary_res: dict = csv_to_list(summary_file_path)[0]
+                        elif summary_file_path.endswith('json'):
+                            summary_res: dict = json_to_dict(summary_file_path)
                         file_name = os.path.basename(summary_file_path).split('.')[0]
                         final_res_list.append({file_name: summary_res})
                 

evalscope/third_party/longbench_write/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.third_party.longbench_write.longbench_write import run_task