evalscope 0.5.3__py3-none-any.whl → 0.5.4__py3-none-any.whl

evalscope/backend/opencompass/backend_manager.py

@@ -242,4 +242,6 @@ if __name__ == '__main__':
                 'limit': 5
                 }
     )
+    all_datasets = OpenCompassBackendManager.list_datasets()
+    print(f'all_datasets: {all_datasets}')
     oc_backend_manager.run()

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -1,5 +1,5 @@
 from typing import Optional, Union
-from evalscope.utils import is_module_installed, get_module_path, get_valid_list, yaml_to_dict, json_to_dict
+from evalscope.utils import is_module_installed, get_valid_list
 from evalscope.backend.base import BackendManager
 from evalscope.utils.logger import get_logger
 from functools import partial

evalscope/benchmarks/benchmark.py

@@ -46,7 +46,7 @@ class Benchmark(object):
 
             dataset.dataset_name = dataset_name.split('/')[-1]
             dataset.subset_name = subset
-            dataset.split = split
+            # dataset.split = split
             return dataset
         elif hub == 'HuggingFace':
             # TODO: implement this by xingjun.wxj@alibaba-inc.com

evalscope/evaluator/evaluator.py

@@ -244,8 +244,8 @@ class Evaluator(object):
                 answer_d[AnswerKeys.ORIGIN_PROMPT] = input_prompt
 
                 if debug:
-                    logger.debug(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.debug(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
+                    logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
+                    logger.info(f'**predicted ans: {json.dumps(answer_d, ensure_ascii=False)} \n')
 
                 answers_list.append(answer_d)
 
@@ -349,7 +349,7 @@ class Evaluator(object):
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
             if debug:
-                logger.debug(review_d)
+                logger.info(review_d)
 
             reviews_list.append(review_d)
 

evalscope/models/api/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.models.api.openai_api import OpenaiApi

evalscope/models/api/openai_api.py

@@ -0,0 +1,228 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import json
+import threading
+import time
+from asyncio import Queue
+
+import requests
+from typing import Union, List, Optional, Dict
+from concurrent.futures import ThreadPoolExecutor
+from modelscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class OpenaiApi:
+
+    def __init__(self,
+                 model: str,
+                 openai_api_key,
+                 openai_api_base,
+                 logprobs: Optional[bool] = False,
+                 top_logprobs: Optional[int] = None,
+                 max_new_tokens: int = 4096,
+                 temperature: Optional[float] = 0.0,
+                 repetition_penalty: Optional[float] = 1.0,
+                 is_chat: bool = True,
+                 verbose: bool = True,
+                 retry: int = 3,
+                 query_per_second: int = 10,     # TODO
+                 **kwargs):
+
+        self.temperature = temperature
+        self.repetition_penalty = repetition_penalty
+        self.max_tokens = max_new_tokens
+        self.logprobs = logprobs
+        self.top_logprobs = top_logprobs
+
+        self.openai_api_key = openai_api_key
+        self.url = openai_api_base
+        self.model = model
+        self.is_chat = is_chat
+        self.retry = retry
+        self.verbose = verbose
+
+        self.token_bucket = TokenBucket(query_per_second, verbose)
+
+    def generate_simple(self, inputs: Union[List[str]]):
+
+        def process_one(in_data: str):
+
+            if self.is_chat:
+                data = dict(
+                    model=self.model,
+                    messages=[{'role': 'user', 'content': in_data}],
+                    max_tokens=self.max_tokens,
+                    n=1,
+                    logprobs=self.logprobs,
+                    top_logprobs=self.top_logprobs,
+                    stop=None,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+            else:
+                data = dict(
+                    model=self.model,
+                    prompt=in_data,
+                    max_tokens=self.max_tokens,
+                    temperature=self.temperature,
+                    repetition_penalty=self.repetition_penalty,
+                )
+
+            # todo
+            openai_api_key = self.openai_api_key or ''
+            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            data = json.dumps(data, ensure_ascii=False)
+
+            if self.verbose:
+                print(f'>>data in generate_simple: {data}')
+
+            resp = requests.post(self.url, headers=header, data=data)
+            resp = resp.json()
+            if self.verbose:
+                print(f'>>resp in generate_simple: {resp}')
+
+            if self.logprobs:
+                return resp['choices']
+            else:
+                if self.is_chat:
+                    return resp['choices'][0]['message']['content'].strip()
+                else:
+                    return resp['choices'][0]['text'].strip()
+
+        with ThreadPoolExecutor() as executor:
+            results = list(executor.map(process_one, inputs))
+
+        return results
+
+    def generate(self,
+                 inputs: Union[List[str], List[List]],
+                 **kwargs) -> List[str]:
+        """
+        Generate responses from OpenAI API.
+
+        Args:
+            inputs: The input messages for the model. It can be a string or a list of messages.
+                e.g. ['who are you ?', 'what is your name ?']
+                e.g. [[{'role': 'user', 'content': 'who are you ?'}], ...]
+            kwargs: The optional arguments for the model.
+        """
+        results = []
+        # with ThreadPoolExecutor() as executor:
+        #     results = list(executor.map(self._generate, inputs))
+
+        for input in inputs:
+            results.append(self._generate(input))
+
+        return results
+
+    def _generate(self, messages: Union[str, List[Dict]]) -> str:
+
+        if isinstance(messages, str):
+            messages = [{'role': 'user', 'content': messages}]
+
+        max_num_retries = 0
+        while max_num_retries < self.retry:
+            # self.wait()
+
+            header = {
+                'Authorization': f'Bearer {self.openai_api_key}',
+                'content-type': 'application/json',
+            }
+
+            try:
+                if self.is_chat:
+                    data = dict(
+                        model=self.model,
+                        messages=messages,
+                        max_tokens=self.max_tokens,
+                        n=1,
+                        logprobs=self.logprobs,
+                        top_logprobs=self.top_logprobs,
+                        stop=None,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+                else:
+                    # TODO: This is a temporary solution for non-chat models.
+                    input_prompts = []
+                    for msg in messages:
+                        input_prompts.append(msg['content'])
+
+                    data = dict(
+                        model=self.model,
+                        prompt='\n'.join(input_prompts),
+                        max_tokens=self.max_tokens,
+                        temperature=self.temperature,
+                        repetition_penalty=self.repetition_penalty,
+                    )
+
+                def remove_none_val(input_d: dict):
+                    return {k: v for k, v in input_d.items() if v is not None}
+                data = remove_none_val(data)
+
+                if self.verbose:
+                    logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
+                raw_response = requests.post(self.url,
+                                             headers=header,
+                                             data=json.dumps(data, ensure_ascii=False))
+
+                response = raw_response.json()
+                if self.verbose:
+                    logger.info(f'>> response: {response}')
+
+                if self.logprobs:
+                    return response['choices']
+                else:
+                    if self.is_chat:
+                        return response['choices'][0]['message']['content'].strip()
+                    else:
+                        return response['choices'][0]['text'].strip()
+
+            except Exception as e:
+                logger.error(f'Error occurs: {str(e)}')
+                max_num_retries += 1
+                continue
+
+    def wait(self):
+        return self.token_bucket.get_token()
+
+
+class TokenBucket:
+    """A token bucket for rate limiting.
+
+    Args:
+        query_per_second (float): The rate of the token bucket.
+    """
+
+    def __init__(self, rate, verbose=False):
+        self._rate = rate
+        self._tokens = threading.Semaphore(0)
+        self.started = False
+        self._request_queue = Queue()
+        self.logger = get_logger()
+        self.verbose = verbose
+
+    def _add_tokens(self):
+        """Add tokens to the bucket."""
+        while True:
+            if self._tokens._value < self._rate:
+                self._tokens.release()
+            time.sleep(1 / self._rate)
+
+    def get_token(self):
+        """Get a token from the bucket."""
+        if not self.started:
+            self.started = True
+            threading.Thread(target=self._add_tokens, daemon=True).start()
+        self._tokens.acquire()
+        if self.verbose:
+            cur_time = time.time()
+            while not self._request_queue.empty():
+                if cur_time - self._request_queue.queue[0] > 60:
+                    self._request_queue.get()
+                else:
+                    break
+            self._request_queue.put(cur_time)
+            self.logger.info(f'Current RPM {self._request_queue.qsize()}.')

evalscope/perf/http_client.py

@@ -51,15 +51,15 @@ UNLIMITED_RATE = -1
 
 
 async def on_request_start(session, context, params):
-    logger.debug(f'Starting request: <{params}>')
+    logger.info(f'Starting request: <{params}>')
 
 
 async def on_request_chunk_sent(session, context, params):
-    logger.debug(f'Request body: {params}')
+    logger.info(f'Request body: {params}')
 
 
 async def on_response_chunk_received(session, context, params):
-    logger.debug(f'Response info: <{params}>')
+    logger.info(f'Response info: <{params}>')
 
 
 class AioHttpClient:
@@ -116,7 +116,7 @@ class AioHttpClient:
                 line = line.decode("utf8")
                 line = line.rstrip("\n").rstrip("\r")
                 if self.debug:
-                    logger.debug(line)
+                    logger.info(line)
                 sse_msg = ServerSentEvent.decode(line)
                 if not sse_msg:
                     continue
@@ -567,7 +567,7 @@ async def send_requests_worker(task_id, request_queue: asyncio.Queue, benchmark_
                     else:
                         if response_data:
                             collected_messages.append(response_data)  # save the message
-                            logger.debug(response_data)
+                            logger.info(response_data)
                             benchmark_data["chunk_times"].append(time.perf_counter())
 
                 benchmark_data["response_messages"] = collected_messages

evalscope/third_party/longbench_write/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.third_party.longbench_write.longbench_write import run_task

evalscope/third_party/longbench_write/eval.py

@@ -0,0 +1,284 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright (c) ZhipuAI, Inc. and its affiliates.
+import multiprocessing
+import os
+import json
+import random
+import re
+from concurrent.futures import ThreadPoolExecutor
+
+import matplotlib.pyplot as plt
+import numpy as np
+import requests
+from tqdm import tqdm
+
+from evalscope.utils import jsonl_to_list
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+"""
+This script is used to evaluate results of predictions for the LongWriter model.
+Refer to https://github.com/THUDM/LongWriter for more details.
+
+EvalLength:
+    Evaluate the length of the generated responses.
+    Metrics:
+        score_l: The average score of the length evaluation.
+
+EvalQuality:
+    Evaluate the quality of the generated responses by using Judge Model.
+    Metrics:
+        score_q: The average score of the quality evaluation.
+"""
+
+
+class EvalLength:
+
+    EVAL_L = 'eval_length'
+
+    def __init__(self, model: str, pred_path: str, output_dir: str):
+        self.model = model
+        self.pred_path = pred_path
+        self.output_dir = output_dir
+
+        self.model_id_path = self.model.strip(os.sep).replace(os.sep, '__')
+
+    @staticmethod
+    def score(x, y):
+        if y > x:
+            return 100 * max(0, 1. - (y / x - 1) / 3)
+        else:
+            return 100 * max(0, 1. - (x / y - 1) / 2)
+
+    def eval(self, dump_res: bool = True):
+        # example = {"prompt": "Write an outline for a short 100-word blog post about xxx", "type": "Community Forum", "length": 100, "response_length": 103, "response": "I. Introduction A. xxx"}
+        predictions = [json.loads(line) for line in open(self.pred_path, encoding='utf-8')]
+        x, y, scores = [], [], []
+
+        for pred in tqdm(predictions, total=len(predictions), desc=f'Process of eval_l: '):
+            x.append(pred["length"])
+            y.append(pred["response_length"])
+            scores.append(self.score(pred["length"], pred["response_length"]))
+
+        avg_score_l = np.mean(scores)
+        logger.info(f'Average score of length evaluation: {avg_score_l:.2f}')
+
+        # Dump to output file
+        if dump_res:
+            output_res_path = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_L}.jsonl'
+            with open(output_res_path, 'w') as f:
+                f.write(json.dumps({'score_l': avg_score_l, 'scores': scores}, ensure_ascii=False) + '\n')
+                logger.info(f'Successfully dumped evaluation results to {output_res_path}')
+
+        return x, y, scores
+
+    def plot(self, x: list, y: list):
+        plt = self.plot_img(x, y)
+        output_pic_path = f'{self.output_dir}/{self.model_id_path}/eval_length_scatter.png'
+        plt.savefig(output_pic_path)
+        logger.info(f'Successfully saved scatter plot to {output_pic_path}')
+
+    @staticmethod
+    def plot_img(x: list, y: list):
+        # set plt size 6x6
+        plt.figure(figsize=(6, 6))
+        lmt = 25000
+        # plot x, y
+        plt.scatter(x, y, s=100, c='r', alpha=0.3)
+        # plot x=y
+        plt.plot([0, lmt], [0, lmt], 'k--')
+        plt.xscale('log')
+        plt.yscale('log')
+        plt.xlim(50, lmt)
+        plt.ylim(50, lmt)
+        plt.xlabel('Required Length', fontsize=20, fontweight='bold')
+        plt.ylabel('Output Length', fontsize=20, fontweight='bold')
+        plt.xticks(fontsize=24)
+        plt.yticks(fontsize=24)
+        plt.tight_layout()
+
+        return plt
+
+
+class EvalQuality:
+
+        EVAL_Q = 'eval_quality'
+        OPENAI_BASE_URL = 'https://api.openai.com/v1/chat/completions'
+        DIMS = ["Relevance", "Accuracy", "Coherence", "Clarity", "Breadth and Depth", "Reading Experience"]
+
+        def __init__(self,
+                     model: str,
+                     pred_path: str,
+                     output_dir: str,
+                     prompt_template_path: str,
+                     openai_api_key: str = None,
+                     openai_api_base: str = OPENAI_BASE_URL,
+                     openai_gpt_model: str = 'gpt-4o-2024-05-13',
+                     generation_kwargs: dict = None,
+                     proc_num: int = 8):
+
+            self.model = model
+            self.openai_api_base = openai_api_base
+            self.pred_path = pred_path
+            self.output_dir = output_dir
+            self.proc_num = proc_num
+            self.eval_scores = []
+
+            assert os.path.exists(self.pred_path), f'Prediction file not found: {self.pred_path}'
+
+            # Default: temperature=0.5, max_new_tokens=1024, stop=None
+            if generation_kwargs is None:
+                self.generation_kwargs = dict({
+                    'max_new_tokens': 1024,
+                    'temperature': 0.5,
+                    'stop': None,
+                })
+            else:
+                self.generation_kwargs = generation_kwargs
+
+            self.prompt_template: str = open(prompt_template_path, 'r', encoding='utf-8').read()
+
+            self.model_id_path = self.model.strip(os.sep).replace(os.sep, '__')
+            self.output_res_path = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_Q}.jsonl'
+
+            self.openai_api_key: str = openai_api_key
+            self.openai_gpt_model = openai_gpt_model
+            assert self.openai_api_key, 'Please set `OPENAI_API_KEY` in environment variables.'
+
+        def get_response_gpt4(self, prompt, temperature=0.5, max_new_tokens=1024, stop=None):
+            tries = 0
+            while tries < 1:
+                tries += 1
+                try:
+                    headers = {
+                        'Authorization': "Bearer {}".format(self.openai_api_key),
+                    }
+                    messages = [
+                        {'role': 'user', 'content': prompt},
+                    ]
+                    resp = requests.post(self.openai_api_base, json={
+                        "model": self.openai_gpt_model,
+                        "messages": messages,
+                        "temperature": temperature,
+                        "max_tokens": max_new_tokens,
+                        "stop": stop,
+                    }, headers=headers, timeout=600)
+                    if resp.status_code != 200:
+                        raise Exception(resp.text)
+                    resp = resp.json()
+                    logger.info(f'>>gpt resp: {resp}')
+                    break
+                except KeyboardInterrupt as e:
+                    raise e
+                except Exception as e:
+                    if "maximum context length" in str(e):
+                        raise e
+                    elif "triggering" in str(e):
+                        return 'Trigger OpenAI\'s content management policy'
+                    logger.error("Error Occurs: \"%s\"        Retry ..." % (str(e)))
+            else:
+                logger.error("Max tries. Failed.")
+                return "Max tries. Failed."
+            try:
+                return resp["choices"][0]["message"]["content"]
+            except:
+                return ''
+
+        @staticmethod
+        def extract_info(pattern, text):
+            match = re.search(pattern, text, re.DOTALL)
+            if match:
+                return match.group(1)
+            else:
+                return None
+
+        def process_data(self, item):
+            # for item in tqdm(items, total=len(items), desc=f'Process of eval_q: '):
+            prompt = self.prompt_template.replace('$INST$', item['prompt']).replace('$RESPONSE$', item["response"])
+            scores = None
+            output = self.get_response_gpt4(prompt, **self.generation_kwargs)
+            try:
+                if '```json' in output:
+                    output = self.extract_info(r'```json\n(.*?)\n```', output)
+                output = output.replace('\n', '')
+                scores = json.loads(output)
+                for dim in self.DIMS:
+                    if dim not in scores:
+                        logger.warning(f'Cannot find score for dimension: {dim} in scores {scores}.')
+                        scores = None
+            except Exception as e:
+                logger.error(f'Error occurs during process data: {str(e)}')
+
+            if scores is None:
+                logger.error(f'Failed to extract scores for item: {item}')
+            else:
+                logger.info(f'>>scores: {scores}')
+                item['scores'] = scores
+
+            return item
+
+        def eval(self):
+
+            data_all = jsonl_to_list(self.pred_path)
+            total = len(data_all)
+            assert total > 0, f'No data found in prediction file: {self.pred_path}'
+
+            random.shuffle(data_all)
+
+            with ThreadPoolExecutor() as executor:
+                self.eval_scores = list(executor.map(self.process_data, data_all))
+
+            # self.process_data(items=data)
+            logger.info(f'>>self.eval_scores: {self.eval_scores}')
+
+            total_score = dict()
+            for dim in self.DIMS:
+                # scores = [float(score[dim]) if dim in score else 3 for score in self.eval_scores]
+                scores = [float(item['scores'][dim]) if 'scores' in item and dim in item['scores'] else 3 for item in self.eval_scores]
+                total_score[dim] = ((sum(scores) / len(scores)) - 1) * 25
+            total_score['total'] = sum(total_score.values()) / len(total_score)
+            logger.info(f'Total score of quality evaluation: {total_score["total"]:.2f}')
+
+            output_res_path: str = f'{self.output_dir}/{self.model_id_path}/{self.EVAL_Q}.jsonl'
+            with open(output_res_path, 'w', encoding='utf-8') as fout:
+                fout.write(json.dumps(total_score, ensure_ascii=False) + '\n')
+
+
+def run_eval(model: str,
+             pred_path: str,
+             output_dir: str,
+             prompt_template_path: str,
+             openai_api_key: str,
+             openai_api_base: str,
+             openai_gpt_model: str,
+             generation_kwargs: dict,
+             proc_num: int,
+             stage: list,
+             ):
+    logger.info(f'Got eval stages: {stage}')
+
+    if 'eval_l' in stage:
+        logger.info(f'Processing evaluation of length for model: {model}')
+        eval_length = EvalLength(model=model,
+                                 pred_path=pred_path,
+                                 output_dir=output_dir)
+        x, y, _ = eval_length.eval()
+        eval_length.plot(x, y)
+    else:
+        logger.warning(f'*** Skip `eval_l` stage ***')
+
+    if 'eval_q' in stage:
+        logger.info(f'Processing evaluation of quality for model: {model}')
+        eval_quality = EvalQuality(model=model,
+                                   pred_path=pred_path,
+                                   output_dir=output_dir,
+                                   prompt_template_path=prompt_template_path,
+                                   openai_api_key=openai_api_key,
+                                   openai_api_base=openai_api_base,
+                                   openai_gpt_model=openai_gpt_model,
+                                   generation_kwargs=generation_kwargs,
+                                   proc_num=proc_num)
+        eval_quality.eval()
+    else:
+        logger.warning('*** Skip `eval_q` stage ***')