evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/metrics/metrics.py

@@ -2,18 +2,17 @@
 # Copyright (c) EleutherAI. and its affiliates.
 # Copyright (c) OpenAI. and its affiliates.
 import itertools
-import math
-from collections.abc import Iterable
-from collections import defaultdict
-from typing import Dict, List, Union
-from nltk.translate.bleu_score import sentence_bleu
-from nltk import word_tokenize
 import jieba
-
+import math
 import numpy as np
+import random
 import sacrebleu
 import sklearn.metrics
-import random
+from collections import defaultdict
+from collections.abc import Iterable
+from nltk import word_tokenize
+from nltk.translate.bleu_score import sentence_bleu
+from typing import Dict, List, Union
 
 
 def mean(arr):
@@ -22,12 +21,12 @@ def mean(arr):
 
 def pop_stddev(arr):
     mu = mean(arr)
-    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
+    return math.sqrt(sum([(x - mu)**2 for x in arr]) / len(arr))
 
 
 def sample_stddev(arr):
     mu = mean(arr)
-    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
+    return math.sqrt(sum([(x - mu)**2 for x in arr]) / (len(arr) - 1))
 
 
 def mean_stderr(arr):
@@ -134,13 +133,14 @@ def bleu(items):
     refs, preds = _sacreformat(refs, preds)
     return sacrebleu.corpus_bleu(preds, refs).score
 
+
 def bleu_ngram_one_sample(predict, reference):
     """
     Calculate BLEU-1, BLEU-2, BLEU-3, and BLEU-4 scores
 
     Args:
         items: [(ref, pred)]
-        
+
     Returns:
         {
             'bleu-1': 0.8,
@@ -150,6 +150,7 @@ def bleu_ngram_one_sample(predict, reference):
         }
 
     """
+
     def is_contains_chinese(strs):
         for _char in strs:
             if '\u4e00' <= _char <= '\u9fa5':
@@ -230,6 +231,7 @@ def _sacreformat(refs, preds):
 
 
 class _bootstrap_internal:
+
     def __init__(self, f, n):
         self.f = f
         self.n = n
@@ -260,11 +262,11 @@ def bootstrap_stderr(f, xs, iters):
 
     print('bootstrapping for stddev:', f.__name__)
     for bootstrap in tqdm(
-        pool.imap(
-            _bootstrap_internal(f, chunk_size),
-            [(i, xs) for i in range(iters // chunk_size)],
-        ),
-        total=iters // chunk_size,
+            pool.imap(
+                _bootstrap_internal(f, chunk_size),
+                [(i, xs) for i in range(iters // chunk_size)],
+            ),
+            total=iters // chunk_size,
     ):
         # sample w replacement
         res.extend(bootstrap)
@@ -372,11 +374,9 @@ def calculate_arc_accuracy(question_answers: Dict[str, str], predictions: Dict[s
     return score / len(question_answers)
 
 
-def calculate_pass_at_k(
-    num_samples: Union[int, List[int], np.ndarray],
-    num_correct: Union[List[int], np.ndarray],
-    k: int = 1
-) -> np.ndarray:
+def calculate_pass_at_k(num_samples: Union[int, List[int], np.ndarray],
+                        num_correct: Union[List[int], np.ndarray],
+                        k: int = 1) -> np.ndarray:
     """
     Estimates pass@k of each problem and returns them in an array.
     Examples:

evalscope/metrics/rouge_metric.py

@@ -1,18 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import jieba
 import logging
 from collections import defaultdict
 from pathlib import Path
+from rouge_chinese import Rouge
 from statistics import mean
-
 from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
 from evalscope.metrics.bundled_rouge_score import rouge_scorer
 
-from rouge_chinese import Rouge
-import jieba
-
 
 class DummyTokenizer:
 
@@ -24,9 +22,7 @@ HERE = Path(__file__).absolute().parent
 
 logger = logging.getLogger(__name__)
 
-scorer = rouge_scorer.RougeScorer(
-    ['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer()
-)
+scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
 zh_scorer = Rouge()
 
 
@@ -52,11 +48,7 @@ def compute_rouge_score(predict_l, reference_l):
             result[rouge_key].append(one_sample[rouge_key])
     rlt = {}
     for rouge_key in MetricsConstant.ROUGE_KEYS:
-        rlt[rouge_key] = (
-            mean(result[rouge_key]) * 100
-            if rouge_key in result
-            else MetricsConstant.INVALID_VALUE
-        )
+        rlt[rouge_key] = (mean(result[rouge_key]) * 100 if rouge_key in result else MetricsConstant.INVALID_VALUE)
     return rlt
 
 
@@ -111,9 +103,9 @@ def _to_table(final_result) -> str:
             if not task:
                 continue
             elif task == 'total':
-                row.append(f'{final_result["total"]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["total"]["rouge"][rouge_key]:0.2f}')
             else:
-                row.append(f'{final_result["tasks"][task]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["tasks"][task]["rouge"][rouge_key]:0.2f}')
         table.append('\t'.join(row))
 
     return '\n'.join(table)
@@ -122,23 +114,17 @@ def _to_table(final_result) -> str:
 def run_rouge_eval(data_l, md_level=2, report_metric_key='rouge-l-f'):
     print(f"{'#' * md_level} Rouge Eval")
     for data in tqdm(data_l):
-        data['rouge'] = compute_rouge_score_one_sample(
-            data['gen_tok_str'], data['reference_tok_str']
-        )
+        data['rouge'] = compute_rouge_score_one_sample(data['gen_tok_str'], data['reference_tok_str'])
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:
             task_data_d[task].append(data)
 
     total_rouge = mean([data['rouge'][report_metric_key] for data in data_l])
-    print(
-        f'[total], count: {len(data_l)}, {report_metric_key}: '
-        f'{total_rouge * 100:0.2f}%'
-    )
+    print(f'[total], count: {len(data_l)}, {report_metric_key}: '
+          f'{total_rouge * 100:0.2f}%')
 
     for task, task_data in task_data_d.items():
         task_rouge = mean([data['rouge'][report_metric_key] for data in task_data])
-        print(
-            f'[{task}], count: {len(task_data_d[task])}, {report_metric_key}: '
-            f'{task_rouge * 100:0.2f}%'
-        )
+        print(f'[{task}], count: {len(task_data_d[task])}, {report_metric_key}: '
+              f'{task_rouge * 100:0.2f}%')

evalscope/models/__init__.py

@@ -1,4 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.models.model import BaseModel
-from evalscope.models.model import ChatBaseModel
+from evalscope.models.model import BaseModel, ChatBaseModel

evalscope/models/api/openai_api.py

@@ -1,34 +1,36 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import json
+import requests
 import threading
 import time
 from asyncio import Queue
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from tqdm import tqdm
+from typing import Dict, List, Optional, Union
 
-import requests
-from typing import Union, List, Optional, Dict
-from concurrent.futures import ThreadPoolExecutor
-from modelscope.utils.logger import get_logger
+from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
 class OpenaiApi:
 
-    def __init__(self,
-                 model: str,
-                 openai_api_key,
-                 openai_api_base,
-                 logprobs: Optional[bool] = False,
-                 top_logprobs: Optional[int] = None,
-                 max_new_tokens: int = 4096,
-                 temperature: Optional[float] = 0.0,
-                 repetition_penalty: Optional[float] = 1.0,
-                 is_chat: bool = True,
-                 verbose: bool = True,
-                 retry: int = 3,
-                 query_per_second: int = 10,     # TODO
-                 **kwargs):
+    def __init__(
+            self,
+            model: str,
+            openai_api_key,
+            openai_api_base,
+            logprobs: Optional[bool] = False,
+            top_logprobs: Optional[int] = None,
+            max_new_tokens: int = 4096,
+            temperature: Optional[float] = 0.0,
+            repetition_penalty: Optional[float] = 1.0,
+            is_chat: bool = True,
+            verbose: bool = True,
+            retry: int = 3,
+            query_per_second: int = 10,  # TODO
+            **kwargs):
 
         self.temperature = temperature
         self.repetition_penalty = repetition_penalty
@@ -45,14 +47,17 @@ class OpenaiApi:
 
         self.token_bucket = TokenBucket(query_per_second, verbose)
 
-    def generate_simple(self, inputs: Union[List[str]]):
+    def generate_simple(self, inputs: Union[List[str]], num_proc: int = 8):
 
         def process_one(in_data: str):
 
             if self.is_chat:
                 data = dict(
                     model=self.model,
-                    messages=[{'role': 'user', 'content': in_data}],
+                    messages=[{
+                        'role': 'user',
+                        'content': in_data
+                    }],
                     max_tokens=self.max_tokens,
                     n=1,
                     logprobs=self.logprobs,
@@ -72,7 +77,10 @@ class OpenaiApi:
 
             # todo
             openai_api_key = self.openai_api_key or ''
-            header = {'Authorization': f'Bearer ', 'content-type': 'application/json', }
+            header = {
+                'Authorization': f'Bearer {openai_api_key}',
+                'content-type': 'application/json',
+            }
             data = json.dumps(data, ensure_ascii=False)
 
             if self.verbose:
@@ -91,14 +99,18 @@ class OpenaiApi:
                 else:
                     return resp['choices'][0]['text'].strip()
 
-        with ThreadPoolExecutor() as executor:
-            results = list(executor.map(process_one, inputs))
+        results = []
+        with ThreadPoolExecutor(max_workers=num_proc) as executor:
+            # Submit all tasks
+            future_to_task = {executor.submit(process_one, input_one): input_one for input_one in inputs}
+
+            # Show progress bar
+            for future in tqdm(as_completed(future_to_task), total=len(inputs)):
+                results.append(future.result())
 
         return results
 
-    def generate(self,
-                 inputs: Union[List[str], List[List]],
-                 **kwargs) -> List[str]:
+    def generate(self, inputs: Union[List[str], List[List]], **kwargs) -> List[str]:
         """
         Generate responses from OpenAI API.
 
@@ -160,13 +172,12 @@ class OpenaiApi:
 
                 def remove_none_val(input_d: dict):
                     return {k: v for k, v in input_d.items() if v is not None}
+
                 data = remove_none_val(data)
 
                 if self.verbose:
                     logger.info(f'>> Post data: {json.dumps(data, ensure_ascii=False)}')
-                raw_response = requests.post(self.url,
-                                             headers=header,
-                                             data=json.dumps(data, ensure_ascii=False))
+                raw_response = requests.post(self.url, headers=header, data=json.dumps(data, ensure_ascii=False))
 
                 response = raw_response.json()
                 if self.verbose:

evalscope/models/custom/__init__.py

@@ -1,4 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 from evalscope.models.custom.custom_model import *
-

evalscope/models/custom/custom_model.py

@@ -1,7 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from abc import ABC, abstractmethod
-from typing import Any, Union, Dict, List
 import torch
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Union
 
 
 class CustomModel(ABC):
@@ -11,7 +11,7 @@ class CustomModel(ABC):
         self.kwargs = kwargs
 
         if config.get('model_id', None) is None:
-            raise ValueError(f"**Error: model_id is required in config for CustomModel. Got config: {config}")
+            raise ValueError(f'**Error: model_id is required in config for CustomModel. Got config: {config}')
 
     @abstractmethod
     @torch.no_grad()

evalscope/models/dummy_chat_model.py

@@ -2,6 +2,7 @@
 
 import random
 import time
+
 from evalscope.models import ChatBaseModel
 from evalscope.utils.logger import get_logger
 
@@ -32,15 +33,13 @@ class DummyChatModel(ChatBaseModel):
 
         # Build response
         res = {
-            'choices': [
-                {
-                    'index': 0,
-                    'message': {
-                        'content': choice,
-                        'role': 'assistant'
-                    }
+            'choices': [{
+                'index': 0,
+                'message': {
+                    'content': choice,
+                    'role': 'assistant'
                 }
-            ],
+            }],
             'created': time.time(),
             'model': self.MODEL_ID + '-' + self.REVISION,
             'object': 'chat.completion',