evalscope 0.6.1__py3-none-any.whl → 0.7.1__py3-none-any.whl

tests/test_run_all.py

@@ -0,0 +1,12 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import subprocess
+
+if __name__ == '__main__':
+    cmd = f'TEST_LEVEL_LIST=0,1 python3 -m unittest discover tests'
+    run_res = subprocess.run(cmd, text=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+    if run_res.returncode == 0:
+        print(f'>>test_run_all stdout: {run_res.stdout}')
+    else:
+        print(f'>>test_run_all stderr: {run_res.stderr}')

tests/vlm/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

tests/vlm/test_vlmeval.py

@@ -0,0 +1,59 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import subprocess
+import unittest
+
+from evalscope.run import run_task
+from evalscope.summarizer import Summarizer
+from evalscope.utils import is_module_installed, test_level_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class TestVLMEval(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self._check_env('vlmeval')
+
+    def tearDown(self) -> None:
+        pass
+
+    @staticmethod
+    def _check_env(module_name: str):
+        if is_module_installed(module_name):
+            logger.info(f'{module_name} is installed.')
+        else:
+            raise ModuleNotFoundError(f'run: pip install {module_name}')
+
+    @unittest.skipUnless(0 in test_level_list(), 'skip test in current test level')
+    def test_run_vlm_eval_local(self):
+        task_cfg = {
+            'eval_backend': 'VLMEvalKit',
+            'eval_config': {
+                'data': ['SEEDBench_IMG', 'ChartQA_TEST'],
+                'limit': 20,
+                'mode': 'all',
+                'model': [{
+                    'name': 'qwen-vl-chat',
+                    'model_path': '../models/Qwen-VL-Chat'
+                }],  # model name for VLMEval config
+                'nproc': 1,
+                'reuse': True,
+                'work_dir': 'outputs'
+            }
+        }
+
+        logger.info(f'>> Start to run task: {task_cfg}')
+
+        run_task(task_cfg)
+
+        logger.info('>> Start to get the report with summarizer ...')
+        report_list = Summarizer.get_report_from_cfg(task_cfg)
+        logger.info(f'\n>>The report list: {report_list}')
+
+        assert len(report_list) > 0, f'Failed to get report list: {report_list}'
+
+
+if __name__ == '__main__':
+    unittest.main(buffer=False)

evalscope/perf/_logging.py

@@ -1,32 +0,0 @@
-import logging
-import os
-
-
-logger = logging.getLogger('perf')
-
-
-def enable_logging():
-    level = os.environ.get('LOGGING_LEVEL', 'info')
-    if level is not None:  # set logging level.
-        if level not in ['info', 'debug']:
-            # set logging level env, but invalid value, use default.
-            level = 'info'
-        if level == 'info':
-            logger.setLevel(logging.INFO)
-        else:
-            logger.setLevel(logging.DEBUG)
-        # set default logging handler
-        console_handler = logging.StreamHandler()
-        formatter = logging.Formatter(
-            '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s'  # noqa E501
-        )
-        #formatter = logging.Formatter(
-        #    '%(asctime)s - %(name)s - %(levelname)s - %(message)s'  # noqa E501
-        #)
-        console_handler.setFormatter(formatter)
-        logger.addHandler(console_handler)
-
-
-# in release disable dashscope log
-# you can enable dashscope log for debugger.
-enable_logging()

evalscope/perf/datasets/longalpaca_12k.py

@@ -1,20 +0,0 @@
-import sys
-from typing import Any, Dict, Iterator, List
-from evalscope.perf.dataset_plugin_base import DatasetPluginBase
-
-from evalscope.perf.plugin_registry import register_dataset
-from evalscope.perf.query_parameters import QueryParameters
-
-@register_dataset('longalpaca')
-class LongAlpacaDatasetPlugin(DatasetPluginBase):
-    """Read data from file which is list of requests.
-           Sample: https://huggingface.co/datasets/Yukang/LongAlpaca-12k
-    """
-    def __init__(self, query_parameters: QueryParameters):
-        super().__init__(query_parameters)
-
-    def build_messages(self) -> Iterator[List[Dict]]:
-        for item in self.dataset_json_list(self.query_parameters.dataset_path):
-            prompt = item['instruction'].strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/datasets/openqa.py

@@ -1,22 +0,0 @@
-from sys import maxsize
-import sys
-from typing import Any, Dict, Iterator, List
-import json
-from evalscope.perf.dataset_plugin_base import DatasetPluginBase
-from evalscope.perf.plugin_registry import register_dataset
-from evalscope.perf.query_parameters import QueryParameters
-
-@register_dataset('openqa')
-class OpenqaDatasetPlugin(DatasetPluginBase):
-    """Read dataset and return prompt.
-        Datasets: https://huggingface.co/datasets/Hello-SimpleAI/HC3-Chinese/blob/main/open_qa.jsonl
-    """
-    def __init__(self, query_parameters: QueryParameters):
-        super().__init__(query_parameters)
-        
-    def build_messages(self) -> Iterator[List[Dict]]:
-        for item in self.dataset_line_by_line(self.query_parameters.dataset_path):
-            item = json.loads(item)
-            prompt = item['question'].strip()
-            if len(prompt) > self.query_parameters.min_prompt_length and len(prompt) < self.query_parameters.max_prompt_length:
-                yield [{'role': 'user', 'content': prompt}]

evalscope/perf/plugin_registry.py

@@ -1,35 +0,0 @@
-
-from typing import Any
-
-
-class PluginRegistry:
-    def __init__(self):
-        self._registry = {}
-
-    def register(self, name, cls):
-        self._registry[name] = cls
-        return cls
-
-    def get_class(self, name):
-        return self._registry[name]
-
-    def all_classes(self):
-        return list(self._registry.keys())
-    
-    def __call__(self, name: str) -> Any:
-        return self.get_class(name)
-
-dataset_registry = PluginRegistry()
-api_registry = PluginRegistry()
-
-def register_dataset(name: str):
-    def class_decorator(cls):
-        dataset_registry.register(name, cls)
-        return cls
-    return class_decorator
-
-def register_api(name: str):
-    def class_decorator(cls):
-        api_registry.register(name, cls)
-        return cls
-    return class_decorator

evalscope/perf/query_parameters.py

@@ -1,42 +0,0 @@
-from dataclasses import dataclass
-from typing import Optional
-
-
-@dataclass
-class QueryParameters:
-    model: str
-    prompt: Optional[str]
-    dataset: Optional[str]
-    query_template: Optional[str]
-    dataset_path: Optional[str]
-    frequency_penalty: Optional[float]
-    logprobs: Optional[bool]
-    max_tokens: Optional[int]
-    n_choices: Optional[int]
-    seed: Optional[int]
-    stop: Optional[str]
-    stream: Optional[bool]
-    temperature: Optional[float]
-    top_p: Optional[float]
-    max_prompt_length: Optional[int]
-    min_prompt_length: Optional[int]    
-    include_usage: Optional[bool]
-    
-    def __init__(self, args):
-        self.model = args.model
-        self.prompt = args.prompt
-        self.dataset = args.dataset
-        self.query_template = args.query_template
-        self.dataset_path = args.dataset_path
-        self.frequency_penalty = args.frequency_penalty
-        self.logprobs = args.logprobs
-        self.max_tokens = args.max_tokens
-        self.n_choices = args.n_choices
-        self.seed = args.seed
-        self.stop = args.stop
-        self.stream = args.stream
-        self.temperature = args.temperature
-        self.top_p = args.top_p
-        self.max_prompt_length = args.max_prompt_length
-        self.min_prompt_length = args.min_prompt_length
-        self.stop_token_ids = args.stop_token_ids

evalscope/perf/server_sent_event.py

@@ -1,43 +0,0 @@
-from dataclasses import dataclass
-
-@dataclass
-class ServerSentEvent(object):
-    def __init__(self, data='', event=None, id=None, retry=None):
-        self.data = data
-        self.event = event
-        self.id = id 
-        self.retry = retry
-
-    @classmethod
-    def decode(cls, line):
-        """ Decode line to ServerSentEvent
-
-
-        Args:
-            line (str): The line.
-
-        Return:
-            ServerSentEvent (obj:`ServerSentEvent`): The ServerSentEvent object.
-
-        """
-        if not line:
-            return None
-        sse_msg = cls()
-        # format data:xxx
-        field_type, _, field_value = line.partition(":")
-        if field_value.startswith(" "): # compatible with openai api
-            field_value = field_value[1:]
-        if field_type == "event":
-            sse_msg.event = field_value
-        elif field_type == "data":
-            field_value = field_value.rstrip()
-            sse_msg.data = field_value
-        elif field_type == "id":
-            sse_msg.id = field_value
-        elif field_type == "retry":
-            sse_msg.retry = field_value
-        else:
-            pass
-
-        return sse_msg
-

evalscope/preprocess/tokenizers/gpt2_tokenizer.py

@@ -1,221 +0,0 @@
-import logging
-import sys
-from functools import lru_cache
-from typing import Sequence
-
-import json
-import regex as re
-
-logger = logging.getLogger(__name__)
-
-
-def get_pairs(word):
-    """Return set of symbol pairs in a word.
-
-    Word is represented as tuple of symbols (symbols being variable-length strings).
-    """
-    pairs = set()
-    prev_char = word[0]
-    for char in word[1:]:
-        pairs.add((prev_char, char))
-        prev_char = char
-    return pairs
-
-
-@lru_cache()
-def bytes_to_unicode():
-    """
-    Returns list of utf-8 byte and a corresponding list of unicode strings.
-    The reversible bpe codes work on unicode strings.
-    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
-    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
-    This is a signficant percentage of your normal, say, 32K bpe vocab.
-    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
-    And avoids mapping to whitespace/control characters the bpe code barfs on.
-    """
-    bs = list(range(ord('!'),
-                    ord('~') + 1)) + list(range(
-                        ord('¡'),
-                        ord('¬') + 1)) + list(range(ord('®'),
-                                                    ord('ÿ') + 1))
-    cs = bs[:]
-    n = 0
-    for b in range(2**8):
-        if b not in bs:
-            bs.append(b)
-            cs.append(2**8 + n)
-            n += 1
-    cs = [chr(n) for n in cs]
-    return dict(zip(bs, cs))
-
-
-class GPT2Tokenizer(object):
-    """
-    GPT-2 BPE tokenizer. Peculiarities:
-        - Byte-level BPE
-
-    [NOTE]: Copied from megatron.tokenizer.gpt2_tokenization.GPT2Tokenizer.
-    """
-
-    def __init__(self,
-                 vocab_file,
-                 merges_file,
-                 errors='replace',
-                 special_tokens=None,
-                 max_len=None):
-        assert sys.version_info[0] != 2
-
-        self.max_len = max_len if max_len is not None else int(1e12)
-        self.encoder = json.load(open(vocab_file))
-        self.decoder = {v: k for k, v in self.encoder.items()}
-        self.errors = errors  # how to handle errors in decoding
-        self.byte_encoder = bytes_to_unicode()
-        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
-        bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1]
-        bpe_merges = [tuple(merge.split()) for merge in bpe_data]
-        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
-        self.cache = {}
-
-        # Should haved added re.IGNORECASE so BPE merges can happen for
-        # capitalized versions of contractions
-        self.pat = re.compile(
-            r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
-        )
-
-        self.special_tokens = {}
-        self.special_tokens_decoder = {}
-        self.set_special_tokens(special_tokens)
-
-    def __len__(self):
-        return len(self.encoder) + len(self.special_tokens)
-
-    def set_special_tokens(self, special_tokens):
-        """ Add a list of additional tokens to the encoder.
-            The additional tokens are indexed starting from the last index of the
-            current vocabulary in the order of the `special_tokens` list.
-        """
-        if not special_tokens:
-            self.special_tokens = {}
-            self.special_tokens_decoder = {}
-            return
-        self.special_tokens = dict((tok, len(self.encoder) + i)
-                                   for i, tok in enumerate(special_tokens))
-        self.special_tokens_decoder = {
-            v: k
-            for k, v in self.special_tokens.items()
-        }
-        logger.info('Special tokens {}'.format(self.special_tokens))
-
-    def bpe(self, token):
-        if token in self.cache:
-            return self.cache[token]
-        word = tuple(token)
-        pairs = get_pairs(word)
-
-        if not pairs:
-            return token
-
-        while True:
-            bigram = min(
-                pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
-            if bigram not in self.bpe_ranks:
-                break
-            first, second = bigram
-            new_word = []
-            i = 0
-            while i < len(word):
-                try:
-                    j = word.index(first, i)
-                    new_word.extend(word[i:j])
-                    i = j
-                except BaseException:
-                    new_word.extend(word[i:])
-                    break
-
-                if word[i] == first and i < len(word) - 1 and word[
-                        i + 1] == second:
-                    new_word.append(first + second)
-                    i += 2
-                else:
-                    new_word.append(word[i])
-                    i += 1
-            new_word = tuple(new_word)
-            word = new_word
-            if len(word) == 1:
-                break
-            else:
-                pairs = get_pairs(word)
-        word = ' '.join(word)
-        self.cache[token] = word
-        return word
-
-    def tokenize(self, text):
-        """ Tokenize a string. """
-        bpe_tokens = []
-        for token in re.findall(self.pat, text):
-            if sys.version_info[0] == 2:
-                token = ''.join(self.byte_encoder[ord(b)] for b in token)
-            else:
-                token = ''.join(self.byte_encoder[b]
-                                for b in token.encode('utf-8'))
-            bpe_tokens.extend(
-                bpe_token for bpe_token in self.bpe(token).split(' '))
-        return bpe_tokens
-
-    def convert_tokens_to_ids(self, tokens):
-        """ Converts a sequence of tokens into ids using the vocab. """
-        ids = []
-        if isinstance(tokens, str):
-            if tokens in self.special_tokens:
-                return self.special_tokens[tokens]
-            else:
-                return self.encoder.get(tokens, 0)
-        for token in tokens:
-            if token in self.special_tokens:
-                ids.append(self.special_tokens[token])
-            else:
-                ids.append(self.encoder.get(token, 0))
-        if len(ids) > self.max_len:
-            logger.warning(
-                'Token indices sequence length is longer than the specified maximum '
-                ' sequence length for this OpenAI GPT model ({} > {}). Running this'
-                ' sequence through the model will result in indexing errors'.
-                format(len(ids), self.max_len))
-        return ids
-
-    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
-        """Converts a sequence of ids in BPE tokens using the vocab."""
-        tokens = []
-        for i in ids:
-            if i in self.special_tokens_decoder:
-                if not skip_special_tokens:
-                    tokens.append(self.special_tokens_decoder[i])
-            else:
-                tokens.append(self.decoder[i])
-        return tokens
-
-    def encode(self, text):
-        return self.convert_tokens_to_ids(self.tokenize(text))
-
-    def decode(self, tokens):
-        text = ''.join([self.decoder[token] for token in tokens])
-        text = bytearray([self.byte_decoder[c] for c in text]).decode(
-            'utf-8', errors=self.errors)
-        return text
-
-
-class DummyTokenizer:
-
-    def tokenize(self, text: str):
-        return text.split()
-
-
-def get_tokenized_string(tokenizer: GPT2Tokenizer, text_list: Sequence[str]):
-    token_ids_list, tokenized_string_list = [], []
-    for text in text_list:
-        assert tokenizer is not None
-        token_ids = tokenizer.encode(text)
-        tokenized_string = ' '.join(tokenizer.convert_ids_to_tokens(token_ids))
-        token_ids_list.append(token_ids)
-        tokenized_string_list.append(tokenized_string)
-    return token_ids_list, tokenized_string_list