evalscope 0.5.0__py3-none-any.whl

evalscope/perf/openai_api.py

@@ -0,0 +1,130 @@
+from typing import Any, Dict, Iterator, List
+import json
+from evalscope.perf.api_plugin_base import ApiPluginBase
+from transformers import AutoTokenizer
+from evalscope.perf.plugin_registry import register_api
+from evalscope.perf.query_parameters import QueryParameters
+
+@register_api("openai")
+class OpenaiPlugin(ApiPluginBase):
+    """Base of openai interface.
+    """
+    def __init__(self, mode_path: str):
+        """Init the plugin
+
+        Args:
+            mode_path (str): The model path, we use the tokenizer 
+                weight in the model to calculate the number of the
+                input and output tokens.
+        """
+        super().__init__(model_path=mode_path)
+        if mode_path is not None:
+            self.tokenizer = AutoTokenizer.from_pretrained(mode_path)
+        else:
+            self.tokenizer = None
+
+    def build_request(self, messages: List[Dict], param: QueryParameters) -> Dict:
+        """Build the openai format request based on prompt, dataset
+
+        Args:
+            message (Dict): The basic message to generator query.
+            param (QueryParameters): The query parameters.
+
+        Raises:
+            Exception: NotImplemented
+
+        Returns:
+            Dict: The request body. None if prompt format is error.
+        """
+        try:
+            if param.query_template is not None:
+                query = json.loads(param.query_template)
+                query['messages'] = messages   # replace template messages with input messages.
+                return self.__compose_query_from_parameter(query, param)
+            else:
+                query = {'messages': messages}
+                return self.__compose_query_from_parameter(query, param)
+        except Exception as e:
+            print(e)
+            return None
+        
+    def __compose_query_from_parameter(self, payload: Dict, param: QueryParameters):
+        payload['model'] = param.model
+        if param.max_tokens is not None:
+            payload['max_tokens'] = param.max_tokens
+        if param.frequency_penalty is not None:
+            payload['frequency_penalty'] = param.frequency_penalty
+        if param.logprobs is not None:
+            payload['logprobs'] = param.logprobs
+        if param.n_choices is not None:
+            payload['n'] = param.n_choices
+        if param.seed is not None:
+            payload['seed'] = param.seed
+        if param.stop is not None:
+            payload['stop'] = param.stop
+        if param.stream is not None and param.stream:
+            payload['stream'] = param.stream
+            payload['stream_options'] = {"include_usage": True}
+        if param.stop_token_ids is not None:
+            payload['stop_token_ids'] = param.stop_token_ids
+        if param.temperature is not None:
+            payload['temperature'] = param.temperature
+        if param.top_p is not None:
+            payload['top_p'] = param.top_p
+        return payload
+
+    def parse_responses(self, responses, request: Any = None, **kwargs) -> Dict:
+        """Parser responses and return number of request and response tokens.
+           sample of the output delta:
+           {"id":"4","object":"chat.completion.chunk","created":1714030870,"model":"llama3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}]}
+
+
+        Args:
+            responses (List[bytes]): List of http response body, for stream output,
+                there are multiple responses, for general only one. 
+            kwargs: (Any): The command line --parameter content.
+        Returns:
+            Tuple: Return number of prompt token and number of completion tokens.
+        """
+        full_response_content = ''
+        delta_contents = {}
+        input_tokens = None
+        output_tokens = None
+        for response in responses:
+            js = json.loads(response)
+            if js['object'] == 'chat.completion':
+                for choice in js['choices']:
+                    delta_contents[choice['index']] = [choice['message']['content']]     
+                input_tokens = js['usage']['prompt_tokens']
+                output_tokens = js['usage']['completion_tokens']                 
+            else:  # 'object' == "chat.completion.chunk":
+                if 'choices' in js:
+                    for choice in js['choices']:
+                        if 'delta' in choice and 'index' in choice:
+                            delta = choice['delta']
+                            idx = choice['index']
+                            if 'content' in delta:
+                                delta_content = delta['content']
+                                if idx in delta_contents:
+                                    delta_contents[idx].append(delta_content)
+                                else:
+                                    delta_contents[idx] = [delta_content]
+                # usage in chunk: {"id":"","object":"chat.completion.chunk","created":1718269986,"model":"llama3",
+                # "choices":[],"usage":{"prompt_tokens":32,"total_tokens":384,"completion_tokens":352}}
+                if 'usage' in js and js['usage']:
+                    input_tokens = js['usage']['prompt_tokens']
+                    output_tokens = js['usage']['completion_tokens']     
+        if input_tokens is None and output_tokens is None and self.tokenizer is not None:                
+            input_tokens = 0
+            output_tokens = 0
+            for idx, choice_contents in delta_contents.items():
+                full_response_content = ''.join([m for m in choice_contents])
+                input_tokens += len(self.tokenizer.encode(request['messages'][0]['content']))
+                output_tokens += len(self.tokenizer.encode(full_response_content))
+        elif input_tokens is None and output_tokens is None:  # no usage info get.
+            input_tokens = 0
+            output_tokens = 0            
+        
+        return input_tokens, output_tokens
+        
+        

evalscope/perf/plugin_registry.py

@@ -0,0 +1,35 @@
+
+from typing import Any
+
+
+class PluginRegistry:
+    def __init__(self):
+        self._registry = {}
+
+    def register(self, name, cls):
+        self._registry[name] = cls
+        return cls
+
+    def get_class(self, name):
+        return self._registry[name]
+
+    def all_classes(self):
+        return list(self._registry.keys())
+    
+    def __call__(self, name: str) -> Any:
+        return self.get_class(name)
+
+dataset_registry = PluginRegistry()
+api_registry = PluginRegistry()
+
+def register_dataset(name: str):
+    def class_decorator(cls):
+        dataset_registry.register(name, cls)
+        return cls
+    return class_decorator
+
+def register_api(name: str):
+    def class_decorator(cls):
+        api_registry.register(name, cls)
+        return cls
+    return class_decorator

evalscope/perf/query_parameters.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class QueryParameters:
+    model: str
+    prompt: Optional[str]
+    dataset: Optional[str]
+    query_template: Optional[str]
+    dataset_path: Optional[str]
+    frequency_penalty: Optional[float]
+    logprobs: Optional[bool]
+    max_tokens: Optional[int]
+    n_choices: Optional[int]
+    seed: Optional[int]
+    stop: Optional[str]
+    stream: Optional[bool]
+    temperature: Optional[float]
+    top_p: Optional[float]
+    max_prompt_length: Optional[int]
+    min_prompt_length: Optional[int]    
+    include_usage: Optional[bool]
+    
+    def __init__(self, args):
+        self.model = args.model
+        self.prompt = args.prompt
+        self.dataset = args.dataset
+        self.query_template = args.query_template
+        self.dataset_path = args.dataset_path
+        self.frequency_penalty = args.frequency_penalty
+        self.logprobs = args.logprobs
+        self.max_tokens = args.max_tokens
+        self.n_choices = args.n_choices
+        self.seed = args.seed
+        self.stop = args.stop
+        self.stream = args.stream
+        self.temperature = args.temperature
+        self.top_p = args.top_p
+        self.max_prompt_length = args.max_prompt_length
+        self.min_prompt_length = args.min_prompt_length
+        self.stop_token_ids = args.stop_token_ids

evalscope/perf/server_sent_event.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+
+@dataclass
+class ServerSentEvent(object):
+    def __init__(self, data='', event=None, id=None, retry=None):
+        self.data = data
+        self.event = event
+        self.id = id 
+        self.retry = retry
+
+    @classmethod
+    def decode(cls, line):
+        """ Decode line to ServerSentEvent
+
+
+        Args:
+            line (str): The line.
+
+        Return:
+            ServerSentEvent (obj:`ServerSentEvent`): The ServerSentEvent object.
+
+        """
+        if not line:
+            return None
+        sse_msg = cls()
+        # format data:xxx
+        field_type, _, field_value = line.partition(":")
+        if field_value.startswith(" "): # compatible with openai api
+            field_value = field_value[1:]
+        if field_type == "event":
+            sse_msg.event = field_value
+        elif field_type == "data":
+            field_value = field_value.rstrip()
+            sse_msg.data = field_value
+        elif field_type == "id":
+            sse_msg.id = field_value
+        elif field_type == "retry":
+            sse_msg.retry = field_value
+        else:
+            pass
+
+        return sse_msg
+

evalscope/preprocess/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

evalscope/preprocess/tokenizers/gpt2_tokenizer.py

@@ -0,0 +1,221 @@
+import logging
+import sys
+from functools import lru_cache
+from typing import Sequence
+
+import json
+import regex as re
+
+logger = logging.getLogger(__name__)
+
+
+def get_pairs(word):
+    """Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a corresponding list of unicode strings.
+    The reversible bpe codes work on unicode strings.
+    This means you need a large # of unicode characters in your vocab if you want to avoid UNKs.
+    When you're at something like a 10B token dataset you end up needing around 5K for decent coverage.
+    This is a signficant percentage of your normal, say, 32K bpe vocab.
+    To avoid that, we want lookup tables between utf-8 bytes and unicode strings.
+    And avoids mapping to whitespace/control characters the bpe code barfs on.
+    """
+    bs = list(range(ord('!'),
+                    ord('~') + 1)) + list(range(
+                        ord('¡'),
+                        ord('¬') + 1)) + list(range(ord('®'),
+                                                    ord('ÿ') + 1))
+    cs = bs[:]
+    n = 0
+    for b in range(2**8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2**8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+class GPT2Tokenizer(object):
+    """
+    GPT-2 BPE tokenizer. Peculiarities:
+        - Byte-level BPE
+
+    [NOTE]: Copied from megatron.tokenizer.gpt2_tokenization.GPT2Tokenizer.
+    """
+
+    def __init__(self,
+                 vocab_file,
+                 merges_file,
+                 errors='replace',
+                 special_tokens=None,
+                 max_len=None):
+        assert sys.version_info[0] != 2
+
+        self.max_len = max_len if max_len is not None else int(1e12)
+        self.encoder = json.load(open(vocab_file))
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1]
+        bpe_merges = [tuple(merge.split()) for merge in bpe_data]
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        self.cache = {}
+
+        # Should haved added re.IGNORECASE so BPE merges can happen for
+        # capitalized versions of contractions
+        self.pat = re.compile(
+            r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""
+        )
+
+        self.special_tokens = {}
+        self.special_tokens_decoder = {}
+        self.set_special_tokens(special_tokens)
+
+    def __len__(self):
+        return len(self.encoder) + len(self.special_tokens)
+
+    def set_special_tokens(self, special_tokens):
+        """ Add a list of additional tokens to the encoder.
+            The additional tokens are indexed starting from the last index of the
+            current vocabulary in the order of the `special_tokens` list.
+        """
+        if not special_tokens:
+            self.special_tokens = {}
+            self.special_tokens_decoder = {}
+            return
+        self.special_tokens = dict((tok, len(self.encoder) + i)
+                                   for i, tok in enumerate(special_tokens))
+        self.special_tokens_decoder = {
+            v: k
+            for k, v in self.special_tokens.items()
+        }
+        logger.info('Special tokens {}'.format(self.special_tokens))
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(
+                pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf')))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                    new_word.extend(word[i:j])
+                    i = j
+                except BaseException:
+                    new_word.extend(word[i:])
+                    break
+
+                if word[i] == first and i < len(word) - 1 and word[
+                        i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = ' '.join(word)
+        self.cache[token] = word
+        return word
+
+    def tokenize(self, text):
+        """ Tokenize a string. """
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            if sys.version_info[0] == 2:
+                token = ''.join(self.byte_encoder[ord(b)] for b in token)
+            else:
+                token = ''.join(self.byte_encoder[b]
+                                for b in token.encode('utf-8'))
+            bpe_tokens.extend(
+                bpe_token for bpe_token in self.bpe(token).split(' '))
+        return bpe_tokens
+
+    def convert_tokens_to_ids(self, tokens):
+        """ Converts a sequence of tokens into ids using the vocab. """
+        ids = []
+        if isinstance(tokens, str):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.encoder.get(tokens, 0)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.encoder.get(token, 0))
+        if len(ids) > self.max_len:
+            logger.warning(
+                'Token indices sequence length is longer than the specified maximum '
+                ' sequence length for this OpenAI GPT model ({} > {}). Running this'
+                ' sequence through the model will result in indexing errors'.
+                format(len(ids), self.max_len))
+        return ids
+
+    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):
+        """Converts a sequence of ids in BPE tokens using the vocab."""
+        tokens = []
+        for i in ids:
+            if i in self.special_tokens_decoder:
+                if not skip_special_tokens:
+                    tokens.append(self.special_tokens_decoder[i])
+            else:
+                tokens.append(self.decoder[i])
+        return tokens
+
+    def encode(self, text):
+        return self.convert_tokens_to_ids(self.tokenize(text))
+
+    def decode(self, tokens):
+        text = ''.join([self.decoder[token] for token in tokens])
+        text = bytearray([self.byte_decoder[c] for c in text]).decode(
+            'utf-8', errors=self.errors)
+        return text
+
+
+class DummyTokenizer:
+
+    def tokenize(self, text: str):
+        return text.split()
+
+
+def get_tokenized_string(tokenizer: GPT2Tokenizer, text_list: Sequence[str]):
+    token_ids_list, tokenized_string_list = [], []
+    for text in text_list:
+        assert tokenizer is not None
+        token_ids = tokenizer.encode(text)
+        tokenized_string = ' '.join(tokenizer.convert_ids_to_tokens(token_ids))
+        token_ids_list.append(token_ids)
+        tokenized_string_list.append(tokenized_string)
+    return token_ids_list, tokenized_string_list

evalscope/registry/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

evalscope/registry/tasks/arc.yaml

@@ -0,0 +1,29 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args:
+  arc:
+    prompt_template: 'The following are multiple choice questions, please output correct answer in the form of A or B or C or D, do not output explanation:'
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - arc
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/bbh.yaml

@@ -0,0 +1,27 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - bbh
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/bbh_mini.yaml

@@ -0,0 +1,27 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {'bbh': {'subset_list': ['temporal_sequences', 'multistep_arithmetic_two']}}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - bbh
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/ceval.yaml

@@ -0,0 +1,27 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - ceval
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/ceval_mini.yaml

@@ -0,0 +1,27 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {'ceval': {'subset_list': ['computer_network', 'operating_system']}}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - ceval
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/cmmlu.yaml

@@ -0,0 +1,27 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: default
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - cmmlu
+outputs: null    # structure: configs, logs, predictions, reviews, reports # TODO: need to parse
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: null

evalscope/registry/tasks/eval_qwen-7b-chat_v100.yaml

@@ -0,0 +1,28 @@
+model_args:    # model args should be followed by benchmark requirements
+  revision: v1.0.0
+  precision: torch.float16
+  device_map: auto
+#  model_name_or_path: qwen/qwen-7b-chat
+generation_config:
+  temperature: 0.3
+  max_length: 2048
+  max_new_tokens: 512
+  top_k: 50
+  top_p: 0.85
+  do_sample: false
+  num_beams: 1
+  repetition_penalty: 1.0
+#  eos_token_id: null
+#  pad_token_id: null
+dataset_args: {}
+dry_run: false
+model: null   # Note: to be implemented as CustomModel
+eval_type: custom
+datasets:
+  - arc
+  - gsm8k
+outputs: ./outputs/eval_qwen-7b-chat_v100    # Directory to save the outputs, structure: logs, predictions, reviews, reports
+use_cache: false
+stage: all
+dataset_hub: ModelScope    # `Local` or `ModelScope`
+limit: 10