evalscope 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

evalscope/metrics/math_accuracy.py

@@ -1,57 +1,200 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import re
-from collections import defaultdict
-from tqdm import tqdm
 
-from evalscope.constants import MetricsConstant
+# Adapted from https://github.com/EleutherAI/lm-evaluation-harness/blob/master/lm_eval/tasks/hendrycks_math.py
+def is_equiv(str1, str2, verbose=False):
+    if str1 is None and str2 is None:
+        print('WARNING: Both None')
+        return True
+    if str1 is None or str2 is None:
+        return False
 
+    try:
+        ss1 = strip_string(str1)
+        ss2 = strip_string(str2)
+        if verbose:
+            print(ss1, ss2)
+        return ss1 == ss2
+    except Exception:
+        return str1 == str2
 
-def get_last_number(s):
-    match = re.search(r'[-+]?\d*\.\d+|\d+', s[::-1])
-    if match:
-        last_digit = match.group()[::-1]
+
+def remove_boxed(s):
+    if '\\boxed ' in s:
+        left = '\\boxed '
+        assert s[:len(left)] == left
+        return s[len(left):]
+
+    left = '\\boxed{'
+
+    assert s[:len(left)] == left
+    assert s[-1] == '}'
+
+    return s[len(left):-1]
+
+
+def last_boxed_only_string(string):
+    idx = string.rfind('\\boxed')
+    if '\\boxed ' in string:
+        return '\\boxed ' + string.split('\\boxed ')[-1].split('$')[0]
+    if idx < 0:
+        idx = string.rfind('\\fbox')
+        if idx < 0:
+            return None
+
+    i = idx
+    right_brace_idx = None
+    num_left_braces_open = 0
+    while i < len(string):
+        if string[i] == '{':
+            num_left_braces_open += 1
+        if string[i] == '}':
+            num_left_braces_open -= 1
+            if num_left_braces_open == 0:
+                right_brace_idx = i
+                break
+        i += 1
+
+    if right_brace_idx is None:
+        retval = None
     else:
-        last_digit = -100000
-    return float(last_digit)
-
-
-def compute_math_accuracy_one_sample(predict, reference):
-    if isinstance(predict, list):
-        predict = predict[0]
-    if isinstance(reference, list):
-        reference = reference[0]
-    predict_number = get_last_number(predict)
-    reference_number = get_last_number(reference)
-    if abs(predict_number - reference_number) <= MetricsConstant.EPSILON:
-        return 1
+        retval = string[idx:right_brace_idx + 1]
+
+    return retval
+
+
+def fix_fracs(string):
+    substrs = string.split('\\frac')
+    new_str = substrs[0]
+    if len(substrs) > 1:
+        substrs = substrs[1:]
+        for substr in substrs:
+            new_str += '\\frac'
+            if substr[0] == '{':
+                new_str += substr
+            else:
+                try:
+                    assert len(substr) >= 2
+                except AssertionError:
+                    return string
+                a = substr[0]
+                b = substr[1]
+                if b != '{':
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += '{' + a + '}{' + b + '}' + post_substr
+                    else:
+                        new_str += '{' + a + '}{' + b + '}'
+                else:
+                    if len(substr) > 2:
+                        post_substr = substr[2:]
+                        new_str += '{' + a + '}' + b + post_substr
+                    else:
+                        new_str += '{' + a + '}' + b
+    string = new_str
+    return string
+
+
+def fix_a_slash_b(string):
+    if len(string.split('/')) != 2:
+        return string
+    a = string.split('/')[0]
+    b = string.split('/')[1]
+    try:
+        a = int(a)
+        b = int(b)
+        assert string == '{}/{}'.format(a, b)
+        new_string = '\\frac{' + str(a) + '}{' + str(b) + '}'
+        return new_string
+    except AssertionError:
+        return string
+
+
+def remove_right_units(string):
+    # "\\text{ " only ever occurs (at least in the val set) when describing units
+    if '\\text{ ' in string:
+        splits = string.split('\\text{ ')
+        assert len(splits) == 2
+        return splits[0]
     else:
-        return 0
-
-
-def compute_math_accuracy(predict_l, reference_l):
-    assert len(predict_l) == len(reference_l)
-    if len(predict_l) == 0:
-        return 0
-    total_cnt = len(predict_l)
-    correct_cnt = 0
-    for predict, reference in zip(predict_l, reference_l):
-        correct_cnt += compute_math_accuracy_one_sample(predict, reference)
-    return {'math accuracy': correct_cnt / total_cnt}
-
-
-def run_math_eval(data_l, md_level=2):
-    print(f"{'#' * md_level} Math Eval(math accuracy)")
-    for data in tqdm(data_l):
-        data['math_accuracy'] = compute_math_accuracy_one_sample(data['gen'], data['target'])
-    task_data_d = defaultdict(list)
-    for data in data_l:
-        for task in data['task_tags']:
-            task_data_d[task].append(data)
-    correct_cnt = sum([data['math_accuracy'] for data in data_l])
-    print(f'[total], count: {len(data_l)}, math accuracy: '
-          f'{correct_cnt / len(data_l) * 100:0.2f}%')
-    for task in task_data_d.keys():
-        correct_cnt = sum([data['math_accuracy'] for data in task_data_d[task]])
-        print(f'[{task}], count: {len(task_data_d[task])}, math accuracy: '
-              f'{correct_cnt / len(task_data_d[task]) * 100:0.2f}%')
+        return string
+
+
+def fix_sqrt(string):
+    if '\\sqrt' not in string:
+        return string
+    splits = string.split('\\sqrt')
+    new_string = splits[0]
+    for split in splits[1:]:
+        if split[0] != '{':
+            a = split[0]
+            new_substr = '\\sqrt{' + a + '}' + split[1:]
+        else:
+            new_substr = '\\sqrt' + split
+        new_string += new_substr
+    return new_string
+
+
+def strip_string(string):
+    # linebreaks
+    string = string.replace('\n', '')
+
+    # remove inverse spaces
+    string = string.replace('\\!', '')
+
+    # replace \\ with \
+    string = string.replace('\\\\', '\\')
+
+    # replace tfrac and dfrac with frac
+    string = string.replace('tfrac', 'frac')
+    string = string.replace('dfrac', 'frac')
+
+    # remove \left and \right
+    string = string.replace('\\left', '')
+    string = string.replace('\\right', '')
+
+    # Remove circ (degrees)
+    string = string.replace('^{\\circ}', '')
+    string = string.replace('^\\circ', '')
+
+    # remove dollar signs
+    string = string.replace('\\$', '')
+
+    # remove units (on the right)
+    string = remove_right_units(string)
+
+    # remove percentage
+    string = string.replace('\\%', '')
+    string = string.replace('\%', '')  # noqa: W605
+
+    # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
+    string = string.replace(' .', ' 0.')
+    string = string.replace('{.', '{0.')
+    # if empty, return empty string
+    if len(string) == 0:
+        return string
+    if string[0] == '.':
+        string = '0' + string
+
+    # to consider: get rid of e.g. "k = " or "q = " at beginning
+    if len(string.split('=')) == 2:
+        if len(string.split('=')[0]) <= 2:
+            string = string.split('=')[1]
+
+    # fix sqrt3 --> sqrt{3}
+    string = fix_sqrt(string)
+
+    # remove spaces
+    string = string.replace(' ', '')
+
+    # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b}  # noqa: E501
+    string = fix_fracs(string)
+
+    # manually change 0.5 --> \frac{1}{2}
+    if string == '0.5':
+        string = '\\frac{1}{2}'
+
+    # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y
+    string = fix_a_slash_b(string)
+
+    return string

evalscope/metrics/metrics.py

@@ -2,16 +2,12 @@
 # Copyright (c) EleutherAI. and its affiliates.
 # Copyright (c) OpenAI. and its affiliates.
 import itertools
-import jieba
 import math
 import numpy as np
 import random
 import sacrebleu
-import sklearn.metrics
 from collections import defaultdict
 from collections.abc import Iterable
-from nltk import word_tokenize
-from nltk.translate.bleu_score import sentence_bleu
 from typing import Dict, List, Union
 
 
@@ -38,6 +34,8 @@ def median(arr):
 
 
 def matthews_corrcoef(items):
+    import sklearn.metrics
+
     unzipped_list = list(zip(*items))
     golds = unzipped_list[0]
     preds = unzipped_list[1]
@@ -45,6 +43,8 @@ def matthews_corrcoef(items):
 
 
 def f1_score(items):
+    import sklearn.metrics
+
     unzipped_list = list(zip(*items))
     golds = unzipped_list[0]
     preds = unzipped_list[1]
@@ -150,6 +150,9 @@ def bleu_ngram_one_sample(predict, reference):
         }
 
     """
+    import jieba
+    from nltk import word_tokenize
+    from nltk.translate.bleu_score import sentence_bleu
 
     def is_contains_chinese(strs):
         for _char in strs:

evalscope/metrics/rouge_metric.py

@@ -1,15 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 import jieba
-import logging
 from collections import defaultdict
-from pathlib import Path
 from rouge_chinese import Rouge
 from statistics import mean
 from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
 from evalscope.metrics.bundled_rouge_score import rouge_scorer
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
 
 
 class DummyTokenizer:
@@ -18,10 +19,6 @@ class DummyTokenizer:
         return text.split()
 
 
-HERE = Path(__file__).absolute().parent
-
-logger = logging.getLogger(__name__)
-
 scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
 zh_scorer = Rouge()
 
@@ -58,7 +55,11 @@ def compute_rouge_score_one_sample_zh(predict, reference):
         p = ' '.join(jieba.cut(p)) if is_contains_chinese(p) else p
         r = ' '.join(jieba.cut(r)) if is_contains_chinese(r) else r
 
-        score = zh_scorer.get_scores(p, r)[0]
+        try:
+            score = zh_scorer.get_scores(p, r, ignore_empty=True)[0]
+        except Exception as e:
+            logger.warning(f'rouge score error: {p} {r} {e}')
+            continue
         result['rouge-1-r'] = score['rouge-1']['r']
         result['rouge-1-p'] = score['rouge-1']['p']
         result['rouge-1-f'] = score['rouge-1']['f']
@@ -75,7 +76,11 @@ def compute_rouge_score_one_sample_zh(predict, reference):
 def compute_rouge_score_one_sample(predict, reference):
     result = dict()
     for p, r in zip(predict, reference):
-        score = scorer.score(p, r)
+        try:
+            score = scorer.score(p, r)
+        except Exception as e:
+            logger.warning(f'rouge score error: {p} {r} {e}')
+            continue
         result['rouge-1-r'] = score['rouge1'].recall
         result['rouge-1-p'] = score['rouge1'].precision
         result['rouge-1-f'] = score['rouge1'].fmeasure

evalscope/models/__init__.py

@@ -1,3 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.models.model import BaseModel, ChatBaseModel
+from evalscope.models.base_adapter import BaseModelAdapter, initialize_model_adapter
+from evalscope.models.chat_adapter import ChatGenerationModelAdapter
+from evalscope.models.choice_adapter import ContinuationLogitsModelAdapter, MultiChoiceModelAdapter
+from evalscope.models.custom import CustomModel
+from evalscope.models.custom_adapter import CustomModelAdapter
+from evalscope.models.local_model import LocalModel, get_local_model
+from evalscope.models.model import BaseModel, ChatBaseModel, OpenAIModel
+from evalscope.models.server_adapter import ServerModelAdapter
+
+__all__ = [
+    'CustomModel', 'BaseModel', 'ChatBaseModel', 'OpenAIModel', 'BaseModelAdapter', 'ChatGenerationModelAdapter',
+    'MultiChoiceModelAdapter', 'ContinuationLogitsModelAdapter', 'CustomModelAdapter', 'ServerModelAdapter',
+    'LocalModel', 'get_local_model', 'initialize_model_adapter'
+]

evalscope/models/base_adapter.py

@@ -0,0 +1,52 @@
+import torch
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Optional, Union
+
+from evalscope.constants import EvalType
+from evalscope.models.custom import CustomModel
+from evalscope.models.local_model import LocalModel
+
+if TYPE_CHECKING:
+    from evalscope.config import TaskConfig
+
+
+class BaseModelAdapter(ABC):
+
+    def __init__(self, model: Optional[Union[LocalModel, CustomModel]], **kwargs):
+        if model is None:
+            self.model_cfg = kwargs.get('model_cfg', None)
+        elif isinstance(model, LocalModel):
+            self.model = model.model
+            self.model_id = model.model_id
+            self.model_revision = model.model_revision
+            self.device = model.device
+            self.tokenizer = model.tokenizer
+            self.model_cfg = model.model_cfg
+        elif isinstance(model, CustomModel):
+            self.model_cfg = model.config
+        else:
+            raise ValueError(f'Unsupported model type: {type(model)}')
+
+    @abstractmethod
+    @torch.no_grad()
+    def predict(self, *args, **kwargs) -> Any:
+        raise NotImplementedError
+
+
+def initialize_model_adapter(task_cfg: 'TaskConfig', model_adapter_cls: 'BaseModelAdapter', base_model: 'LocalModel'):
+    """Initialize the model adapter based on the task configuration."""
+    if task_cfg.dry_run:
+        from evalscope.models.model import DummyChatModel
+        return DummyChatModel(model_cfg=dict())
+    elif task_cfg.eval_type == EvalType.CUSTOM:
+        if not isinstance(task_cfg.model, CustomModel):
+            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
+        from evalscope.models import CustomModelAdapter
+        return CustomModelAdapter(custom_model=task_cfg.model)
+    elif task_cfg.eval_type == EvalType.SERVICE:
+        from evalscope.models import ServerModelAdapter
+        return ServerModelAdapter(
+            api_url=task_cfg.api_url, model_id=task_cfg.model, api_key=task_cfg.api_key, seed=task_cfg.seed)
+    else:
+        return model_adapter_cls(
+            model=base_model, generation_config=task_cfg.generation_config, chat_template=task_cfg.chat_template)

evalscope/models/chat_adapter.py

@@ -0,0 +1,138 @@
+import os
+import time
+import torch
+from typing import Union
+
+from evalscope.models.base_adapter import BaseModelAdapter
+from evalscope.models.local_model import LocalModel
+from evalscope.utils.chat_service import ChatCompletionResponse, ChatCompletionResponseChoice, ChatMessage
+from evalscope.utils.logger import get_logger
+from evalscope.utils.model_utils import fix_do_sample_warning
+
+logger = get_logger()
+
+
+class ChatGenerationModelAdapter(BaseModelAdapter):
+    """
+    Chat generation model adapter.
+    """
+
+    def __init__(self, model: LocalModel, **kwargs):
+        super().__init__(model)
+
+        self.generation_config = self._parse_generation_config(self.tokenizer, self.model)
+
+        custom_generation_config = kwargs.pop('generation_config', None)
+        custom_chat_template = kwargs.pop('chat_template', None)
+
+        if custom_generation_config:
+            logger.info('Updating generation config ...')
+            self.generation_config.update(**custom_generation_config)
+
+        if custom_chat_template:
+            self.tokenizer.chat_template = custom_chat_template
+            logger.info(f'Using custom chat template: {custom_chat_template}')
+
+    def _parse_generation_config(self, tokenizer, model):
+        from modelscope import GenerationConfig
+
+        generation_config = getattr(model, 'generation_config', GenerationConfig(do_sample=False))
+
+        try:
+            remote_config = GenerationConfig.from_pretrained(
+                self.model_id, revision=self.model_revision, trust_remote_code=True)
+            generation_config.update(**remote_config.to_dict())
+        except Exception:
+            logger.warning(f'Failed to get generation config of {self.model_id} from model hub, use default.')
+
+        if isinstance(self.model_id, str) and os.path.exists(self.model_id):
+            logger.warning(f'Got local model dir: {self.model_id}')
+
+        if tokenizer.eos_token_id is not None:
+            generation_config.eos_token_id = tokenizer.eos_token_id
+        if tokenizer.pad_token_id is not None:
+            generation_config.pad_token_id = tokenizer.pad_token_id
+        if generation_config.max_new_tokens is None:
+            generation_config.max_new_tokens = 2048
+
+        return generation_config
+
+    def _model_generate(self, query: str, system_prompt: str = None, infer_cfg: dict = {}) -> str:
+        """
+        Args:
+            query: The input query.
+            system_prompt: The system prompt.
+            infer_cfg: The inference configuration.
+        Returns:
+            The prediction result.
+        """
+        # For chat model, use the chat template to format the input
+        if self.tokenizer.chat_template is not None:
+            messages = [ChatMessage(role='user', content=query)]
+            if system_prompt:
+                messages = [ChatMessage(role='system', content=system_prompt)] + messages
+            formatted_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        else:
+            # For base model, use the query as the input
+            formatted_prompt = query
+
+        inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=True).to(self.device)
+        input_ids = inputs['input_ids']
+
+        # Process infer_cfg
+        if isinstance(infer_cfg.get('num_return_sequences'), int) and infer_cfg['num_return_sequences'] > 1:
+            infer_cfg['do_sample'] = True
+
+        # stop settings
+        stop = infer_cfg.get('stop', None)
+        eos_token_id = self.tokenizer.encode(stop, add_special_tokens=False)[0] \
+            if stop else self.tokenizer.eos_token_id
+
+        if eos_token_id is not None:
+            infer_cfg['eos_token_id'] = eos_token_id
+            infer_cfg['pad_token_id'] = eos_token_id  # setting eos_token_id as pad token
+
+        self.generation_config.update(**infer_cfg)
+        fix_do_sample_warning(self.generation_config)
+
+        # Run inference
+        output_ids = self.model.generate(**inputs, generation_config=self.generation_config)
+
+        response = self.tokenizer.decode(output_ids[0, len(input_ids[0]):], skip_special_tokens=True)
+        return response
+
+    @torch.no_grad()
+    def predict(self, inputs: Union[str, dict, list], infer_cfg: dict = {}) -> dict:
+        """
+        Args:
+            inputs: The input data.
+            infer_cfg: The inference configuration.
+        Returns:
+            The prediction result.
+        """
+
+        # Process inputs
+        if isinstance(inputs, str):
+            query = inputs
+            system_prompt = None
+        elif isinstance(inputs, dict):
+            query = inputs['data'][0]
+            system_prompt = inputs.get('system_prompt', None)
+        elif isinstance(inputs, list):
+            query = '\n'.join(inputs)
+            system_prompt = None
+        else:
+            raise TypeError(f'Unsupported inputs type: {type(inputs)}')
+
+        response = self._model_generate(query, system_prompt, infer_cfg)
+
+        choices_list = [
+            ChatCompletionResponseChoice(
+                index=0, message=ChatMessage(content=response, role='assistant'), finish_reason='stop')
+        ]
+
+        res_d = ChatCompletionResponse(
+            model=self.model_id, choices=choices_list, object='chat.completion', created=int(time.time()),
+            usage=None).model_dump(exclude_unset=True)
+
+        return res_d