evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/utils/logger.py

@@ -1,37 +1,74 @@
+import colorlog
 import importlib.util as iutil
 import logging
 import os
-from typing import Optional
+from logging import Logger
+from typing import List, Optional
 
 init_loggers = {}
+# Define log formats
+data_format = '%Y-%m-%d %H:%M:%S'
+# For console output
+color_detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(log_color)s%(levelname)s%(reset)s: %(message)s'  # noqa:E501
+color_simple_format = '%(asctime)s - %(name)s - %(log_color)s%(levelname)s%(reset)s: %(message)s'
+color_detailed_formatter = colorlog.ColoredFormatter(color_detailed_format, datefmt=data_format)
+color_simple_formatter = colorlog.ColoredFormatter(color_simple_format, datefmt=data_format)
+# For file output
+detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s: %(message)s'  # noqa:E501
+simple_format = '%(asctime)s - %(name)s - %(levelname)s: %(message)s'
+plain_detailed_formatter = logging.Formatter(detailed_format, datefmt=data_format)
+plain_simple_formatter = logging.Formatter(simple_format, datefmt=data_format)
+
+DEFAULT_LEVEL = logging.DEBUG if os.getenv('EVALSCOPE_LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
+
+logging.basicConfig(format=simple_format, level=logging.INFO, force=True)
+
+# set logging level
+logging.getLogger('datasets').setLevel(logging.WARNING)
+logging.getLogger('httpx').setLevel(logging.WARNING)
+logging.getLogger('modelscope').setLevel(logging.ERROR)
 
-detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s'
-simple_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+info_set = set()
+warning_set = set()
 
-detailed_formatter = logging.Formatter(detailed_format)
-simple_formatter = logging.Formatter(simple_format)
-DEFAULT_LEVEL = logging.DEBUG if os.getenv('LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
 
-logging.basicConfig(format=simple_format, level=DEFAULT_LEVEL)
+def info_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in info_set:
+        return
+    info_set.add(hash_id)
+    self.info(msg)
 
-# disable datasets logging
-logging.getLogger('datasets').setLevel(logging.WARNING)
-logging.getLogger('modelscope').setLevel(logging.WARNING)
-logging.getLogger('httpx').setLevel(logging.WARNING)
+
+def warning_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in warning_set:
+        return
+    warning_set.add(hash_id)
+    self.warning(msg)
 
 
-def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, file_mode: str = 'w', force=False):
+def get_logger(
+    log_file: Optional[str] = None,
+    name: Optional[str] = None,
+    log_level: int = DEFAULT_LEVEL,
+    file_mode: str = 'w',
+    force: bool = False,
+):
     """Get logging logger
 
     Args:
-        log_file: Log filename, if specified, file handler will be added to
-            logger
-        log_level: Logging level.
-        file_mode: Specifies the mode to open the file, if filename is
-            specified (if filemode is unspecified, it defaults to 'w').
+        log_file: Log filename. If specified, a file handler will be added to the logger.
+        name: Logical component name. Used to derive the logger name.
+        log_level: Logging level to set.
+        file_mode: Mode to open the file when log_file is provided (default 'w').
+        force: If True, reconfigure the existing logger (levels, formatters, handlers).
     """
 
-    logger_name = __name__.split('.')[0]
+    if name:
+        logger_name = f"evalscope.{name.split('.')[-1]}"
+    else:
+        logger_name = 'evalscope'
     logger = logging.getLogger(logger_name)
     logger.propagate = False
 
@@ -40,7 +77,16 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
             logger.setLevel(log_level)
             for handler in logger.handlers:
                 handler.setLevel(log_level)
-                handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
+                # Select formatter by handler type
+                if isinstance(handler, logging.FileHandler):
+                    handler.setFormatter(
+                        plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter
+                    )
+                else:
+                    handler.setFormatter(
+                        color_detailed_formatter if log_level == logging.DEBUG else color_simple_formatter
+                    )
+            # Ensure file handler points to current log_file (replace if needed)
             add_file_handler_if_needed(logger, log_file, file_mode, log_level)
         return logger
 
@@ -62,11 +108,15 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
     handlers = [stream_handler]
 
     if is_worker0 and log_file is not None:
-        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler = logging.FileHandler(log_file, file_mode, encoding='utf-8')
         handlers.append(file_handler)
 
     for handler in handlers:
-        handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
+        # 区分不同类型的 handler，使用相应的格式化器
+        if isinstance(handler, logging.FileHandler):
+            handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
+        else:
+            handler.setFormatter(color_detailed_formatter if log_level == logging.DEBUG else color_simple_formatter)
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
@@ -88,20 +138,60 @@ def configure_logging(debug: bool, log_file: Optional[str] = None):
         get_logger(log_level=logging.DEBUG, force=True)
 
 
-def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
-    for handler in logger.handlers:
-        if isinstance(handler, logging.FileHandler):
-            return
+def add_file_handler_if_needed(
+    logger: logging.Logger,
+    log_file: Optional[str],
+    file_mode: str,
+    log_level: int,
+) -> None:
+    """Ensure logger has a FileHandler targeting log_file.
+    - If no FileHandler exists, add one.
+    - If a FileHandler exists but points to a different file, replace it.
+    """
+    if log_file is None:
+        return
 
+    # Only worker-0 writes files
     if iutil.find_spec('torch') is not None:
         from modelscope.utils.torch_utils import is_master
-
         is_worker0 = is_master()
     else:
         is_worker0 = True
 
-    if is_worker0 and log_file is not None:
-        file_handler = logging.FileHandler(log_file, file_mode)
-        file_handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
-        file_handler.setLevel(log_level)
-        logger.addHandler(file_handler)
+    if not is_worker0:
+        return
+
+    target_path = os.path.abspath(log_file)
+    existing_file_handlers = [h for h in logger.handlers if isinstance(h, logging.FileHandler)]
+
+    # If there is a FileHandler already pointing to the target file, nothing to do.
+    for fh in existing_file_handlers:
+        try:
+            if os.path.abspath(getattr(fh, 'baseFilename', '')) == target_path:
+                return
+        except Exception:
+            # If any issue retrieving baseFilename, fall through to replacement
+            pass
+
+    # Replace all existing FileHandlers with the new one
+    for fh in existing_file_handlers:
+        try:
+            logger.removeHandler(fh)
+            fh.flush()
+            fh.close()
+        except Exception:
+            pass
+
+    file_handler = logging.FileHandler(target_path, file_mode, encoding='utf-8')
+    file_handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
+    file_handler.setLevel(log_level)
+    logger.addHandler(file_handler)
+
+
+def warn_once(logger: Logger, message: str) -> None:
+    if message not in _warned:
+        logger.warning(message)
+        _warned.append(message)
+
+
+_warned: List[str] = []

evalscope/utils/model_utils.py

@@ -1,5 +1,9 @@
+import numpy as np
+import random
 from enum import Enum
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
+
+from evalscope.utils.import_utils import check_import
 
 if TYPE_CHECKING:
     from transformers import GenerationConfig
@@ -22,3 +26,55 @@ def fix_do_sample_warning(generation_config: 'GenerationConfig') -> None:
         generation_config.temperature = 1.
         generation_config.top_p = 1.
         generation_config.top_k = 50
+
+
+def get_device() -> str:
+    from transformers.utils import is_torch_cuda_available, is_torch_mps_available, is_torch_npu_available
+
+    if is_torch_npu_available():
+        device = 'npu'
+    elif is_torch_mps_available():
+        device = 'mps'
+    elif is_torch_cuda_available():
+        device = 'cuda'
+    else:
+        device = 'cpu'
+
+    return device
+
+
+def dict_torch_dtype_to_str(d: Dict[str, Any]) -> dict:
+    """
+        Checks whether the passed dictionary and its nested dicts have a *torch_dtype* key and if it's not None,
+        converts torch.dtype to a string of just the type. For example, `torch.float32` get converted into *"float32"*
+        string, which can then be stored in the json format.
+
+        Refer to: https://github.com/huggingface/transformers/pull/16065/files for details.
+        """
+    if d.get('torch_dtype', None) is not None and not isinstance(d['torch_dtype'], str):
+        d['torch_dtype'] = str(d['torch_dtype']).split('.')[1]
+
+    for value in d.values():
+        if isinstance(value, dict):
+            dict_torch_dtype_to_str(value)
+
+    return d
+
+
+def seed_everything(seed: int):
+    """Set all random seeds to a fixed value for reproducibility.
+
+    Args:
+        seed (int): The seed value.
+    """
+    random.seed(seed)
+    np.random.seed(seed)
+
+    if check_import('torch', raise_warning=False):
+        import torch
+
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+            torch.backends.cudnn.deterministic = True
+            torch.backends.cudnn.benchmark = False

evalscope/utils/multi_choices.py

@@ -0,0 +1,303 @@
+# flake8: noqa: E501
+import re
+from typing import List, Optional, Union
+
+from evalscope.api.evaluator import Choices, Target, TaskState
+
+FEW_SHOT_TEMPLATE = r"""Here are some examples of how to answer similar questions:
+
+{fewshot}
+
+""".lstrip()
+
+CHINESE_FEW_SHOT_TEMPLATE = r"""以下是一些示例问题：
+
+{fewshot}
+
+""".lstrip()
+
+CHINESE_SINGLE_ANSWER_TEMPLATE = r"""回答下面的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式："答案：LETTER"（不带引号），其中 LETTER 是 {letters} 中的一个。
+
+问题：{question}
+选项：
+{choices}
+""".lstrip()
+
+CHINESE_SINGLE_ANSWER_TEMPLATE_COT = r"""回答下面的单项选择题，请选出其中的正确答案。你的回答的最后一行应该是这样的格式："答案：LETTER"（不带引号），其中 LETTER 是 {letters} 中的一个。请在回答前进行一步步思考。
+
+问题：{question}
+选项：
+{choices}
+""".lstrip()
+
+SINGLE_ANSWER_TEMPLATE = r"""
+Answer the following multiple choice question. The entire content of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+
+{choices}
+""".strip()
+
+SINGLE_ANSWER_TEMPLATE_COT = r"""
+Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}. Think step by step before answering.
+
+{question}
+
+{choices}
+""".strip()
+
+MULTIPLE_ANSWER_TEMPLATE = r"""
+Answer the following multiple choice question where multiple answers may be correct. The entire content of your response should be of the following format: 'ANSWER: $LETTERS' (without quotes) where LETTERS is one or more of {letters}.
+
+{question}
+
+{choices}
+""".strip()
+
+MULTIPLE_ANSWER_TEMPLATE_COT = r"""
+Answer the following multiple choice question where multiple answers may be correct. The last line of your response should be of the following format: 'ANSWER: $LETTERS' (without quotes) where LETTERS is one or more of {letters}. Think step by step before answering.
+
+{question}
+
+{choices}
+""".strip()
+
+
+def unshuffle_choices(choices: Choices) -> Choices:
+    # `sorted` returns `list[Choice]`, but for consistency we wrap this back
+    # into a `Choices` object
+    return Choices(sorted(choices, key=lambda choice: choice.original_position))
+
+
+def answer_options(choices: Choices) -> str:
+    r"""
+    Returns the `choices` formatted as a multiple choice question, e.g.:
+
+    ["choice 1", "choice 2", "choice 3"] ->
+        "A) choice 1\nB) choice 2\nC) choice 3"
+    """
+    indexes = list(range(len(choices)))
+
+    return '\n'.join([f'{answer_character(i)}) {choices[j].value}' for i, j in enumerate(indexes)])
+
+
+def format_letter_choices(choices: Union[Choices, List[str]]) -> str:
+    """
+    Returns the `choices` formatted as a letter list, e.g.:
+
+    ["choice 1", "choice 2", "choice 3"] ->
+        "A,B,C"
+    """
+    if isinstance(choices, list):
+        choices = Choices(choices)
+
+    indexes = list(range(len(choices)))
+
+    return ','.join([f'{answer_character(i)}' for i in indexes])
+
+
+def prompt(question: str, choices: Union[Choices, List[str]], template: str, fewshot: Optional[str] = None) -> str:
+    if isinstance(choices, list):
+        choices = Choices(choices)
+
+    choices_text = answer_options(choices)
+    letters = format_letter_choices(choices)
+    if not fewshot:
+        return template.format(
+            choices=choices_text,
+            letters=letters,
+            question=question,
+        )
+    else:
+        return template.format(
+            choices=choices_text,
+            letters=letters,
+            question=question,
+            fewshot=fewshot,
+        )
+
+
+def format_example(
+    question: str,
+    choices: Choices,
+    answer: Target,
+) -> str:
+    """Format a single example for few-shot learning.
+
+    Args:
+        question (str): The question text.
+        choices (list[str]): The list of choices.
+        answer (list[str]): The correct answers.
+
+    Returns:
+        str: Formatted example string.
+    """
+    choices_text = answer_options(choices)
+    return f'{question}\n{choices_text}\nANSWER: {answer.text}'
+
+
+def _fallback_parse_answer(completion: str) -> Optional[set[str]]:
+    # Fallback to find the last upper case letter
+    for letter in reversed(completion):
+        if letter.isupper():
+            return {letter}
+    return None
+
+
+def parse_answers(state: TaskState, multiple_correct: bool = False) -> set[str]:
+    """
+    Convenience function for extracting answers from the state output.
+
+    The generated response must be in the format 'ANSWER: <answers>',
+    otherwise we can't extract what the model thinks is "true". We can be a
+    bit flexible whether these are "AB" vs "A,B" vs "A B".
+
+    However, if the answer isn't in the expected format the model has
+    failed in the task so we'll ultimately just mark it as incorrect
+    """
+    # First check whether the string strictly ends with the expected answer
+    # In this case, we're looking for a single line which contains the expected
+    # ANSWER: <answer> string with only whitespace or a period/full stop at the end.
+    match = re.search(
+        r'(?i)^ANSWER\s*:\s*([A-Za-z\d ,]+)\s*(?:$|\n|\.)',
+        state.output.completion,
+        flags=re.MULTILINE,
+    )
+
+    # If we couldn't match the strict version, we can try the less strict
+    # version for backward compatibility
+    if match is None:
+        match = re.search(
+            r'(?i)ANSWER\s*:\s*([A-Za-z\d ,]+)(?:[^\w]|\n|$|\.)',
+            state.output.completion,
+        )
+
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
+    if match is None:
+        return set()
+
+    matched = match.group(1)
+
+    # Strip trailing period / full stop
+    matched = matched.strip()
+    matched = matched.rstrip('.')
+
+    allowed_options = set(answer_character(i) for i in range(len(state.choices)))
+
+    if multiple_correct:
+        # Match must contain only the allowed choices
+        # (may be separated by commas, spaces, the word 'and', or nothing at all)
+
+        matched = matched.replace(' and ', '')
+
+        matched = matched.replace(' ', '')
+
+        split_comma = set(matched.split(','))
+        if split_comma.issubset(allowed_options):
+            answers = split_comma
+            return answers
+
+        split_nothing = set(matched)
+        if split_nothing.issubset(allowed_options):
+            answers = split_nothing
+            return answers
+
+    else:
+        # Match must contain a single letter in the allowed choices
+        if matched in allowed_options:
+            answers = {matched}
+            return answers
+
+    return set()
+
+
+def parse_answers_zh(state: TaskState, multiple_correct: bool = False) -> set[str]:
+    """
+    Convenience function for extracting answers from the state output in Chinese format.
+
+    The generated response must be in the format '答案：选项',
+    otherwise we can't extract what the model thinks is "true". We can be a
+    bit flexible whether these are "AB" vs "A,B" vs "A B".
+    """
+    # Simple pattern to capture answers with optional bold markdown
+    pattern = r'答案\s*[:：]\s*([A-Za-z0-9,，]+)'
+    match = re.search(pattern, state.output.completion, flags=re.MULTILINE)
+
+    if match is None:
+        fallback_answer = _fallback_parse_answer(state.output.completion)
+        if fallback_answer:
+            return fallback_answer
+
+    if match is None:
+        return set()
+
+    matched = match.group(1).strip().rstrip('。.')
+    allowed_options = set(answer_character(i) for i in range(len(state.choices)))
+
+    if multiple_correct:
+        # Handle comma-separated or continuous letters
+        matched = matched.replace(' 和 ', '').replace(' ', '').replace('，', ',')
+        answers = set(matched.split(',')) if ',' in matched else set(matched)
+        return answers if answers.issubset(allowed_options) else set()
+    else:
+        # Single answer
+        return {matched} if matched in allowed_options else set()
+
+
+def set_choices_based_on_generated_response(state: TaskState, answers: set[str]) -> None:
+    true_answers = [answer_index(letter) for letter in answers]
+
+    for i in range(len(state.choices)):
+        if i in true_answers:
+            state.choices.mark_choice(i, True)
+        else:
+            state.choices.mark_choice(i, False)
+
+
+def valid_template(template: str) -> bool:
+    """Check if a template has the required capture groups for a multiple choice question"""
+    return bool(re.search(r'\{question\}', template) and re.search(r'\{choices\}', template))
+
+
+class MultipleChoiceTemplate:
+    """
+    Templates for multiple choice questions.
+    """
+
+    SINGLE_ANSWER = SINGLE_ANSWER_TEMPLATE
+    SINGLE_ANSWER_COT = SINGLE_ANSWER_TEMPLATE_COT
+    MULTIPLE_ANSWER = MULTIPLE_ANSWER_TEMPLATE
+    MULTIPLE_ANSWER_COT = MULTIPLE_ANSWER_TEMPLATE_COT
+    CHINESE_FEW_SHOT_TEMPLATE = CHINESE_FEW_SHOT_TEMPLATE
+    CHINESE_SINGLE_ANSWER_TEMPLATE = CHINESE_SINGLE_ANSWER_TEMPLATE
+    CHINESE_SINGLE_ANSWER_TEMPLATE_COT = CHINESE_SINGLE_ANSWER_TEMPLATE_COT
+
+
+def answer_character(index: int) -> str:
+    r"""
+    Helper to go from array index to char, for example:
+
+        0 -> 'A', 1 -> 'B', etc
+    """
+    if index < 26:
+        return chr(ord('A') + index)
+    else:
+        return str(index - 25)
+
+
+def answer_index(char: str) -> int:
+    r"""
+    Helper to go from char to array index, for example:
+
+        'A' -> 0, 'B' -> 1, etc
+    """
+    if char.isalpha() or char == ',' or char == ' ':
+        return ord(char.upper()) - ord('A')
+    elif char.isnumeric():
+        return 25 + int(char)
+    else:
+        raise ValueError(f'Unepxected multiple choice answer: {char} (must be a letter or number)')