evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/run.py

@@ -13,9 +13,6 @@ from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.utils.model_utils import seed_everything
 
-if TYPE_CHECKING:
-    from evalscope.models import LocalModel
-
 logger = get_logger()
 
 
@@ -109,27 +106,42 @@ def get_backend_manager_class(eval_backend: EvalBackend):
         raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')
 
 
-def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
+def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
-    from evalscope.models import get_local_model
+    from evalscope.api.evaluator import Evaluator
+    from evalscope.api.model import get_model_with_task_config
+    from evalscope.api.registry import get_benchmark
+    from evalscope.evaluator import DefaultEvaluator
     from evalscope.report import gen_table
 
     # Initialize evaluator
     eval_results = {}
-    base_model = get_local_model(task_cfg)
-    evaluators = []
-    for dataset_name in task_cfg.datasets:
-        evaluator = create_evaluator(task_cfg, dataset_name, outputs, base_model)
+    # Initialize model
+    model = get_model_with_task_config(task_config=task_config)
+    # Initialize evaluators for each dataset
+    evaluators: List[Evaluator] = []
+    for dataset_name in task_config.datasets:
+        # Create evaluator for each dataset
+        benchmark = get_benchmark(dataset_name, task_config)
+        evaluator = DefaultEvaluator(
+            task_config=task_config,
+            model=model,
+            benchmark=benchmark,
+            outputs=outputs,
+        )
         evaluators.append(evaluator)
 
+        # Update task_config.dataset_args with benchmark metadata
+        task_config.dataset_args[dataset_name] = benchmark.to_dict()
+
     # dump task_cfg to outputs.configs_dir after creating evaluators
-    task_cfg.dump_yaml(outputs.configs_dir)
-    logger.info(task_cfg)
+    task_config.dump_yaml(outputs.configs_dir)
+    logger.info(task_config)
 
     # Run evaluation for each evaluator
     for evaluator in evaluators:
         res_dict = evaluator.eval()
-        eval_results[evaluator.dataset_name] = res_dict
+        eval_results[evaluator.benchmark.name] = res_dict
 
     # Make overall report
     try:
@@ -139,11 +151,11 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
         logger.error('Failed to generate report table.')
 
     # Clean up
-    if base_model is not None:
+    if model is not None:
         import gc
         import torch
 
-        del base_model
+        del model
         del evaluators
         torch.cuda.empty_cache()
         gc.collect()
@@ -151,36 +163,6 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     return eval_results
 
 
-def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: 'LocalModel'):
-    """Create an evaluator object for the specified dataset."""
-    from evalscope.benchmarks import Benchmark, BenchmarkMeta
-    from evalscope.evaluator import Evaluator
-    from evalscope.models import initialize_model_adapter
-
-    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
-
-    if dataset_name == DataCollection.NAME:
-        # EvaluatorCollection is a collection of evaluators
-        from evalscope.collections import EvaluatorCollection
-        data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
-        return EvaluatorCollection(task_cfg, data_adapter, outputs, base_model)
-
-    # Initialize data adapter first to update config
-    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
-    # Initialize model adapter
-    model_adapter = initialize_model_adapter(task_cfg, data_adapter, base_model)
-
-    # update task_cfg.dataset_args
-    task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()
-
-    return Evaluator(
-        data_adapter=data_adapter,
-        model_adapter=model_adapter,
-        outputs=outputs,
-        task_cfg=task_cfg,
-    )
-
-
 def main():
     from evalscope.arguments import parse_args
     args = parse_args()

evalscope/summarizer.py

@@ -80,7 +80,7 @@ class Summarizer:
 
                 summary_file_path = summary_files[0]
                 # Example: [{'dataset': 'gsm8k', 'version': '1d7fe4', 'metric': 'accuracy', 'mode': 'gen', 'qwen-7b-chat': '53.98'} # noqa: E501
-                summary_res: List[dict] = csv_to_list(file_path=summary_file_path)
+                summary_res: List[dict] = csv_to_list(summary_file_path)
                 final_res_list.extend(summary_res)
             elif eval_backend == EvalBackend.VLM_EVAL_KIT:
                 eval_config = Summarizer.parse_eval_config(candidate_task)

evalscope/utils/__init__.py

@@ -7,9 +7,22 @@ from .import_utils import _LazyModule
 if TYPE_CHECKING:
     from .argument_utils import BaseArgument, get_supported_params, parse_int_or_float
     from .deprecation_utils import deprecated
+    from .function_utils import run_once, thread_safe
     from .import_utils import get_module_path, is_module_installed
-    from .io_utils import (OutputsStructure, csv_to_jsonl, csv_to_list, dict_to_yaml, gen_hash, get_latest_folder_path,
-                           get_valid_list, json_to_dict, jsonl_to_csv, jsonl_to_list, yaml_to_dict)
+    from .io_utils import (
+        OutputsStructure,
+        csv_to_jsonl,
+        csv_to_list,
+        dict_to_yaml,
+        gen_hash,
+        get_latest_folder_path,
+        get_valid_list,
+        json_to_dict,
+        jsonl_to_csv,
+        jsonl_to_list,
+        safe_filename,
+        yaml_to_dict,
+    )
     from .logger import configure_logging, get_logger
     from .model_utils import EvalBackend, dict_torch_dtype_to_str, fix_do_sample_warning, get_device, seed_everything
 
@@ -31,6 +44,10 @@ else:
             'is_module_installed',
             'get_module_path',
         ],
+        'function_utils': [
+            'thread_safe',
+            'run_once',
+        ],
         'io_utils': [
             'OutputsStructure',
             'csv_to_list',
@@ -44,6 +61,8 @@ else:
             'jsonl_to_list',
             'gen_hash',
             'get_valid_list',
+            'safe_filename',
+            'thread_safe',
         ],
         'deprecation_utils': [
             'deprecated',

evalscope/utils/chat_service.py

@@ -204,7 +204,8 @@ class ChatService:
 
     def _prepare_chat_inputs(self, request: ChatCompletionRequest):
         formatted_prompt = self.tokenizer.apply_chat_template(
-            request.messages, tokenize=False, add_generation_prompt=True)
+            request.messages, tokenize=False, add_generation_prompt=True
+        )
         inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return formatted_prompt, inputs, prompt_tokens

evalscope/utils/deprecation_utils.py

@@ -1,5 +1,6 @@
 import functools
 import inspect
+import os
 from typing import Callable, Optional
 
 from .logger import get_logger
@@ -22,7 +23,7 @@ def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optiona
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             # Get the file name where the function is defined
-            file_name = inspect.getfile(func)
+            file_name = os.path.basename(inspect.getfile(func))
 
             # Construct the warning message
             warning_parts = [
@@ -40,3 +41,13 @@ def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optiona
         return wrapper
 
     return decorator
+
+
+def deprecated_warning(logger, message: str):
+    """
+    Log a deprecation warning.
+
+    :param logger: Logger instance to log the warning
+    :param message: Warning message to log
+    """
+    logger.warning(f'Deprecated: {message}')

evalscope/utils/function_utils.py

@@ -0,0 +1,29 @@
+import threading
+from functools import wraps
+
+
+def run_once(func):
+    """Decorator to ensure a function is only run once."""
+    has_run = False
+    result = None
+
+    def wrapper(*args, **kwargs):
+        nonlocal has_run, result
+        if not has_run:
+            result = func(*args, **kwargs)
+            has_run = True
+        return result
+
+    return wrapper
+
+
+def thread_safe(func):
+    """Thread-safe decorator for functions that need to be executed in a thread-safe manner."""
+    lock = threading.RLock()
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with lock:
+            return func(*args, **kwargs)
+
+    return wrapper

evalscope/utils/io_utils.py

@@ -5,6 +5,8 @@ import json
 import jsonlines as jsonl
 import os
 import re
+import string
+import unicodedata
 import yaml
 from io import BytesIO
 from PIL import Image
@@ -33,7 +35,7 @@ class OutputsStructure:
             'configs_dir': None
         }
 
-    def _get_dir(self, attr_name, dir_name):
+    def _get_dir(self, attr_name, dir_name) -> str:
         if self._dirs[attr_name] is None:
             dir_path = os.path.join(self.outputs_dir, dir_name)
             if self.is_make:
@@ -72,10 +74,20 @@ def jsonl_to_list(jsonl_file):
     Returns:
         list: list of lines. Each line is a dict.
     """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
+    try:
+        res_list = []
+        with jsonl.open(jsonl_file, mode='r') as reader:
+            for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
+                res_list.append(line)
+    except Exception:
+        # Fallback to reading line by line
+        res_list = []
+        with open(jsonl_file, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():  # Skip empty lines
+                    res_list.append(json.loads(line.strip()))
+    if not res_list:
+        logger.warning(f'No data found in {jsonl_file}.')
     return res_list
 
 
@@ -272,7 +284,90 @@ def get_valid_list(input_list, candidate_list):
 
 
 def PIL_to_base64(image: Image.Image, format: str = 'JPEG') -> str:
+    """
+    Convert a PIL Image to a base64 encoded string.
+
+    Args:
+        image (Image.Image): The PIL Image to convert.
+        format (str): The format to save the image in. Default is 'JPEG'.
+    Returns:
+        str: Base64 encoded string of the image.
+    """
     buffered = BytesIO()
     image.save(buffered, format=format)
     img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
     return img_str
+
+
+def safe_filename(s: str, max_length: int = 255) -> str:
+    """
+    Convert a string into a safe filename by removing or replacing unsafe characters.
+
+    Args:
+        s (str): The input string to convert
+        max_length (int): Maximum length of the resulting filename (default 255)
+
+    Returns:
+        str: A safe filename string
+
+    Examples:
+        >>> safe_filename("Hello/World?.txt")
+        'Hello_World.txt'
+    """
+    # normalize unicode characters
+    s = unicodedata.normalize('NFKD', s)
+    s = s.encode('ASCII', 'ignore').decode('ASCII')
+
+    # remove or replace unsafe characters
+    # Keep only alphanumeric characters, dots, dashes, and underscores
+    safe_chars = string.ascii_letters + string.digits + '.-_'
+    s = ''.join(c if c in safe_chars else '_' for c in s)
+
+    # remove consecutive underscores
+    s = re.sub(r'_+', '_', s)
+
+    # remove leading/trailing periods and underscores
+    s = s.strip('._')
+
+    # handle empty string case
+    if not s:
+        s = 'untitled'
+
+    # handle starting with a period (hidden files)
+    if s.startswith('.'):
+        s = '_' + s
+
+    # enforce length limit
+    if len(s) > max_length:
+        # If we need to truncate, preserve the file extension if present
+        name, ext = os.path.splitext(s)
+        ext_len = len(ext)
+        if ext_len > 0:
+            max_name_length = max_length - ext_len
+            s = name[:max_name_length] + ext
+        else:
+            s = s[:max_length]
+
+    return s
+
+
+def convert_numpy_types(obj):
+    """Recursively convert numpy types to native Python types for JSON serialization."""
+    import numpy as np
+
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_numpy_types(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_numpy_types(item) for item in obj]
+    elif isinstance(obj, tuple):
+        return tuple(convert_numpy_types(item) for item in obj)
+    else:
+        return obj

evalscope/utils/json_schema.py

@@ -0,0 +1,208 @@
+import types
+import typing
+from copy import deepcopy
+from dataclasses import is_dataclass
+from datetime import date, datetime, time
+from enum import EnumMeta
+from pydantic import BaseModel, Field
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+    Union,
+    cast,
+    get_args,
+    get_origin,
+    get_type_hints,
+    is_typeddict,
+)
+
+JSONType = Literal['string', 'integer', 'number', 'boolean', 'array', 'object', 'null']
+"""Valid types within JSON schema."""
+
+
+class JSONSchema(BaseModel):
+    """JSON Schema for type."""
+
+    type: Optional[JSONType] = Field(default=None)
+    """JSON type of tool parameter."""
+
+    format: Optional[str] = Field(default=None)
+    """Format of the parameter (e.g. date-time)."""
+
+    description: Optional[str] = Field(default=None)
+    """Parameter description."""
+
+    default: Any = Field(default=None)
+    """Default value for parameter."""
+
+    enum: Optional[List[Any]] = Field(default=None)
+    """Valid values for enum parameters."""
+
+    items: Optional['JSONSchema'] = Field(default=None)
+    """Valid type for array parameters."""
+
+    properties: Optional[Dict[str, 'JSONSchema']] = Field(default=None)
+    """Valid fields for object parametrs."""
+
+    additionalProperties: Optional[Union['JSONSchema', bool]] = Field(default=None)
+    """Are additional properties allowed?"""
+
+    anyOf: Optional[List['JSONSchema']] = Field(default=None)
+    """Valid types for union parameters."""
+
+    required: Optional[List[str]] = Field(default=None)
+    """Required fields for object parameters."""
+
+
+def json_schema(t: Type[Any]) -> JSONSchema:
+    """Provide a JSON Schema for the specified type.
+
+    Schemas can be automatically inferred for a wide variety of
+    Python class types including Pydantic BaseModel, dataclasses,
+    and typed dicts.
+
+    Args:
+        t: Python type
+
+    Returns:
+        JSON Schema for type.
+    """
+    origin = get_origin(t)
+    args = get_args(t)
+
+    if origin is None:
+        if t is int:
+            return JSONSchema(type='integer')
+        elif t is float:
+            return JSONSchema(type='number')
+        elif t is str:
+            return JSONSchema(type='string')
+        elif t is bool:
+            return JSONSchema(type='boolean')
+        elif t is datetime:
+            return JSONSchema(type='string', format='date-time')
+        elif t is date:
+            return JSONSchema(type='string', format='date')
+        elif t is time:
+            return JSONSchema(type='string', format='time')
+        elif t is list or t is set:
+            return JSONSchema(type='array', items=JSONSchema())
+        elif t is dict:
+            return JSONSchema(type='object', additionalProperties=JSONSchema())
+        elif (is_dataclass(t) or is_typeddict(t) or (isinstance(t, type) and issubclass(t, BaseModel))):
+            return cls_json_schema(t)
+        elif isinstance(t, EnumMeta):
+            return JSONSchema(enum=[item.value for item in t])
+        elif t is type(None):
+            return JSONSchema(type='null')
+        else:
+            return JSONSchema()
+    elif (origin is list or origin is List or origin is tuple or origin is Tuple or origin is set or origin is Set):
+        return JSONSchema(type='array', items=json_schema(args[0]) if args else JSONSchema())
+    elif origin is dict or origin is Dict:
+        return JSONSchema(
+            type='object',
+            additionalProperties=json_schema(args[1]) if len(args) > 1 else JSONSchema(),
+        )
+    elif origin is Union or origin is types.UnionType:
+        return JSONSchema(anyOf=[json_schema(arg) for arg in args])
+    elif origin is Optional:
+        return JSONSchema(anyOf=[json_schema(arg) for arg in args] + [JSONSchema(type='null')])
+    elif origin is typing.Literal:
+        return JSONSchema(enum=list(args))
+
+    return JSONSchema()  # Default case if we can't determine the type
+
+
+def cls_json_schema(cls: Type[Any]) -> JSONSchema:
+    properties: Dict[str, JSONSchema] = {}
+    required: List[str] = []
+
+    if is_dataclass(cls):
+        fields = cls.__dataclass_fields__  # type: ignore
+        for name, field in fields.items():
+            properties[name] = json_schema(field.type)  # type: ignore
+            if field.default == field.default_factory:
+                required.append(name)
+    elif isinstance(cls, type) and issubclass(cls, BaseModel):
+        schema = cls.model_json_schema()
+        schema = resolve_schema_references(schema)
+        for name, prop in schema.get('properties', {}).items():
+            properties[name] = JSONSchema(**prop)
+        required = schema.get('required', [])
+    elif is_typeddict(cls):
+        annotations = get_type_hints(cls)
+        for name, type_hint in annotations.items():
+            properties[name] = json_schema(type_hint)
+            if name in cls.__required_keys__:
+                required.append(name)
+
+    return JSONSchema(
+        type='object',
+        properties=properties,
+        required=required if required else None,
+        additionalProperties=False,
+    )
+
+
+def python_type_to_json_type(python_type: Optional[str]) -> JSONType:
+    if python_type == 'str':
+        return 'string'
+    elif python_type == 'int':
+        return 'integer'
+    elif python_type == 'float':
+        return 'number'
+    elif python_type == 'bool':
+        return 'boolean'
+    elif python_type == 'list':
+        return 'array'
+    elif python_type == 'dict':
+        return 'object'
+    elif python_type == 'None':
+        return 'null'
+    elif python_type is None:
+        # treat 'unknown' as string as anything can be converted to string
+        return 'string'
+    else:
+        raise ValueError(f'Unsupported type: {python_type} for Python to JSON conversion.')
+
+
+def resolve_schema_references(schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Resolves all $ref references in a JSON schema by inlining the definitions."""
+    schema = deepcopy(schema)
+    definitions = schema.pop('$defs', {})
+
+    def _resolve_refs(obj: Any) -> Any:
+        if isinstance(obj, dict):
+            if '$ref' in obj and obj['$ref'].startswith('#/$defs/'):
+                ref_key = obj['$ref'].split('/')[-1]
+                if ref_key in definitions:
+                    # Replace with a deep copy of the definition
+                    resolved = deepcopy(definitions[ref_key])
+                    # Process any nested references in the definition
+                    resolved = _resolve_refs(resolved)
+
+                    # Merge in the current object fields, which should take priority
+                    # This means that if you have e.g.
+                    # {"$ref": "#/$defs/SubType", "description": "subtype of type SubType"},
+                    # and SubType resolves to
+                    # {"description": "The SubType Class", "parameters": {"param1": {"type": "string"}}},
+                    # the final result will be:
+                    # {"description": "subtype of type SubType", "parameters": {"param1": {"type": "string"}}}
+                    return resolved | {k: o for k, o in obj.items() if k != '$ref'}
+
+            # Process all entries in the dictionary
+            return {k: _resolve_refs(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [_resolve_refs(item) for item in obj]
+        else:
+            return obj
+
+    return cast(Dict[str, Any], _resolve_refs(schema))
+    return cast(Dict[str, Any], _resolve_refs(schema))

evalscope/utils/logger.py

@@ -1,18 +1,27 @@
+import colorlog
 import importlib.util as iutil
 import logging
 import os
-from typing import Optional
+from logging import Logger
+from typing import List, Optional
 
 init_loggers = {}
+# Define log formats
+data_format = '%Y-%m-%d %H:%M:%S'
+# For console output
+color_detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(log_color)s%(levelname)s%(reset)s: %(message)s'  # noqa:E501
+color_simple_format = '%(asctime)s - %(name)s - %(log_color)s%(levelname)s%(reset)s: %(message)s'
+color_detailed_formatter = colorlog.ColoredFormatter(color_detailed_format, datefmt=data_format)
+color_simple_formatter = colorlog.ColoredFormatter(color_simple_format, datefmt=data_format)
+# For file output
+detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s: %(message)s'  # noqa:E501
+simple_format = '%(asctime)s - %(name)s - %(levelname)s: %(message)s'
+plain_detailed_formatter = logging.Formatter(detailed_format, datefmt=data_format)
+plain_simple_formatter = logging.Formatter(simple_format, datefmt=data_format)
 
-detailed_format = '%(asctime)s - %(name)s - %(filename)s - %(funcName)s - %(lineno)d - %(levelname)s - %(message)s'
-simple_format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-
-detailed_formatter = logging.Formatter(detailed_format)
-simple_formatter = logging.Formatter(simple_format)
 DEFAULT_LEVEL = logging.DEBUG if os.getenv('EVALSCOPE_LOG_LEVEL', 'INFO') == 'DEBUG' else logging.INFO
 
-logging.basicConfig(format=simple_format, level=DEFAULT_LEVEL, force=True)
+logging.basicConfig(format=simple_format, level=logging.INFO, force=True)
 
 # set logging level
 logging.getLogger('datasets').setLevel(logging.WARNING)
@@ -20,7 +29,13 @@ logging.getLogger('httpx').setLevel(logging.WARNING)
 logging.getLogger('modelscope').setLevel(logging.ERROR)
 
 
-def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, file_mode: str = 'w', force=False):
+def get_logger(
+    log_file: Optional[str] = None,
+    name: Optional[str] = None,
+    log_level: int = DEFAULT_LEVEL,
+    file_mode: str = 'w',
+    force=False
+):
     """Get logging logger
 
     Args:
@@ -31,7 +46,10 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
             specified (if filemode is unspecified, it defaults to 'w').
     """
 
-    logger_name = __name__.split('.')[0]
+    if name:
+        logger_name = f"evalscope.{name.split('.')[-1]}"
+    else:
+        logger_name = 'evalscope'
     logger = logging.getLogger(logger_name)
     logger.propagate = False
 
@@ -40,7 +58,15 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
             logger.setLevel(log_level)
             for handler in logger.handlers:
                 handler.setLevel(log_level)
-                handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
+                # 区分不同类型的 handler，使用相应的格式化器
+                if isinstance(handler, logging.FileHandler):
+                    handler.setFormatter(
+                        plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter
+                    )
+                else:
+                    handler.setFormatter(
+                        color_detailed_formatter if log_level == logging.DEBUG else color_simple_formatter
+                    )
             add_file_handler_if_needed(logger, log_file, file_mode, log_level)
         return logger
 
@@ -66,7 +92,11 @@ def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, f
         handlers.append(file_handler)
 
     for handler in handlers:
-        handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
+        # 区分不同类型的 handler，使用相应的格式化器
+        if isinstance(handler, logging.FileHandler):
+            handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
+        else:
+            handler.setFormatter(color_detailed_formatter if log_level == logging.DEBUG else color_simple_formatter)
         handler.setLevel(log_level)
         logger.addHandler(handler)
 
@@ -102,6 +132,15 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
 
     if is_worker0 and log_file is not None:
         file_handler = logging.FileHandler(log_file, file_mode)
-        file_handler.setFormatter(detailed_formatter if log_level == logging.DEBUG else simple_formatter)
+        file_handler.setFormatter(plain_detailed_formatter if log_level == logging.DEBUG else plain_simple_formatter)
         file_handler.setLevel(log_level)
         logger.addHandler(file_handler)
+
+
+def warn_once(logger: Logger, message: str) -> None:
+    if message not in _warned:
+        logger.warning(message)
+        _warned.append(message)
+
+
+_warned: List[str] = []