evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/run.py

@@ -13,9 +13,6 @@ from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.utils.model_utils import seed_everything
 
-if TYPE_CHECKING:
-    from evalscope.models import LocalModel
-
 logger = get_logger()
 
 
@@ -109,27 +106,43 @@ def get_backend_manager_class(eval_backend: EvalBackend):
         raise NotImplementedError(f'Not implemented for evaluation backend {eval_backend}')
 
 
-def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
+def evaluate_model(task_config: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
-    from evalscope.models import get_local_model
+    from evalscope.api.evaluator import Evaluator
+    from evalscope.api.model import get_model_with_task_config
+    from evalscope.api.registry import get_benchmark
+    from evalscope.evaluator import DefaultEvaluator
     from evalscope.report import gen_table
 
     # Initialize evaluator
     eval_results = {}
-    base_model = get_local_model(task_cfg)
-    evaluators = []
-    for dataset_name in task_cfg.datasets:
-        evaluator = create_evaluator(task_cfg, dataset_name, outputs, base_model)
+    # Initialize model
+    model = get_model_with_task_config(task_config=task_config)
+    # Initialize evaluators for each dataset
+    evaluators: List[Evaluator] = []
+    for dataset_name in task_config.datasets:
+        # Create evaluator for each dataset
+        benchmark = get_benchmark(dataset_name, task_config)
+        evaluator = DefaultEvaluator(
+            task_config=task_config,
+            model=model,
+            benchmark=benchmark,
+            outputs=outputs,
+        )
         evaluators.append(evaluator)
 
+        # Update task_config.dataset_args with benchmark metadata, except for DataCollection
+        if dataset_name != DataCollection.NAME:
+            task_config.dataset_args[dataset_name] = benchmark.to_dict()
+
     # dump task_cfg to outputs.configs_dir after creating evaluators
-    task_cfg.dump_yaml(outputs.configs_dir)
-    logger.info(task_cfg)
+    task_config.dump_yaml(outputs.configs_dir)
+    logger.info(task_config)
 
     # Run evaluation for each evaluator
     for evaluator in evaluators:
         res_dict = evaluator.eval()
-        eval_results[evaluator.dataset_name] = res_dict
+        eval_results[evaluator.benchmark.name] = res_dict
 
     # Make overall report
     try:
@@ -137,48 +150,21 @@ def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
         logger.info(f'Overall report table: \n{report_table} \n')
     except Exception:
         logger.error('Failed to generate report table.')
-
     # Clean up
-    if base_model is not None:
+    if model is not None:
         import gc
-        import torch
 
-        del base_model
+        del model
         del evaluators
-        torch.cuda.empty_cache()
         gc.collect()
 
-    return eval_results
-
+        from evalscope.utils.import_utils import check_import
+        if check_import('torch'):
+            import torch
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
 
-def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: 'LocalModel'):
-    """Create an evaluator object for the specified dataset."""
-    from evalscope.benchmarks import Benchmark, BenchmarkMeta
-    from evalscope.evaluator import Evaluator
-    from evalscope.models import initialize_model_adapter
-
-    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
-
-    if dataset_name == DataCollection.NAME:
-        # EvaluatorCollection is a collection of evaluators
-        from evalscope.collections import EvaluatorCollection
-        data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
-        return EvaluatorCollection(task_cfg, data_adapter, outputs, base_model)
-
-    # Initialize data adapter first to update config
-    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
-    # Initialize model adapter
-    model_adapter = initialize_model_adapter(task_cfg, data_adapter, base_model)
-
-    # update task_cfg.dataset_args
-    task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()
-
-    return Evaluator(
-        data_adapter=data_adapter,
-        model_adapter=model_adapter,
-        outputs=outputs,
-        task_cfg=task_cfg,
-    )
+    return eval_results
 
 
 def main():

evalscope/summarizer.py

@@ -80,7 +80,7 @@ class Summarizer:
 
                 summary_file_path = summary_files[0]
                 # Example: [{'dataset': 'gsm8k', 'version': '1d7fe4', 'metric': 'accuracy', 'mode': 'gen', 'qwen-7b-chat': '53.98'} # noqa: E501
-                summary_res: List[dict] = csv_to_list(file_path=summary_file_path)
+                summary_res: List[dict] = csv_to_list(summary_file_path)
                 final_res_list.extend(summary_res)
             elif eval_backend == EvalBackend.VLM_EVAL_KIT:
                 eval_config = Summarizer.parse_eval_config(candidate_task)

evalscope/third_party/toolbench_static/llm/swift_infer.py

@@ -1,9 +1,5 @@
-import os
 from dataclasses import dataclass
-from swift.llm import InferEngine, InferRequest, PtEngine, RequestConfig, get_template
 
-# 设置GPU环境变量
-os.environ['CUDA_VISIBLE_DEVICES'] = '0'
 
 @dataclass
 class SwiftInferArgs:

evalscope/utils/__init__.py

@@ -7,9 +7,22 @@ from .import_utils import _LazyModule
 if TYPE_CHECKING:
     from .argument_utils import BaseArgument, get_supported_params, parse_int_or_float
     from .deprecation_utils import deprecated
+    from .function_utils import run_once, thread_safe
     from .import_utils import get_module_path, is_module_installed
-    from .io_utils import (OutputsStructure, csv_to_jsonl, csv_to_list, dict_to_yaml, gen_hash, get_latest_folder_path,
-                           get_valid_list, json_to_dict, jsonl_to_csv, jsonl_to_list, yaml_to_dict)
+    from .io_utils import (
+        OutputsStructure,
+        csv_to_jsonl,
+        csv_to_list,
+        dict_to_yaml,
+        gen_hash,
+        get_latest_folder_path,
+        get_valid_list,
+        json_to_dict,
+        jsonl_to_csv,
+        jsonl_to_list,
+        safe_filename,
+        yaml_to_dict,
+    )
     from .logger import configure_logging, get_logger
     from .model_utils import EvalBackend, dict_torch_dtype_to_str, fix_do_sample_warning, get_device, seed_everything
 
@@ -31,6 +44,10 @@ else:
             'is_module_installed',
             'get_module_path',
         ],
+        'function_utils': [
+            'thread_safe',
+            'run_once',
+        ],
         'io_utils': [
             'OutputsStructure',
             'csv_to_list',
@@ -44,6 +61,8 @@ else:
             'jsonl_to_list',
             'gen_hash',
             'get_valid_list',
+            'safe_filename',
+            'thread_safe',
         ],
         'deprecation_utils': [
             'deprecated',

evalscope/utils/chat_service.py

@@ -1,6 +1,5 @@
 import os
 import time
-import torch
 from contextlib import contextmanager
 from functools import partial
 from pydantic import BaseModel, Field
@@ -95,6 +94,7 @@ class TextCompletionResponse(BaseModel):
 class ChatService:
 
     def __init__(self, model_path, attn_implementation):
+        import torch
         from modelscope import AutoModelForCausalLM, AutoTokenizer
         from transformers import TextIteratorStreamer
 
@@ -204,7 +204,8 @@ class ChatService:
 
     def _prepare_chat_inputs(self, request: ChatCompletionRequest):
         formatted_prompt = self.tokenizer.apply_chat_template(
-            request.messages, tokenize=False, add_generation_prompt=True)
+            request.messages, tokenize=False, add_generation_prompt=True
+        )
         inputs = self.tokenizer(formatted_prompt, return_tensors='pt', padding=False).to(self.device)
         prompt_tokens = len(inputs['input_ids'][0])
         return formatted_prompt, inputs, prompt_tokens

evalscope/utils/deprecation_utils.py

@@ -1,5 +1,6 @@
 import functools
 import inspect
+import os
 from typing import Callable, Optional
 
 from .logger import get_logger
@@ -22,7 +23,7 @@ def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optiona
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
             # Get the file name where the function is defined
-            file_name = inspect.getfile(func)
+            file_name = os.path.basename(inspect.getfile(func))
 
             # Construct the warning message
             warning_parts = [
@@ -40,3 +41,13 @@ def deprecated(since: str, remove_in: Optional[str] = None, alternative: Optiona
         return wrapper
 
     return decorator
+
+
+def deprecated_warning(logger, message: str):
+    """
+    Log a deprecation warning.
+
+    :param logger: Logger instance to log the warning
+    :param message: Warning message to log
+    """
+    logger.warning(f'Deprecated: {message}')

evalscope/utils/function_utils.py

@@ -0,0 +1,29 @@
+import threading
+from functools import wraps
+
+
+def run_once(func):
+    """Decorator to ensure a function is only run once."""
+    has_run = False
+    result = None
+
+    def wrapper(*args, **kwargs):
+        nonlocal has_run, result
+        if not has_run:
+            result = func(*args, **kwargs)
+            has_run = True
+        return result
+
+    return wrapper
+
+
+def thread_safe(func):
+    """Thread-safe decorator for functions that need to be executed in a thread-safe manner."""
+    lock = threading.RLock()
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        with lock:
+            return func(*args, **kwargs)
+
+    return wrapper

evalscope/utils/import_utils.py

@@ -5,13 +5,35 @@ import importlib
 import os
 from itertools import chain
 from types import ModuleType
-from typing import Any
+from typing import Any, Optional, Union
 
 from .logger import get_logger
 
 logger = get_logger()  # pylint: disable=invalid-name
 
 
+def check_import(module_name: str, package: Optional[str] = None, raise_error: bool = False) -> bool:
+    """Check if a module can be imported.
+
+    Args:
+        module_name (str): The name of the module to check.
+        package (str, optional): The package to install if the module is not found. Defaults to None.
+        raise_error (bool, optional): Whether to raise an error if the module is not found. Defaults to False.
+    """
+    try:
+        importlib.import_module(module_name)
+        return True
+    except ImportError:
+        error_msg = f'`{module_name}` not found.'
+        if package:
+            error_msg += f' Please run `pip install {package}` to use this feature.'
+            logger.warning(error_msg)
+
+        if raise_error:
+            raise ImportError(error_msg)
+        return False
+
+
 class _LazyModule(ModuleType):
     """
     Module class that surfaces all objects but only performs associated imports when the objects are requested.

evalscope/utils/io_utils.py

@@ -1,10 +1,13 @@
 import base64
 import csv
 import hashlib
+import io
 import json
 import jsonlines as jsonl
 import os
 import re
+import string
+import unicodedata
 import yaml
 from io import BytesIO
 from PIL import Image
@@ -33,7 +36,7 @@ class OutputsStructure:
             'configs_dir': None
         }
 
-    def _get_dir(self, attr_name, dir_name):
+    def _get_dir(self, attr_name, dir_name) -> str:
         if self._dirs[attr_name] is None:
             dir_path = os.path.join(self.outputs_dir, dir_name)
             if self.is_make:
@@ -72,10 +75,20 @@ def jsonl_to_list(jsonl_file):
     Returns:
         list: list of lines. Each line is a dict.
     """
-    res_list = []
-    with jsonl.open(jsonl_file, mode='r') as reader:
-        for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
-            res_list.append(line)
+    try:
+        res_list = []
+        with jsonl.open(jsonl_file, mode='r') as reader:
+            for line in reader.iter(type=dict, allow_none=True, skip_invalid=False):
+                res_list.append(line)
+    except Exception:
+        # Fallback to reading line by line
+        res_list = []
+        with open(jsonl_file, 'r', encoding='utf-8') as f:
+            for line in f:
+                if line.strip():  # Skip empty lines
+                    res_list.append(json.loads(line.strip()))
+    if not res_list:
+        logger.warning(f'No data found in {jsonl_file}.')
     return res_list
 
 
@@ -271,8 +284,131 @@ def get_valid_list(input_list, candidate_list):
            [i for i in input_list if i not in candidate_list]
 
 
-def PIL_to_base64(image: Image.Image, format: str = 'JPEG') -> str:
+def PIL_to_base64(image: Image.Image, format: str = 'JPEG', add_header: bool = False) -> str:
+    """
+    Convert a PIL Image to a base64 encoded string.
+
+    Args:
+        image (Image.Image): The PIL Image to convert.
+        format (str): The format to save the image in. Default is 'JPEG'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
+    Returns:
+        str: Base64 encoded string of the image.
+    """
     buffered = BytesIO()
     image.save(buffered, format=format)
     img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format.lower()};base64,{img_str}'
+    return img_str
+
+
+def bytes_to_base64(bytes_data: bytes, format: str = 'png', add_header: bool = False) -> str:
+    """Convert image bytes to a base64 encoded string.
+
+    Args:
+        bytes_data (bytes): The bytes to convert.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
+    Returns:
+        str: Base64 encoded string of the bytes.
+    """
+    img_str = base64.b64encode(bytes_data).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format};base64,{img_str}'
     return img_str
+
+
+def base64_to_PIL(base64_str):
+    """Convert a base64 encoded string to a PIL Image.
+
+    Args:
+        base64_str (str): The base64 encoded string.
+
+    Returns:
+        Image.Image: The decoded PIL Image.
+    """
+    # remove header
+    if ',' in base64_str:
+        base64_str = base64_str.split(',', 1)[1]
+
+    # decode
+    img_data = base64.b64decode(base64_str)
+    img_file = io.BytesIO(img_data)
+    img = Image.open(img_file)
+    return img
+
+
+def safe_filename(s: str, max_length: int = 255) -> str:
+    """
+    Convert a string into a safe filename by removing or replacing unsafe characters.
+
+    Args:
+        s (str): The input string to convert
+        max_length (int): Maximum length of the resulting filename (default 255)
+
+    Returns:
+        str: A safe filename string
+
+    Examples:
+        >>> safe_filename("Hello/World?.txt")
+        'Hello_World.txt'
+    """
+    # normalize unicode characters
+    s = unicodedata.normalize('NFKD', s)
+    s = s.encode('ASCII', 'ignore').decode('ASCII')
+
+    # remove or replace unsafe characters
+    # Keep only alphanumeric characters, dots, dashes, and underscores
+    safe_chars = string.ascii_letters + string.digits + '.-_'
+    s = ''.join(c if c in safe_chars else '_' for c in s)
+
+    # remove consecutive underscores
+    s = re.sub(r'_+', '_', s)
+
+    # remove leading/trailing periods and underscores
+    s = s.strip('._')
+
+    # handle empty string case
+    if not s:
+        s = 'untitled'
+
+    # handle starting with a period (hidden files)
+    if s.startswith('.'):
+        s = '_' + s
+
+    # enforce length limit
+    if len(s) > max_length:
+        # If we need to truncate, preserve the file extension if present
+        name, ext = os.path.splitext(s)
+        ext_len = len(ext)
+        if ext_len > 0:
+            max_name_length = max_length - ext_len
+            s = name[:max_name_length] + ext
+        else:
+            s = s[:max_length]
+
+    return s
+
+
+def convert_numpy_types(obj):
+    """Recursively convert numpy types to native Python types for JSON serialization."""
+    import numpy as np
+
+    if isinstance(obj, np.bool_):
+        return bool(obj)
+    elif isinstance(obj, np.integer):
+        return int(obj)
+    elif isinstance(obj, np.floating):
+        return float(obj)
+    elif isinstance(obj, np.ndarray):
+        return obj.tolist()
+    elif isinstance(obj, dict):
+        return {key: convert_numpy_types(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_numpy_types(item) for item in obj]
+    elif isinstance(obj, tuple):
+        return tuple(convert_numpy_types(item) for item in obj)
+    else:
+        return obj