evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/utils/function_utils.py

@@ -1,29 +1,266 @@
+import asyncio
 import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, wait
+from contextlib import contextmanager
 from functools import wraps
+from tqdm import tqdm
+from typing import Any, Awaitable, Callable, List, Optional, Sequence, TypeVar, Union
 
+from evalscope.utils.logger import get_logger
 
-def run_once(func):
-    """Decorator to ensure a function is only run once."""
-    has_run = False
-    result = None
+logger = get_logger()
 
+T = TypeVar('T')
+R = TypeVar('R')
+
+# Global lock to safely create per-instance locks in decorators
+_THREAD_SAFE_GLOBAL_LOCK = threading.RLock()
+
+
+def thread_safe(func: Callable[..., T]) -> Callable[..., T]:
+    """Thread-safe decorator.
+    - If decorating a bound method, uses a per-instance, per-method lock.
+    - If decorating a function, uses a function-scoped lock.
+    """
+    func_lock = threading.RLock()
+    lock_attr_name = f'__lock_{func.__name__}'
+
+    @wraps(func)
     def wrapper(*args, **kwargs):
-        nonlocal has_run, result
-        if not has_run:
-            result = func(*args, **kwargs)
-            has_run = True
-        return result
+        # Prefer per-instance lock if the first arg looks like 'self'
+        if args and hasattr(args[0], '__dict__'):
+            self_obj = args[0]
+            lock = getattr(self_obj, lock_attr_name, None)
+            if lock is None:
+                with _THREAD_SAFE_GLOBAL_LOCK:
+                    lock = getattr(self_obj, lock_attr_name, None)
+                    if lock is None:
+                        lock = threading.RLock()
+                        setattr(self_obj, lock_attr_name, lock)
+        else:
+            lock = func_lock
+
+        with lock:
+            return func(*args, **kwargs)
 
     return wrapper
 
 
-def thread_safe(func):
-    """Thread-safe decorator for functions that need to be executed in a thread-safe manner."""
+def run_once(func: Callable[..., T]) -> Callable[..., T]:
+    """Decorator to ensure a function is executed at most once across threads."""
     lock = threading.RLock()
+    has_run: bool = False
+    result: Optional[T] = None
 
     @wraps(func)
     def wrapper(*args, **kwargs):
+        nonlocal has_run, result
+        if has_run:
+            return result
+        # Double-checked locking to avoid redundant locking on hot path
         with lock:
-            return func(*args, **kwargs)
+            if not has_run:
+                result = func(*args, **kwargs)
+                has_run = True
+        return result
 
     return wrapper
+
+
+def retry_func(retries=3, sleep_interval=0):
+    """A decorator that retries a function call up to `retries` times if an exception occurs."""
+
+    def decorator(func):
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(retries):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    last_exception = e
+                    if sleep_interval > 0:
+                        time.sleep(sleep_interval)
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+@contextmanager
+def retry_context(retries=3, sleep_interval=0):
+    """A context manager that retries the code block up to `retries` times if an exception occurs."""
+    last_exception = None
+    for attempt in range(retries):
+        try:
+            yield
+            return  # If no exception, exit successfully
+        except Exception as e:
+            last_exception = e
+            if sleep_interval > 0:
+                time.sleep(sleep_interval)
+            if attempt == retries - 1:  # Last attempt
+                break
+    raise last_exception
+
+
+class AsyncioLoopRunner:
+    """Singleton background asyncio loop runner for sync→async bridging."""
+    _instance: Optional['AsyncioLoopRunner'] = None
+    _inst_lock = threading.Lock()
+
+    def __init__(self) -> None:
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._start_loop()
+
+    def _start_loop(self) -> None:
+        loop = asyncio.new_event_loop()
+        self._loop = loop
+
+        def run_loop() -> None:
+            asyncio.set_event_loop(loop)
+            loop.run_forever()
+
+        self._thread = threading.Thread(target=run_loop, daemon=True, name='AsyncioLoopRunner')
+        self._thread.start()
+
+    @classmethod
+    def instance(cls) -> 'AsyncioLoopRunner':
+        if cls._instance is not None:
+            return cls._instance
+        with cls._inst_lock:
+            if cls._instance is None:
+                cls._instance = AsyncioLoopRunner()
+        return cls._instance
+
+    @classmethod
+    def run(cls, coro: Awaitable[T], timeout: Optional[float] = None) -> T:
+        """Submit a coroutine to the background loop and wait for result."""
+        inst = cls.instance()
+        fut = asyncio.run_coroutine_threadsafe(coro, inst._loop)
+        return fut.result(timeout=timeout)
+
+    @property
+    def loop(self) -> Optional[asyncio.AbstractEventLoop]:
+        """Access the underlying event loop (read-only use)."""
+        return self._loop
+
+    def stop(self, join_timeout: float = 5.0) -> None:
+        """Optional shutdown of the background loop (generally not needed)."""
+        if not self._loop:
+            return
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        if self._thread:
+            self._thread.join(timeout=join_timeout)
+
+
+def run_in_threads_with_progress(
+    items: Sequence[T],
+    worker: Callable[[T], R],
+    *,
+    desc: str,
+    max_workers: int,
+    heartbeat_sec: int,
+    on_result: Optional[Callable[[T, R], None]] = None,
+    on_error: Optional[Callable[[T, Exception], None]] = None,
+    filter_none_results: bool = False,
+) -> List[R]:
+    """
+    Execute a collection of tasks concurrently with a ThreadPoolExecutor while
+    displaying a tqdm progress bar and emitting periodic heartbeat logs.
+
+    Key behaviors:
+    - Concurrency: Uses up to `min(len(items), max_workers)` threads.
+    - Progress: A tqdm bar advances when each task finishes (success or failure).
+    - Heartbeat: If no tasks finish within `heartbeat_sec`, a status line is logged.
+    - Ordering: Results are appended in completion order (not the original order).
+    - Error handling:
+        * If `on_error` is provided, it is called for each failed item; execution continues
+          unless `on_error` itself raises.
+        * If `on_error` is None, the first exception is raised immediately and stops processing.
+    - Callbacks:
+        * `on_result(item, result)` is called after a successful result is obtained.
+        * Both callbacks run in the main thread (not worker threads).
+
+    Args:
+        items: A sequence of items (inputs) to process. Converted to a list internally.
+        worker: A callable executed in threads to process a single item and return a result.
+        desc: A short text shown as the tqdm progress bar description.
+        max_workers: Upper bound on the number of concurrent threads.
+        heartbeat_sec: Interval (in seconds) to wait before emitting a heartbeat log if
+            no tasks complete in that window.
+        on_result: Optional callback invoked as on_result(item, result) after success.
+        on_error: Optional callback invoked as on_error(item, exception) on failure. If omitted,
+            the exception is propagated and the function terminates early.
+
+    Returns:
+        A list of results collected as tasks complete (completion order).
+        If some tasks fail and `on_error` is provided (and does not re-raise), those failures
+        are skipped and not included in the returned results.
+
+    Raises:
+        Exception: Propagates the first task exception if `on_error` is not provided, or if
+        `on_error` re-raises.
+
+    Notes:
+        - The function is blocking until all tasks complete or an exception is propagated.
+        - Use `on_error` to implement "best-effort" processing where failures are logged
+          and the rest continue.
+    """
+    # Defensive copy to avoid consuming a generator multiple times and to compute pool size.
+    pending_items: List[T] = list(items)
+    if not pending_items:
+        return []
+
+    # Include indices to ensure results are returned in input order
+    indexed_items = list(enumerate(items))
+    results: List[Optional[R]] = [None] * len(items)  # Preallocate results list
+
+    # Bound the pool by actual workload size for efficiency.
+    with ThreadPoolExecutor(max_workers=min(len(indexed_items), max_workers)) as executor:
+        # Submit all tasks up-front and map futures back to their originating item.
+        future_to_index = {executor.submit(worker, item): index for index, item in indexed_items}
+
+        # Progress bar reflects total number of submitted tasks; updated per finished future.
+        with tqdm(total=len(indexed_items), desc=desc, mininterval=1, dynamic_ncols=True) as pbar:
+            # Track unfinished futures and poll with a timeout to enable heartbeat logs.
+            pending = set(future_to_index.keys())
+            while pending:
+                # Wait with timeout to detect stalls and emit heartbeats proactively.
+                done, not_done = wait(pending, timeout=heartbeat_sec)
+                if not done:
+                    # Heartbeat when nothing has completed within the window.
+                    logger.info(f'{desc} still processing... pending={len(not_done)}')
+                    continue
+
+                # Consume completed futures.
+                for future in done:
+                    index = future_to_index[future]
+                    try:
+                        res = future.result()
+                        results[index] = res  # Store result at the correct index
+                        # Invoke success callback in caller thread (not in worker).
+                        if on_result is not None:
+                            on_result(items[index], res)
+                    except Exception as exc:
+                        # Delegate failure handling to on_error if provided; otherwise bubble up.
+                        if on_error is not None:
+                            on_error(items[index], exc)
+                        else:
+                            raise
+                    finally:
+                        # Always advance progress for completed futures (success or failure).
+                        pbar.update(1)
+
+                # Continue polling remaining futures.
+                pending = not_done
+
+    # Return results, which are now guaranteed to be in input order
+    if filter_none_results:
+        # Filter out None results if on_error was used and some tasks failed
+        results = [res for res in results if res is not None]
+    return results

evalscope/utils/import_utils.py

@@ -5,13 +5,85 @@ import importlib
 import os
 from itertools import chain
 from types import ModuleType
-from typing import Any
+from typing import Any, Optional, Union
 
+from evalscope.constants import IS_BUILD_DOC
 from .logger import get_logger
 
 logger = get_logger()  # pylint: disable=invalid-name
 
 
+def check_import(
+    module_name: Union[str, list[str]],
+    package: Optional[Union[str, list[str]]] = None,
+    raise_warning: bool = True,
+    raise_error: bool = False,
+    feature_name: Optional[str] = 'this feature',
+) -> bool:
+    """Check if a module or list of modules can be imported.
+
+    Args:
+        module_name (Union[str, list[str]]): The name(s) of the module(s) to check.
+        package (Union[str, list[str]], optional): The package(s) to install if the module(s) are not found.
+            Defaults to None.
+        raise_error (bool, optional): Whether to raise an error if any module is not found. Defaults to False.
+        raise_warning (bool, optional): Whether to log a warning if any module is not found. Defaults to True.
+        feature_name (str, optional): The feature name that requires the module(s). Used in the warning/error message.
+            Defaults to 'this feature'.
+
+    Returns:
+        bool: True if all modules can be imported, False otherwise.
+    """
+    # Convert single strings to lists for uniform processing
+    if isinstance(module_name, str):
+        module_names = [module_name]
+    else:
+        module_names = module_name
+
+    if package is None:
+        packages = [None] * len(module_names)
+    elif isinstance(package, str):
+        packages = [package] * len(module_names)
+    else:
+        packages = package
+        # Ensure packages list has same length as module_names
+        if len(packages) < len(module_names):
+            packages.extend([None] * (len(module_names) - len(packages)))
+
+    missing_modules = []
+    missing_packages = []
+
+    for i, mod_name in enumerate(module_names):
+        try:
+            importlib.import_module(mod_name)
+        except ImportError:
+            missing_modules.append(mod_name)
+            if i < len(packages) and packages[i]:
+                missing_packages.append(packages[i])
+
+    if missing_modules:
+        if len(missing_modules) == 1:
+            error_msg = f'`{missing_modules[0]}` not found.'
+        else:
+            error_msg = f'The following modules are not found: {", ".join(f"`{mod}`" for mod in missing_modules)}.'
+
+        if missing_packages:
+            if len(missing_packages) == 1:
+                error_msg += f' Please run `pip install {missing_packages[0]}` to use {feature_name}.'
+            else:
+                unique_packages = list(dict.fromkeys(missing_packages))  # Remove duplicates while preserving order
+                error_msg += f' Please run `pip install {" ".join(unique_packages)}` to use {feature_name}.'
+
+        if raise_warning:
+            logger.warning(error_msg)
+
+        if not IS_BUILD_DOC and raise_error:
+            raise ImportError(error_msg)
+        return False
+
+    return True
+
+
 class _LazyModule(ModuleType):
     """
     Module class that surfaces all objects but only performs associated imports when the objects are requested.

evalscope/utils/io_utils.py

@@ -1,6 +1,7 @@
 import base64
 import csv
 import hashlib
+import io
 import json
 import jsonlines as jsonl
 import os
@@ -8,8 +9,10 @@ import re
 import string
 import unicodedata
 import yaml
+from datetime import datetime
 from io import BytesIO
 from PIL import Image
+from typing import Tuple
 
 from evalscope.constants import DumpMode
 from evalscope.utils.logger import get_logger
@@ -122,6 +125,9 @@ def dump_jsonl_data(data_list, jsonl_file, dump_mode=DumpMode.OVERWRITE):
     if not isinstance(data_list, list):
         data_list = [data_list]
 
+    # Convert non-serializable types to serializable ones
+    data_list = convert_normal_types(data_list)
+
     if dump_mode == DumpMode.OVERWRITE:
         dump_mode = 'w'
     elif dump_mode == DumpMode.APPEND:
@@ -168,6 +174,24 @@ def csv_to_list(csv_file) -> list:
     return res_list
 
 
+def tsv_to_list(tsv_file) -> list:
+    """
+    Read tsv file to list.
+
+    Args:
+        tsv_file: tsv file path.
+
+    Returns:
+        list: list of lines. Each line is a dict.
+    """
+    res_list = []
+    with open(tsv_file, 'r', encoding='utf-8') as f:
+        reader = csv.DictReader(f, delimiter='\t')
+        for row in reader:
+            res_list.append(row)
+    return res_list
+
+
 def csv_to_jsonl(csv_file, jsonl_file):
     """
     Convert csv file to jsonl file.
@@ -283,22 +307,64 @@ def get_valid_list(input_list, candidate_list):
            [i for i in input_list if i not in candidate_list]
 
 
-def PIL_to_base64(image: Image.Image, format: str = 'JPEG') -> str:
+def PIL_to_base64(image: Image.Image, format: str = 'JPEG', add_header: bool = False) -> str:
     """
     Convert a PIL Image to a base64 encoded string.
 
     Args:
         image (Image.Image): The PIL Image to convert.
         format (str): The format to save the image in. Default is 'JPEG'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+
     Returns:
         str: Base64 encoded string of the image.
     """
     buffered = BytesIO()
     image.save(buffered, format=format)
     img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
+    if add_header:
+        img_str = f'data:image/{format.lower()};base64,{img_str}'
     return img_str
 
 
+def bytes_to_base64(bytes_data: bytes, *, format: str = 'png', add_header: bool = False, content_type='image') -> str:
+    """Convert bytes to a base64 encoded string.
+
+    Args:
+        bytes_data (bytes): The bytes to convert.
+        format (str): The format of the image. Default is 'png'.
+        add_header (bool): Whether to add the base64 header. Default is False.
+        content_type (str): The type of the data, 'image' or 'audio'. Default is 'image'.
+
+    Returns:
+        str: Base64 encoded string of the bytes.
+    """
+    base64_str = base64.b64encode(bytes_data).decode('utf-8')
+    if add_header:
+        base64_str = f'data:{content_type}/{format};base64,{base64_str}'
+    return base64_str
+
+
+def base64_to_PIL(base64_str):
+    """Convert a base64 encoded string to a PIL Image.
+
+    Args:
+        base64_str (str): The base64 encoded string.
+
+    Returns:
+        Image.Image: The decoded PIL Image.
+    """
+    # remove header
+    if ',' in base64_str:
+        base64_str = base64_str.split(',', 1)[1]
+
+    # decode
+    img_data = base64.b64decode(base64_str)
+    img_file = io.BytesIO(img_data)
+    img = Image.open(img_file)
+    return img
+
+
 def safe_filename(s: str, max_length: int = 255) -> str:
     """
     Convert a string into a safe filename by removing or replacing unsafe characters.
@@ -351,11 +417,13 @@ def safe_filename(s: str, max_length: int = 255) -> str:
     return s
 
 
-def convert_numpy_types(obj):
-    """Recursively convert numpy types to native Python types for JSON serialization."""
+def convert_normal_types(obj):
+    """Recursively convert numpy types and datetime objects to native Python types for JSON serialization."""
     import numpy as np
 
-    if isinstance(obj, np.bool_):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    elif isinstance(obj, np.bool_):
         return bool(obj)
     elif isinstance(obj, np.integer):
         return int(obj)
@@ -364,10 +432,67 @@ def convert_numpy_types(obj):
     elif isinstance(obj, np.ndarray):
         return obj.tolist()
     elif isinstance(obj, dict):
-        return {key: convert_numpy_types(value) for key, value in obj.items()}
+        return {key: convert_normal_types(value) for key, value in obj.items()}
     elif isinstance(obj, list):
-        return [convert_numpy_types(item) for item in obj]
+        return [convert_normal_types(item) for item in obj]
     elif isinstance(obj, tuple):
-        return tuple(convert_numpy_types(item) for item in obj)
+        return tuple(convert_normal_types(item) for item in obj)
+    elif isinstance(obj, os.PathLike):
+        return str(obj)
     else:
         return obj
+
+
+def compress_image_to_limit(image_bytes: bytes, max_bytes: int = 10_000_000) -> Tuple[bytes, str]:
+    """
+    Ensure image bytes are under max_bytes by re-encoding to JPEG with quality reduction
+    and optional downscaling. Returns (processed_bytes, format_str).
+    If the original bytes are already below the limit, returns them as PNG.
+    """
+    if len(image_bytes) <= max_bytes:
+        return image_bytes, 'png'
+
+    try:
+        img = Image.open(BytesIO(image_bytes))
+    except Exception as exc:
+        logger.warning(f'Failed to open image bytes with PIL, sending original image; may exceed API limit: {exc}')
+        return image_bytes, 'png'
+
+    # Convert to RGB for JPEG if needed
+    if img.mode not in ('RGB', 'L'):
+        img = img.convert('RGB')
+
+    def encode_jpeg(source: Image.Image, quality: int) -> bytes:
+        buf = BytesIO()
+        source.save(buf, format='JPEG', quality=quality, optimize=True, progressive=True)
+        return buf.getvalue()
+
+    # Start with moderate quality and reduce
+    quality: int = 85
+    out: bytes = encode_jpeg(img, quality)
+    quality_floor: int = 40
+
+    while len(out) > max_bytes and quality > quality_floor:
+        quality -= 10
+        out = encode_jpeg(img, quality)
+
+    # If still too large, progressively downscale
+    min_side_floor: int = 256
+    scale: float = 0.9
+    while len(out) > max_bytes and min(img.size) > min_side_floor:
+        new_w = max(min_side_floor, int(img.width * scale))
+        new_h = max(min_side_floor, int(img.height * scale))
+        if (new_w, new_h) == img.size:
+            break
+        img = img.resize((new_w, new_h), Image.LANCZOS)
+        out = encode_jpeg(img, quality)
+
+    if len(out) > max_bytes:
+        logger.warning(f'Image remains above limit after compression: size={len(out)} bytes (limit={max_bytes}).')
+    else:
+        logger.info(
+            f'Compressed image from {len(image_bytes)} to {len(out)} bytes; '
+            f'quality={quality}, size={img.width}x{img.height}.'
+        )
+
+    return out, 'jpeg'

evalscope/utils/json_schema.py

@@ -4,7 +4,7 @@ from copy import deepcopy
 from dataclasses import is_dataclass
 from datetime import date, datetime, time
 from enum import EnumMeta
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator, model_validator
 from typing import (
     Any,
     Dict,
@@ -59,6 +59,28 @@ class JSONSchema(BaseModel):
     required: Optional[List[str]] = Field(default=None)
     """Required fields for object parameters."""
 
+    @model_validator(mode='before')
+    def convert_type_before_validation(cls, values):
+        values = deepcopy(values)
+
+        def recursive_convert_type(obj):
+            if isinstance(obj, dict):
+                # Convert 'type' field if it's a string
+                if 'type' in obj and isinstance(obj['type'], str):
+                    try:
+                        obj['type'] = python_type_to_json_type(obj['type'])
+                    except ValueError:
+                        # If conversion fails, leave it as is
+                        pass
+                # Recursively process nested structures
+                for k, v in obj.items():
+                    obj[k] = recursive_convert_type(v)
+            elif isinstance(obj, list):
+                return [recursive_convert_type(item) for item in obj]
+            return obj
+
+        return recursive_convert_type(values)
+
 
 def json_schema(t: Type[Any]) -> JSONSchema:
     """Provide a JSON Schema for the specified type.
@@ -152,6 +174,8 @@ def cls_json_schema(cls: Type[Any]) -> JSONSchema:
 
 
 def python_type_to_json_type(python_type: Optional[str]) -> JSONType:
+    if python_type is not None and python_type in get_args(JSONType):
+        return python_type
     if python_type == 'str':
         return 'string'
     elif python_type == 'int':
@@ -205,4 +229,3 @@ def resolve_schema_references(schema: Dict[str, Any]) -> Dict[str, Any]:
             return obj
 
     return cast(Dict[str, Any], _resolve_refs(schema))
-    return cast(Dict[str, Any], _resolve_refs(schema))