evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/utils/function_utils.py

@@ -0,0 +1,266 @@
+import asyncio
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, wait
+from contextlib import contextmanager
+from functools import wraps
+from tqdm import tqdm
+from typing import Any, Awaitable, Callable, List, Optional, Sequence, TypeVar, Union
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+T = TypeVar('T')
+R = TypeVar('R')
+
+# Global lock to safely create per-instance locks in decorators
+_THREAD_SAFE_GLOBAL_LOCK = threading.RLock()
+
+
+def thread_safe(func: Callable[..., T]) -> Callable[..., T]:
+    """Thread-safe decorator.
+    - If decorating a bound method, uses a per-instance, per-method lock.
+    - If decorating a function, uses a function-scoped lock.
+    """
+    func_lock = threading.RLock()
+    lock_attr_name = f'__lock_{func.__name__}'
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # Prefer per-instance lock if the first arg looks like 'self'
+        if args and hasattr(args[0], '__dict__'):
+            self_obj = args[0]
+            lock = getattr(self_obj, lock_attr_name, None)
+            if lock is None:
+                with _THREAD_SAFE_GLOBAL_LOCK:
+                    lock = getattr(self_obj, lock_attr_name, None)
+                    if lock is None:
+                        lock = threading.RLock()
+                        setattr(self_obj, lock_attr_name, lock)
+        else:
+            lock = func_lock
+
+        with lock:
+            return func(*args, **kwargs)
+
+    return wrapper
+
+
+def run_once(func: Callable[..., T]) -> Callable[..., T]:
+    """Decorator to ensure a function is executed at most once across threads."""
+    lock = threading.RLock()
+    has_run: bool = False
+    result: Optional[T] = None
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        nonlocal has_run, result
+        if has_run:
+            return result
+        # Double-checked locking to avoid redundant locking on hot path
+        with lock:
+            if not has_run:
+                result = func(*args, **kwargs)
+                has_run = True
+        return result
+
+    return wrapper
+
+
+def retry_func(retries=3, sleep_interval=0):
+    """A decorator that retries a function call up to `retries` times if an exception occurs."""
+
+    def decorator(func):
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            last_exception = None
+            for attempt in range(retries):
+                try:
+                    return func(*args, **kwargs)
+                except Exception as e:
+                    last_exception = e
+                    if sleep_interval > 0:
+                        time.sleep(sleep_interval)
+            raise last_exception
+
+        return wrapper
+
+    return decorator
+
+
+@contextmanager
+def retry_context(retries=3, sleep_interval=0):
+    """A context manager that retries the code block up to `retries` times if an exception occurs."""
+    last_exception = None
+    for attempt in range(retries):
+        try:
+            yield
+            return  # If no exception, exit successfully
+        except Exception as e:
+            last_exception = e
+            if sleep_interval > 0:
+                time.sleep(sleep_interval)
+            if attempt == retries - 1:  # Last attempt
+                break
+    raise last_exception
+
+
+class AsyncioLoopRunner:
+    """Singleton background asyncio loop runner for sync→async bridging."""
+    _instance: Optional['AsyncioLoopRunner'] = None
+    _inst_lock = threading.Lock()
+
+    def __init__(self) -> None:
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._thread: Optional[threading.Thread] = None
+        self._start_loop()
+
+    def _start_loop(self) -> None:
+        loop = asyncio.new_event_loop()
+        self._loop = loop
+
+        def run_loop() -> None:
+            asyncio.set_event_loop(loop)
+            loop.run_forever()
+
+        self._thread = threading.Thread(target=run_loop, daemon=True, name='AsyncioLoopRunner')
+        self._thread.start()
+
+    @classmethod
+    def instance(cls) -> 'AsyncioLoopRunner':
+        if cls._instance is not None:
+            return cls._instance
+        with cls._inst_lock:
+            if cls._instance is None:
+                cls._instance = AsyncioLoopRunner()
+        return cls._instance
+
+    @classmethod
+    def run(cls, coro: Awaitable[T], timeout: Optional[float] = None) -> T:
+        """Submit a coroutine to the background loop and wait for result."""
+        inst = cls.instance()
+        fut = asyncio.run_coroutine_threadsafe(coro, inst._loop)
+        return fut.result(timeout=timeout)
+
+    @property
+    def loop(self) -> Optional[asyncio.AbstractEventLoop]:
+        """Access the underlying event loop (read-only use)."""
+        return self._loop
+
+    def stop(self, join_timeout: float = 5.0) -> None:
+        """Optional shutdown of the background loop (generally not needed)."""
+        if not self._loop:
+            return
+        self._loop.call_soon_threadsafe(self._loop.stop)
+        if self._thread:
+            self._thread.join(timeout=join_timeout)
+
+
+def run_in_threads_with_progress(
+    items: Sequence[T],
+    worker: Callable[[T], R],
+    *,
+    desc: str,
+    max_workers: int,
+    heartbeat_sec: int,
+    on_result: Optional[Callable[[T, R], None]] = None,
+    on_error: Optional[Callable[[T, Exception], None]] = None,
+    filter_none_results: bool = False,
+) -> List[R]:
+    """
+    Execute a collection of tasks concurrently with a ThreadPoolExecutor while
+    displaying a tqdm progress bar and emitting periodic heartbeat logs.
+
+    Key behaviors:
+    - Concurrency: Uses up to `min(len(items), max_workers)` threads.
+    - Progress: A tqdm bar advances when each task finishes (success or failure).
+    - Heartbeat: If no tasks finish within `heartbeat_sec`, a status line is logged.
+    - Ordering: Results are appended in completion order (not the original order).
+    - Error handling:
+        * If `on_error` is provided, it is called for each failed item; execution continues
+          unless `on_error` itself raises.
+        * If `on_error` is None, the first exception is raised immediately and stops processing.
+    - Callbacks:
+        * `on_result(item, result)` is called after a successful result is obtained.
+        * Both callbacks run in the main thread (not worker threads).
+
+    Args:
+        items: A sequence of items (inputs) to process. Converted to a list internally.
+        worker: A callable executed in threads to process a single item and return a result.
+        desc: A short text shown as the tqdm progress bar description.
+        max_workers: Upper bound on the number of concurrent threads.
+        heartbeat_sec: Interval (in seconds) to wait before emitting a heartbeat log if
+            no tasks complete in that window.
+        on_result: Optional callback invoked as on_result(item, result) after success.
+        on_error: Optional callback invoked as on_error(item, exception) on failure. If omitted,
+            the exception is propagated and the function terminates early.
+
+    Returns:
+        A list of results collected as tasks complete (completion order).
+        If some tasks fail and `on_error` is provided (and does not re-raise), those failures
+        are skipped and not included in the returned results.
+
+    Raises:
+        Exception: Propagates the first task exception if `on_error` is not provided, or if
+        `on_error` re-raises.
+
+    Notes:
+        - The function is blocking until all tasks complete or an exception is propagated.
+        - Use `on_error` to implement "best-effort" processing where failures are logged
+          and the rest continue.
+    """
+    # Defensive copy to avoid consuming a generator multiple times and to compute pool size.
+    pending_items: List[T] = list(items)
+    if not pending_items:
+        return []
+
+    # Include indices to ensure results are returned in input order
+    indexed_items = list(enumerate(items))
+    results: List[Optional[R]] = [None] * len(items)  # Preallocate results list
+
+    # Bound the pool by actual workload size for efficiency.
+    with ThreadPoolExecutor(max_workers=min(len(indexed_items), max_workers)) as executor:
+        # Submit all tasks up-front and map futures back to their originating item.
+        future_to_index = {executor.submit(worker, item): index for index, item in indexed_items}
+
+        # Progress bar reflects total number of submitted tasks; updated per finished future.
+        with tqdm(total=len(indexed_items), desc=desc, mininterval=1, dynamic_ncols=True) as pbar:
+            # Track unfinished futures and poll with a timeout to enable heartbeat logs.
+            pending = set(future_to_index.keys())
+            while pending:
+                # Wait with timeout to detect stalls and emit heartbeats proactively.
+                done, not_done = wait(pending, timeout=heartbeat_sec)
+                if not done:
+                    # Heartbeat when nothing has completed within the window.
+                    logger.info(f'{desc} still processing... pending={len(not_done)}')
+                    continue
+
+                # Consume completed futures.
+                for future in done:
+                    index = future_to_index[future]
+                    try:
+                        res = future.result()
+                        results[index] = res  # Store result at the correct index
+                        # Invoke success callback in caller thread (not in worker).
+                        if on_result is not None:
+                            on_result(items[index], res)
+                    except Exception as exc:
+                        # Delegate failure handling to on_error if provided; otherwise bubble up.
+                        if on_error is not None:
+                            on_error(items[index], exc)
+                        else:
+                            raise
+                    finally:
+                        # Always advance progress for completed futures (success or failure).
+                        pbar.update(1)
+
+                # Continue polling remaining futures.
+                pending = not_done
+
+    # Return results, which are now guaranteed to be in input order
+    if filter_none_results:
+        # Filter out None results if on_error was used and some tasks failed
+        results = [res for res in results if res is not None]
+    return results

evalscope/utils/import_utils.py

@@ -0,0 +1,154 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright 2023-present the HuggingFace Inc. team.
+
+import importlib
+import os
+from itertools import chain
+from types import ModuleType
+from typing import Any, Optional, Union
+
+from evalscope.constants import IS_BUILD_DOC
+from .logger import get_logger
+
+logger = get_logger()  # pylint: disable=invalid-name
+
+
+def check_import(
+    module_name: Union[str, list[str]],
+    package: Optional[Union[str, list[str]]] = None,
+    raise_warning: bool = True,
+    raise_error: bool = False,
+    feature_name: Optional[str] = 'this feature',
+) -> bool:
+    """Check if a module or list of modules can be imported.
+
+    Args:
+        module_name (Union[str, list[str]]): The name(s) of the module(s) to check.
+        package (Union[str, list[str]], optional): The package(s) to install if the module(s) are not found.
+            Defaults to None.
+        raise_error (bool, optional): Whether to raise an error if any module is not found. Defaults to False.
+        raise_warning (bool, optional): Whether to log a warning if any module is not found. Defaults to True.
+        feature_name (str, optional): The feature name that requires the module(s). Used in the warning/error message.
+            Defaults to 'this feature'.
+
+    Returns:
+        bool: True if all modules can be imported, False otherwise.
+    """
+    # Convert single strings to lists for uniform processing
+    if isinstance(module_name, str):
+        module_names = [module_name]
+    else:
+        module_names = module_name
+
+    if package is None:
+        packages = [None] * len(module_names)
+    elif isinstance(package, str):
+        packages = [package] * len(module_names)
+    else:
+        packages = package
+        # Ensure packages list has same length as module_names
+        if len(packages) < len(module_names):
+            packages.extend([None] * (len(module_names) - len(packages)))
+
+    missing_modules = []
+    missing_packages = []
+
+    for i, mod_name in enumerate(module_names):
+        try:
+            importlib.import_module(mod_name)
+        except ImportError:
+            missing_modules.append(mod_name)
+            if i < len(packages) and packages[i]:
+                missing_packages.append(packages[i])
+
+    if missing_modules:
+        if len(missing_modules) == 1:
+            error_msg = f'`{missing_modules[0]}` not found.'
+        else:
+            error_msg = f'The following modules are not found: {", ".join(f"`{mod}`" for mod in missing_modules)}.'
+
+        if missing_packages:
+            if len(missing_packages) == 1:
+                error_msg += f' Please run `pip install {missing_packages[0]}` to use {feature_name}.'
+            else:
+                unique_packages = list(dict.fromkeys(missing_packages))  # Remove duplicates while preserving order
+                error_msg += f' Please run `pip install {" ".join(unique_packages)}` to use {feature_name}.'
+
+        if raise_warning:
+            logger.warning(error_msg)
+
+        if not IS_BUILD_DOC and raise_error:
+            raise ImportError(error_msg)
+        return False
+
+    return True
+
+
+class _LazyModule(ModuleType):
+    """
+    Module class that surfaces all objects but only performs associated imports when the objects are requested.
+    """
+
+    # Very heavily inspired by optuna.integration._IntegrationModule
+    # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py
+    def __init__(self, name, module_file, import_structure, module_spec=None, extra_objects=None):
+        super().__init__(name)
+        self._modules = set(import_structure.keys())
+        self._class_to_module = {}
+        for key, values in import_structure.items():
+            for value in values:
+                self._class_to_module[value] = key
+        # Needed for autocompletion in an IDE
+        self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values()))
+        self.__file__ = module_file
+        self.__spec__ = module_spec
+        self.__path__ = [os.path.dirname(module_file)]
+        self._objects = {} if extra_objects is None else extra_objects
+        self._name = name
+        self._import_structure = import_structure
+
+    # Needed for autocompletion in an IDE
+    def __dir__(self):
+        result = super().__dir__()
+        # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether
+        # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir.
+        for attr in self.__all__:
+            if attr not in result:
+                result.append(attr)
+        return result
+
+    def __getattr__(self, name: str) -> Any:
+        if name in self._objects:
+            return self._objects[name]
+        if name in self._modules:
+            value = self._get_module(name)
+        elif name in self._class_to_module.keys():
+            module = self._get_module(self._class_to_module[name])
+            value = getattr(module, name)
+        else:
+            raise AttributeError(f'module {self.__name__} has no attribute {name}')
+
+        setattr(self, name, value)
+        return value
+
+    def _get_module(self, module_name: str):
+        return importlib.import_module('.' + module_name, self.__name__)
+
+    def __reduce__(self):
+        return self.__class__, (self._name, self.__file__, self._import_structure)
+
+
+def is_module_installed(module_name):
+    try:
+        importlib.import_module(module_name)
+        return True
+    except ImportError:
+        return False
+
+
+def get_module_path(module_name):
+    spec = importlib.util.find_spec(module_name)
+    if spec and spec.origin:
+        return os.path.abspath(spec.origin)
+    else:
+        raise ValueError(f'Cannot find module: {module_name}')