oafuncs 0.0.98.3__py3-none-any.whl → 0.0.98.4__py3-none-any.whl

oafuncs/_script/parallel.py

@@ -1,565 +1,214 @@
-#!/usr/bin/env python
-# coding=utf-8
-"""
-Author: Liu Kun && 16031215@qq.com
-Date: 2025-04-04 20:19:23
-LastEditors: Liu Kun && 16031215@qq.com
-LastEditTime: 2025-04-04 20:19:23
-FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\_script\\parallel.py
-Description:
-EditPlatform: vscode
-ComputerInfo: XPS 15 9510
-SystemInfo: Windows 11
-Python Version: 3.12
-"""
-
-
-
-import contextlib
+import atexit
 import logging
 import multiprocessing as mp
-import os
 import platform
+import threading
 import time
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 import psutil
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
-__all__ = ["Simple_ParallelExecutor", "ParallelExecutor"]
-
-
-class Simple_ParallelExecutor:
-    """
-    A class for parallel execution of tasks using threads or processes.
-
-    If mode is "process", the tasks are executed in separate processes.
-    If mode is "thread", the tasks are executed in separate threads.
-
-    Parameters:
-        mode (str): The execution mode. Supported values are "process" and "thread".
-                    process ~ Must use top function to run, can't use in jupyter notebook
-                    thread ~ Function can not be top function, can use in jupyter notebook
-        max_workers (int): The maximum number of workers to use. Defaults to CPU count - 1.
-
-    Note:!!!
-    If Jupyter notebook is used, the mode should be "thread" to avoid hanging issues.
-    """
-
-    def __init__(self, mode="process", max_workers=None):
-        if mode not in {"process", "thread"}:
-            raise ValueError("Invalid mode. Supported values are 'process' and 'thread'.")
-        # process: Must use top function to run, can't use in jupyter notebook
-        # thread: Can use in jupyter notebook
-        self.mode = mode
-        self.max_workers = max_workers or max(1, mp.cpu_count() - 1)
-        self.executor_class = ProcessPoolExecutor if mode == "process" else ThreadPoolExecutor
-
-    def run(self, func, param_list):
-        """
-        Run a function in parallel using the specified executor.
-
-        Args:
-            func (callable): The function to execute.
-            param_list (list): A list of parameter tuples to pass to the function.
-
-        Returns:
-            list: Results of the function execution.
-        """
-        if not callable(func):
-            raise ValueError("func must be callable.")
-        if not isinstance(param_list, list) or not all(isinstance(p, tuple) for p in param_list):
-            raise ValueError("param_list must be a list of tuples.")
-
-        results = [None] * len(param_list)
-        logging.info("Starting parallel execution in %s mode with %d workers.", self.mode, self.max_workers)
-
-        with self.executor_class(max_workers=self.max_workers) as executor:
-            future_to_index = {executor.submit(func, *params): idx for idx, params in enumerate(param_list)}
-
-            for future in as_completed(future_to_index):
-                idx = future_to_index[future]
-                try:
-                    results[idx] = future.result()
-                except Exception as e:
-                    logging.error("Task %d failed with error: %s", idx, e)
-                    results[idx] = e
-
-        logging.info("Parallel execution completed.")
-        return results
-
-
-def _compute_square(x):
-    return x * x
-
-
-def _example():
-    def _compute_sum(a, b):
-        return a + b
-
-    executor1 = Simple_ParallelExecutor(mode="process", max_workers=4)
-    params1 = [(i,) for i in range(10)]
-    results1 = executor1.run(_compute_square, params1)
-    print("Results (compute_square):", results1)
-
-    executor2 = Simple_ParallelExecutor(mode="thread", max_workers=2)
-    params2 = [(1, 2), (3, 4), (5, 6)]
-    results2 = executor2.run(_compute_sum, params2)
-    print("Results (compute_sum):", results2)
+__all__ = ["ParallelExecutor"]
 
 
 class ParallelExecutor:
-    """
-    自动优化的并行执行器，根据平台和任务特性自动选择最佳执行模式和工作线程/进程数量。
-
-    特性:
-    - 自动检测平台并选择最佳执行模式
-    - 动态调整工作线程/进程数量
-    - 针对Linux和Windows的特定优化
-    - 任务批处理功能以提高小任务的效率
-    - 自动故障转移机制
-    """
-
-    def __init__(self):
-        # 检测平台
+    def __init__(
+        self,
+        max_workers: Optional[int] = None,
+        chunk_size: Optional[int] = None,
+        mem_per_process: float = 1.0,  # GB
+        timeout_per_task: int = 3600,
+        max_retries: int = 3,
+    ):
         self.platform = self._detect_platform()
-        # 自动选择最佳执行模式和工作线程/进程数量
-        self.mode, self.max_workers = self._determine_optimal_settings()
-        # 初始化执行器
+        self.mem_per_process = mem_per_process
+        self.timeout_per_task = timeout_per_task
+        self.max_retries = max_retries
+        self.running = True
+        self.task_history = []
         self._executor = None
-        self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
-        # 进程池重用策略
-        self.reuse_pool = self.mode == "process" and self.platform != "windows"
-
-        # 特定于平台的优化参数
-        self.mp_context = None
-        self.chunk_size = self._get_default_chunk_size()
-        self.timeout_per_task = 3600  # 默认任务超时时间（秒）
-        self.worker_init_func = None
-
-        # 针对Linux的特定优化
-        if self.platform == "linux":
-            self._setup_linux_optimizations()
-        # 针对Windows的特定优化
-        elif self.platform == "windows":
-            self._setup_windows_optimizations()
 
-        logging.info(f"Initialized {self.__class__.__name__} with mode={self.mode}, max_workers={self.max_workers} on {self.platform} platform")
+        self.mode, default_workers = self._determine_optimal_settings()
+        self.max_workers = max_workers or default_workers
+        self.chunk_size = chunk_size or self._get_default_chunk_size()
 
-    def _detect_platform(self):
-        """检测当前运行的平台"""
-        system = platform.system().lower()
-        if system == "linux":
-            return "linux"
-        elif system == "windows":
-            return "windows"
-        elif system == "darwin":
-            return "macos"
-        else:
-            return "unknown"
+        self._init_platform_settings()
+        self._start_resource_monitor()
+        atexit.register(self.shutdown)
 
-    def _determine_optimal_settings(self):
-        """确定最佳执行模式和工作线程/进程数量"""
-        mode = "process"  # 默认使用进程模式
+        logging.info(f"Initialized {self.__class__.__name__} on {self.platform} (mode={self.mode}, workers={self.max_workers})")
 
-        # Linux平台优化
-        if self.platform == "linux":
-            # 在Linux上，根据之前的问题，我们优先使用进程模式
-            mode = "process"
-
-            # 检查是否在容器中运行（如Docker）
-            in_container = self._is_in_container()
-
-            # 获取物理和逻辑CPU核心数
-            physical_cores = psutil.cpu_count(logical=False) or 1
-            logical_cores = psutil.cpu_count(logical=True) or 1
-
-            # 获取系统内存信息
-            mem = psutil.virtual_memory()
-            # total_mem_gb = mem.total / (1024**3)
-            available_mem_gb = mem.available / (1024**3)
-
-            # 每个进程估计内存使用（根据应用程序特性调整）
-            est_mem_per_process_gb = 0.5
-
-            # 根据可用内存限制工作进程数
-            mem_limited_workers = max(1, int(available_mem_gb / est_mem_per_process_gb))
-
-            # 在容器环境中更保守一些
-            if in_container:
-                max_workers = min(physical_cores, mem_limited_workers, 4)
-            else:
-                max_workers = min(logical_cores, mem_limited_workers)
+    def _detect_platform(self) -> str:
+        system = platform.system().lower()
+        if system == "linux":
+            return "wsl" if "microsoft" in platform.release().lower() else "linux"
+        return system
 
-        # Windows平台优化
+    def _init_platform_settings(self):
+        if self.platform in ["linux", "wsl"]:
+            self.mp_context = mp.get_context("fork")
         elif self.platform == "windows":
-            # Windows上进程创建较快，线程和进程都可以考虑
-            # 但进程间通信开销大，所以对于小型任务，线程可能更高效
-            mode = "process"  # 默认也使用进程模式，因为通常更可靠
-
-            # Windows通常使用超线程，所以我们可以使用逻辑核心数
-            logical_cores = psutil.cpu_count(logical=True) or 1
-
-            # Windows建议使用更少的进程以减少开销
-            if logical_cores > 4:
-                max_workers = logical_cores - 1
-            else:
-                max_workers = max(1, logical_cores)
-
-        # macOS平台优化
-        elif self.platform == "macos":
-            mode = "process"
-            logical_cores = psutil.cpu_count(logical=True) or 1
-            max_workers = max(1, logical_cores - 1)
-
-        # 未知平台的保守设置
+            mp.set_start_method("spawn", force=True)
+            self.mp_context = mp.get_context("spawn")
         else:
-            mode = "process"
-            max_workers = max(1, (psutil.cpu_count(logical=True) or 2) - 1)
-
-        return mode, max_workers
-
-    def _is_in_container(self):
-        """检测是否在容器环境中运行"""
-        # 检查常见的容器环境指标
-        if os.path.exists("/.dockerenv"):
-            return True
-
-        try:
-            with open("/proc/1/cgroup", "rt") as f:
-                return any(("docker" in line or "kubepods" in line) for line in f)
-        except Exception:
-            pass
-
-        return False
-
-    def _setup_linux_optimizations(self):
-        """设置Linux特定的优化参数"""
-        try:
-            # 在Linux上，选择最适合的多进程上下文
-            # fork: 最快但可能会导致多线程程序出现问题
-            # spawn: 更安全但更慢
-            # forkserver: 中间解决方案
-
-            # 根据应用程序特性选择合适的上下文
-            self.mp_context = mp.get_context("fork")
-
-            # 设置进程初始化函数来设置CPU亲和性
-            self.worker_init_func = self._linux_worker_init
-
-        except Exception as e:
-            logging.warning(f"Failed to set Linux optimizations: {e}")
             self.mp_context = None
 
-    def _setup_windows_optimizations(self):
-        """设置Windows特定的优化参数"""
-        # Windows优化参数
-        # 进程创建和启动开销在Windows上较高，因此增加每批的任务数
-        self.chunk_size = 10
-        # Windows通常不需要特殊的工作进程初始化
-        self.worker_init_func = None
+    def _determine_optimal_settings(self) -> Tuple[str, int]:
+        logical_cores = psutil.cpu_count(logical=True) or 1
+        available_mem = psutil.virtual_memory().available / 1024**3  # GB
 
-    def _linux_worker_init(self):
-        """Linux工作进程初始化函数"""
-        try:
-            # 获取当前进程
-            p = psutil.Process()
+        mem_limit = max(1, int(available_mem / self.mem_per_process))
+        return ("process", min(logical_cores, mem_limit))
 
-            # 设置进程优先级为稍低于正常，以避免争抢重要系统资源
-            p.nice(10)
+    def _get_default_chunk_size(self) -> int:
+        return max(10, 100 // (psutil.cpu_count() or 1))
 
-            # 尝试设置CPU亲和性以提高缓存局部性
-            # 这里我们不设置特定的CPU核心，让系统调度，因为手动设置可能导致不平衡
+    def _start_resource_monitor(self):
+        def monitor():
+            threshold = self.mem_per_process * 1024**3
+            while self.running:
+                try:
+                    if psutil.virtual_memory().available < threshold:
+                        self._scale_down_workers()
+                    time.sleep(1)
+                except Exception as e:
+                    logging.error(f"Resource monitor error: {e}")
 
-            # 设置进程I/O优先级
-            # 需要root权限，所以只是尝试一下
-            try:
-                os.system(f"ionice -c 2 -n 4 -p {os.getpid()} > /dev/null 2>&1")
-            except Exception:
-                pass
-
-        except Exception as e:
-            logging.debug(f"Worker initialization warning (non-critical): {e}")
-            pass  # 失败不中断程序运行
-
-    def _get_default_chunk_size(self):
-        """获取默认任务分块大小"""
-        if self.platform == "linux":
-            # Linux下进程创建较快，可以使用较小的块大小
-            return 5
-        elif self.platform == "windows":
-            # Windows下进程创建较慢，使用较大的块大小
-            return 10
-        else:
-            return 5
+        threading.Thread(target=monitor, daemon=True).start()
 
-    @property
-    def executor(self):
-        """懒加载并重用执行器"""
-        if self._executor is None and self.reuse_pool:
-            kwargs = {}
-            if self.mode == "process" and self.mp_context:
-                kwargs["mp_context"] = self.mp_context
+    def _scale_down_workers(self):
+        if self.max_workers > 1:
+            new_count = self.max_workers - 1
+            logging.warning(f"Scaling down workers from {self.max_workers} to {new_count}")
+            self.max_workers = new_count
+            self._restart_executor()
 
-            if self.worker_init_func and self.mode == "process":
-                kwargs["initializer"] = self.worker_init_func
+    def _restart_executor(self):
+        if self._executor:
+            self._executor.shutdown(wait=False)
+            self._executor = None
 
-            self._executor = self.executor_class(max_workers=self.max_workers, **kwargs)
+    def _get_executor(self):
+        if not self._executor:
+            Executor = ThreadPoolExecutor if self.mode == "thread" else ProcessPoolExecutor
+            self._executor = Executor(max_workers=self.max_workers, mp_context=self.mp_context if self.mode == "process" else None)
         return self._executor
 
-    @contextlib.contextmanager
-    def get_executor(self):
-        """获取执行器的上下文管理器"""
-        if self.reuse_pool and self._executor:
-            yield self._executor
-        else:
-            kwargs = {}
-            if self.mode == "process" and self.mp_context:
-                kwargs["mp_context"] = self.mp_context
-
-            if self.worker_init_func and self.mode == "process":
-                kwargs["initializer"] = self.worker_init_func
-
-            with self.executor_class(max_workers=self.max_workers, **kwargs) as executor:
-                yield executor
-
-    def run(self, func, param_list, chunk_size=None, fallback_on_failure=True):
-        """
-        并行执行函数
-
-        Args:
-            func (callable): 要执行的函数
-            param_list (list): 参数元组列表
-            chunk_size (int, optional): 任务分块大小，None表示使用默认值
-            fallback_on_failure (bool): 如果主执行模式失败，是否尝试其他模式
-
-        Returns:
-            list: 函数执行结果
-        """
-        if not callable(func):
-            raise ValueError("func must be callable.")
-        if not isinstance(param_list, list):
-            raise ValueError("param_list must be a list.")
-
-        # 空列表直接返回
-        if not param_list:
+    def run(self, func: Callable, params: List[Tuple], chunk_size: Optional[int] = None) -> List[Any]:
+        chunk_size = chunk_size or self.chunk_size
+        for retry in range(self.max_retries + 1):
+            try:
+                start_time = time.monotonic()
+                results = self._execute_batch(func, params, chunk_size)
+                self._update_settings(time.monotonic() - start_time, len(params))
+                return results
+            except Exception as e:
+                logging.error(f"Attempt {retry + 1} failed: {e}")
+                self._handle_failure()
+        raise RuntimeError(f"Failed after {self.max_retries} retries")
+
+    def _execute_batch(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
+        if not params:
             return []
 
-        # 使用默认分块大小或自定义大小
-        effective_chunk_size = chunk_size or self.chunk_size
-
-        # 任务分块处理
-        if effective_chunk_size and len(param_list) > effective_chunk_size * 2:
-            return self._run_chunked(func, param_list, effective_chunk_size)
-
-        try:
-            return self._execute(func, param_list)
-        except Exception as e:
-            if fallback_on_failure:
-                logging.warning(f"Execution failed with {self.mode} mode: {e}. Trying fallback...")
-                # 如果当前模式失败，尝试其他模式
-                old_mode = self.mode
-                self.mode = "thread" if old_mode == "process" else "process"
-                self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
-                self._executor = None  # 重置执行器
+        if len(params) > chunk_size * 2:
+            return self._chunked_execution(func, params, chunk_size)
 
+        results = [None] * len(params)
+        with self._get_executor() as executor:
+            futures = {executor.submit(func, *args): idx for idx, args in enumerate(params)}
+            for future in as_completed(futures):
+                idx = futures[future]
                 try:
-                    results = self._execute(func, param_list)
-                    logging.info(f"Fallback to {self.mode} mode succeeded.")
-                    return results
-                except Exception as e2:
-                    logging.error(f"Fallback also failed: {e2}")
-                    # 恢复原始模式
-                    self.mode = old_mode
-                    self.executor_class = ProcessPoolExecutor if self.mode == "process" else ThreadPoolExecutor
-                    self._executor = None
-                    raise
-            else:
-                raise
-
-    def _execute(self, func, param_list):
-        """内部执行方法"""
-        results = [None] * len(param_list)
-        logging.info("Starting parallel execution in %s mode with %d workers.", self.mode, self.max_workers)
-
-        start_time = time.time()
-
-        with self.get_executor() as executor:
-            future_to_index = {executor.submit(func, *params): idx for idx, params in enumerate(param_list)}
-
-            for future in as_completed(future_to_index):
-                idx = future_to_index[future]
-                try:
-                    # 添加超时保护
                     results[idx] = future.result(timeout=self.timeout_per_task)
                 except Exception as e:
-                    logging.error("Task %d failed with error: %s", idx, e)
-                    results[idx] = e
-
-        elapsed = time.time() - start_time
-        logging.info("Parallel execution completed in %.2f seconds.", elapsed)
+                    results[idx] = self._handle_error(e, func, params[idx])
         return results
 
-    def _run_chunked(self, func, param_list, chunk_size):
-        """处理大量小任务的批处理执行"""
+    def _chunked_execution(self, func: Callable, params: List[Tuple], chunk_size: int) -> List[Any]:
+        results = []
+        with self._get_executor() as executor:
+            futures = []
+            for i in range(0, len(params), chunk_size):
+                chunk = params[i : i + chunk_size]
+                futures.append(executor.submit(self._process_chunk, func, chunk))
 
-        def process_chunk(chunk):
-            return [func(*params) for params in chunk]
+            for future in as_completed(futures):
+                try:
+                    results.extend(future.result(timeout=self.timeout_per_task))
+                except Exception as e:
+                    logging.error(f"Chunk failed: {e}")
+                    results.extend([None] * chunk_size)
+        return results
+
+    @staticmethod
+    def _process_chunk(func: Callable, chunk: List[Tuple]) -> List[Any]:
+        return [func(*args) for args in chunk]
 
-        # 将参数列表分成多个块
-        chunks = [param_list[i : i + chunk_size] for i in range(0, len(param_list), chunk_size)]
+    def _update_settings(self, duration: float, task_count: int):
+        self.task_history.append((duration, task_count))
+        self.chunk_size = max(5, min(100, self.chunk_size + (1 if duration < 5 else -1)))
 
-        logging.info(f"Processing {len(param_list)} tasks in {len(chunks)} chunks of size ~{chunk_size}")
+    def _handle_error(self, error: Exception, func: Callable, args: Tuple) -> Any:
+        if isinstance(error, TimeoutError):
+            logging.warning(f"Timeout processing {func.__name__}{args}")
+        elif isinstance(error, MemoryError):
+            logging.warning("Memory error detected")
+            self._scale_down_workers()
+        else:
+            logging.error(f"Error processing {func.__name__}{args}: {str(error)}")
+        return None
 
-        chunk_results = self._execute(process_chunk, [(chunk,) for chunk in chunks])
+    def _handle_failure(self):
+        if self.max_workers > 2:
+            self.max_workers = max(1, self.max_workers // 2)
+            self._restart_executor()
 
-        # 将块结果展平成单个结果列表
-        return [result for sublist in chunk_results if isinstance(sublist, list) for result in sublist]
+    def shutdown(self):
+        self.running = False
+        if self._executor:
+            try:
+                self._executor.shutdown(wait=False)
+            except Exception as e:
+                logging.error(f"Shutdown error: {e}")
+            finally:
+                self._executor = None
 
-    def map(self, func, *iterables, timeout=None, chunk_size=None):
-        """
-        类似于内置map函数的并行版本
+    def __enter__(self):
+        return self
 
-        Args:
-            func: 要应用于每个元素的函数
-            *iterables: 一个或多个可迭代对象
-            timeout: 每个任务的超时时间
-            chunk_size: 任务分块大小
+    def __exit__(self, *exc_info):
+        self.shutdown()
 
-        Returns:
-            生成器，产生的结果与输入顺序相同
-        """
-        # 将zip后的可迭代对象转换为参数元组列表
-        param_list = [(args,) for args in zip(*iterables)]
+    def get_stats(self) -> Dict[str, Any]:
+        stats = {
+            "platform": self.platform,
+            "mode": self.mode,
+            "workers": self.max_workers,
+            "chunk_size": self.chunk_size,
+            "total_tasks": sum(count for _, count in self.task_history),
+        }
+        if self.task_history:
+            total_time = sum(time for time, _ in self.task_history)
+            stats["avg_task_throughput"] = stats["total_tasks"] / total_time if total_time else 0
+        return stats
 
-        # 临时存储超时设置
-        original_timeout = self.timeout_per_task
-        if timeout:
-            self.timeout_per_task = timeout
 
-        try:
-            results = self.run(lambda x: func(x), param_list, chunk_size=chunk_size)
-            for r in results:
-                yield r
-        finally:
-            # 恢复原超时设置
-            self.timeout_per_task = original_timeout
+def _test_func(a, b):
+    time.sleep(0.01)
+    return a + b
 
-    def __del__(self):
-        """确保资源被正确释放"""
-        self.shutdown()
 
-    def shutdown(self):
-        """显式关闭执行器"""
-        if self._executor:
-            try:
-                self._executor.shutdown(wait=True)
-            except Exception:
-                pass
-            self._executor = None
+if __name__ == "__main__":
+    params = [(i, i * 2) for i in range(1000)]
 
-    def imap(self, func, *iterables, timeout=None, chunk_size=None):
-        """
-        类似concurrent.futures.Executor.map的接口，但返回迭代器
-        """
-        return self.map(func, *iterables, timeout=timeout, chunk_size=chunk_size)
-
-    def imap_unordered(self, func, *iterables, timeout=None, chunk_size=None):
-        """
-        类似multiprocessing.Pool.imap_unordered的接口，结果可能乱序返回
-        """
-        # 将zip后的可迭代对象转换为参数元组列表
-        param_list = [(args,) for args in zip(*iterables)]
-
-        # 空列表直接返回
-        if not param_list:
-            return
-
-        # 临时存储超时设置
-        original_timeout = self.timeout_per_task
-        if timeout:
-            self.timeout_per_task = timeout
-
-        try:
-            # 使用默认分块大小或自定义大小
-            effective_chunk_size = chunk_size or self.chunk_size
-
-            # 任务分块处理
-            if effective_chunk_size and len(param_list) > effective_chunk_size * 2:
-                chunks = [param_list[i : i + effective_chunk_size] for i in range(0, len(param_list), effective_chunk_size)]
-
-                with self.get_executor() as executor:
-                    futures = [executor.submit(self._process_chunk_for_imap, func, chunk) for chunk in chunks]
-
-                    for future in as_completed(futures):
-                        try:
-                            chunk_results = future.result(timeout=self.timeout_per_task)
-                            for result in chunk_results:
-                                yield result
-                        except Exception as e:
-                            logging.error(f"Chunk processing failed: {e}")
-            else:
-                with self.get_executor() as executor:
-                    futures = [executor.submit(func, *params) for params in param_list]
-
-                    for future in as_completed(futures):
-                        try:
-                            yield future.result(timeout=self.timeout_per_task)
-                        except Exception as e:
-                            logging.error(f"Task failed: {e}")
-                            yield e
-        finally:
-            # 恢复原超时设置
-            self.timeout_per_task = original_timeout
-
-    def _process_chunk_for_imap(self, func, chunk):
-        """处理imap_unordered的数据块"""
-        return [func(*params) for params in chunk]
-
-    def starmap(self, func, iterable, timeout=None, chunk_size=None):
-        """
-        类似于内置starmap函数的并行版本
-
-        Args:
-            func: 要应用于每个元素的函数
-            iterable: 可迭代对象，每个元素是函数参数的元组
-            timeout: 每个任务的超时时间
-            chunk_size: 任务分块大小
-
-        Returns:
-            生成器，产生结果
-        """
-
-        # 将每个元素转换为单参数函数调用
-        def wrapper(args):
-            return func(*args)
-
-        # 使用map实现
-        return self.map(wrapper, iterable, timeout=timeout, chunk_size=chunk_size)
-
-    def gather(self, funcs_and_args):
-        """
-        并行执行多个不同的函数，类似于asyncio.gather
-
-        Args:
-            funcs_and_args: 列表，每个元素是(func, args)元组，
-                            其中args是要传递给func的参数元组
-
-        Returns:
-            list: 函数执行结果，顺序与输入相同
-        """
-        if not isinstance(funcs_and_args, list):
-            raise ValueError("funcs_and_args must be a list of (func, args) tuples")
-
-        def wrapper(func_and_args):
-            func, args = func_and_args
-            return func(*args)
-
-        return self.run(wrapper, [(item,) for item in funcs_and_args])
+    with ParallelExecutor() as executor:
+        results = executor.run(_test_func, params)
 
+    # print("Results:", results)
 
-if __name__ == "__main__":
-    _example()
-    # 也可以不要装饰器，直接运行没啥问题，就是避免在ipynb中使用，最好使用ipynb，或者把这个函数放到一个独立的py文件中运行
-    # 或者，jupyter中使用thread，不要使用process，因为process会导致jupyter挂掉
+    print(f"Processed {len(results)} tasks")
+    print("Execution stats:", executor.get_stats())