ezKit 1.10.0__py3-none-any.whl → 1.10.1__py3-none-any.whl

ezKit/utils.py

@@ -9,12 +9,11 @@ import time
 import tomllib
 from copy import deepcopy
 from itertools import islice
-from multiprocessing import Pool, Process
+from multiprocessing import Pool
 from multiprocessing.pool import ThreadPool
 from pathlib import Path
 from shutil import rmtree
-from threading import Thread
-from typing import Any, Callable, List, Optional, Union
+from typing import Any, Callable, List, Union
 from urllib.parse import ParseResult, urlparse
 from uuid import uuid4
 
@@ -742,31 +741,23 @@ def parent_dir(
 # --------------------------------------------------------------------------------------------------
 
 
-def retry(
-    times: int,
-    func: Callable,
-    **kwargs
-):
+def retry(func: Callable, times: int = 3, **kwargs):
     """重试"""
+
     # 函数传递参数: https://stackoverflow.com/a/803632
     # callable() 判断类型是非为函数: https://stackoverflow.com/a/624939
-    try:
-        _num = 0
-        while True:
-            # 重试次数判断 (0 表示无限次数, 这里条件使用 > 0, 表示有限次数)
-            if times > 0:
-                _num += 1
-                if _num > times:
-                    return
-            # 执行函数
-            try:
-                return func(**kwargs)
-            except Exception as e:
-                logger.exception(e)
+
+    for attempt in range(times):
+        try:
+            # 执行函数并结果
+            return func(**kwargs)
+        except Exception as e:
+            logger.exception(e)
+            if attempt < (times - 1):
                 logger.info('retrying ...')
-                continue
-    except Exception as e:
-        logger.exception(e)
+            else:
+                logger.error("all retries failed")
+                return False
 
 
 # --------------------------------------------------------------------------------------------------
@@ -1199,65 +1190,61 @@ def delete_directory(
 # --------------------------------------------------------------------------------------------------
 
 
-def process_pool(
+def processor(
     process_func: Callable,
-    process_data: Any = None,
+    process_data: List[Any],
     process_num: int = 2,
     thread: bool = False,
     **kwargs
-) -> list | bool:
-    """
-    多线程(MultiThread) | 多进程(MultiProcess)
-    """
-    # ThreadPool 线程池
+) -> Union[List[Any], bool]:
+    """使用多线程或多进程对数据进行并行处理"""
+
+    # :param process_func: 处理函数
+    # :param process_data: 待处理数据列表
+    # :param process_num: 并行数量
+    # :param thread: 是否使用多线程
+    # :param kwargs: 其他可选参数传递给线程池或进程池
+    # :return: 处理后的结果列表或 False（异常情况）
+    #
+    # MultiThread   多线程
+    # MultiProcess  多进程
+    #
+    # ThreadPool    线程池
+    # Pool          进程池
+    #
     # ThreadPool 共享内存, Pool 不共享内存
     # ThreadPool 可以解决 Pool 在某些情况下产生的 Can't pickle local object 的错误
-    # https://stackoverflow.com/a/58897266
-    try:
+    #   https://stackoverflow.com/a/58897266
+    #
+    # 如果要启动一个新的进程或者线程, 将 process_num 设置为 1 即可
 
-        # 处理数据
-        if len(process_data) <= process_num:
-            process_num = len(process_data)
-            _data = process_data
-        else:
-            _data = list_split(process_data, process_num, equally=True)
+    try:
 
-        if _data is None:
+        # 检查参数
+        if not check_arguments([(process_data, list, "process_data")]):
             return False
 
-        # 执行函数
-        if isTrue(thread, bool):
-            # 多线程
-            logger.info("execute multi thread ......")
-            with ThreadPool(process_num, **kwargs) as p:
-                return p.map(process_func, _data)
-        else:
-            # 多进程
-            logger.info("execute multi process ......")
-            with Pool(process_num, **kwargs) as p:
-                return p.map(process_func, _data)
+        # 确保并行数不超过数据量
+        process_num = min(len(process_data), process_num)
+        _data_chunks = (
+            list_split(process_data, process_num, equally=True)
+            if process_num > 1
+            else [process_data]
+        )
 
-    except Exception as e:
-        logger.exception(e)
-        return False
+        if not _data_chunks:
+            logger.error("data chunks error")
+            return False
 
+        logger.info(
+            f"Starting {'multi-threading' if thread else 'multi-processing'} with {process_num} workers..."
+        )
+
+        # 执行多线程或多进程任务
+        pool_cls = ThreadPool if thread else Pool
+        with pool_cls(process_num, **kwargs) as pool:
+            return pool.map(process_func, _data_chunks)
 
-def new_process(
-    process_func: Callable,
-    process_data: Any = None,
-    thread: bool = False,
-    daemon: bool = True,
-    **kwargs
-) -> Thread | Process | bool:
-    """New Process"""
-    try:
-        if isTrue(thread, bool):
-            process = Thread(target=process_func, args=process_data, **kwargs)
-        else:
-            process = Process(target=process_func, args=process_data, **kwargs)
-        process.daemon = daemon
-        process.start()
-        return process
     except Exception as e:
         logger.exception(e)
         return False

{ezKit-1.10.0.dist-info → ezKit-1.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ezKit
-Version: 1.10.0
+Version: 1.10.1
 Summary: Easy Kit
 Author: septvean
 Author-email: septvean@gmail.com

{ezKit-1.10.0.dist-info → ezKit-1.10.1.dist-info}/RECORD

@@ -9,10 +9,10 @@ ezKit/qywx.py,sha256=X_H4fzP-iEqeDEbumr7D1bXi6dxczaxfO8iyutzy02s,7171
 ezKit/redis.py,sha256=g2_V4jvq0djRc20jLZkgeAeF_bYrq-Rbl_kHcCUPZcA,1965
 ezKit/sendemail.py,sha256=tRXCsJm_RfTJ9xEWe_lTQ5kOs2JxHGPXvq0oWA7prq0,7263
 ezKit/token.py,sha256=HKREyZj_T2S8-aFoFIrBXTaCKExQq4zE66OHXhGHqQg,1750
-ezKit/utils.py,sha256=TDsL3PRkQy6NdZgphkgwacbWvHqEmq4LOkfNzmxV4DY,42682
+ezKit/utils.py,sha256=k3hSnOwNSyyRDVwfEzQUXQh_oJJ51KOT-PvLFPYOtOE,42517
 ezKit/xftp.py,sha256=XyIdr_2rxRVLqPofG6fIYWhAMVsFwTyp46dg5P9FLW4,7774
-ezKit-1.10.0.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-ezKit-1.10.0.dist-info/METADATA,sha256=O02PkOxx-UBmJsu2R7TIJNrTw2d4GvsLsbCgvmHmgp0,191
-ezKit-1.10.0.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-ezKit-1.10.0.dist-info/top_level.txt,sha256=aYLB_1WODsqNTsTFWcKP-BN0KCTKcV-HZJ4zlHkCFw8,6
-ezKit-1.10.0.dist-info/RECORD,,
+ezKit-1.10.1.dist-info/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+ezKit-1.10.1.dist-info/METADATA,sha256=7-BrvzaxysoECMgvFVFeBbVHX2RgQGbdYUrZS3xumUo,191
+ezKit-1.10.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+ezKit-1.10.1.dist-info/top_level.txt,sha256=aYLB_1WODsqNTsTFWcKP-BN0KCTKcV-HZJ4zlHkCFw8,6
+ezKit-1.10.1.dist-info/RECORD,,