lazysdk 0.1.32__py3-none-any.whl → 0.1.34__py3-none-any.whl

lazysdk/lazyfile.py

@@ -338,7 +338,7 @@ def save(
         content,
         postfix: str = None,
         path: str = None,
-        overwrite: str = True,
+        overwrite: bool = True,
         encoding: str = 'utf-8'
 ):
     """

lazysdk/lazyprocess.py

@@ -22,12 +22,11 @@ def run(
         subprocess_limit: int = None,
         master_process_delay: int = 1,
         return_data: bool = False,
-
         silence: bool = False
 ):
     """
     多进程 进程控制
-    :param task_list: 任务列表，list格式
+    :param task_list: 任务列表，list格式，会将list中的每个元素传入给task_function中的task_info，作为任务的详细信息；
     :param task_function: 子任务的function，需提前写好，入参为：(task_index, task_info)，例如：task_function(task_index, task_info)
     :param subprocess_keep: 是否保持子进程，True为保持进程，死掉会自动重启；False为不保持，自然退出
     :param subprocess_limit: 进程数限制，0为无限制，否则按照设定的数量限制并行的子进程数量
@@ -43,65 +42,72 @@ def run(
         # 进程详细的内容
         print(task_index, task_info)
     """
+    inner_task_list = copy.deepcopy(task_list)  # 深度拷贝，防止篡改
     if subprocess_limit:
+        # 存在自定义的子进程数量限制，将采用
         pass
     else:
+        # 不存在自定义的子进程数量限制，将使用默认计算方式
         if os_cpu_count > 1:
-            subprocess_limit = os_cpu_count - 1  # 进程数设置为cpu核心数减1
+            # 如果cpu核心数大于1个
+            subprocess_limit = os_cpu_count - 1  # 子进程数设置为cpu核心数减1
         else:
+            # 如果cpu核心数等于1个
             subprocess_limit = 1
-    active_process = dict()  # 活跃进程，存放进程，以task_index为key，进程信息为value的dict
-    task_count = len(task_list)  # 总任务数量
+    active_process = dict()  # 存放活跃进程进程，以task_index为key，进程信息为value的dict
+    total_task_num = len(inner_task_list)  # 总任务数量
     task_index_start = 0  # 用来计算启动的累计进程数
-    q = Queue()  # 生成一个队列对象，以实现进程通信
+    if return_data:
+        q = Queue()  # 生成一个队列对象，以实现进程通信
+    else:
+        pass
 
-    showlog.info(f'正在准备多进程执行任务，总任务数为：{task_count}，进程数限制为：{subprocess_limit}...')
-    # 创建并启动线程
+    if silence:
+        pass
+    else:
+        showlog.info(f'正在准备多进程执行任务，总任务数为：{total_task_num}，进程数限制为：{subprocess_limit}...')
+    # 创建并启动进程
     while True:
-        this_time_start = copy.deepcopy(task_index_start)  # 确定本次循环的起点任务序号
-        for task_index in range(this_time_start, task_count):  # 按照任务量遍历
-            if task_index in active_process.keys():
-                # 进程已存在，不重复创建，跳过
-                continue
-            else:
-                # 进程不存在，待定
-                if subprocess_limit > 0:
-                    # 存在子进程数量限制规则，待定
-                    if len(active_process.keys()) >= subprocess_limit:
-                        # 当前活跃进程数量达到子进程数限制，本次循环不再新增进程，跳出
-                        if silence is False:
-                            showlog.warning('达到进程数限制')
-                        break
-                    else:
-                        # 未达到进程数限制
-                        pass
-                else:
-                    pass
-                # 不存在子进程限制规则/当前活跃进程数量未达到进程数限制，将开启一个新进程
+        this_time_start = copy.deepcopy(task_index_start)  # 深度拷贝累积启动的进程数，以确定本次循环的起点任务序号，假设subprocess_keep为False
+        for task_index in range(this_time_start, total_task_num):  # 按照任务量遍历
+            # 判断是否需要创建新的子进程
+            if len(active_process.keys()) >= subprocess_limit:
+                # 当前活跃进程数量达到子进程数限制，本次循环不再新增进程，跳出
                 if silence is False:
-                    showlog.info(f'发现需要开启的进程：{task_index}')
-                task_info = task_list[task_index]  # 提取将开启的进程的任务内容
-                # ---------- 开启进程 ----------
-                if return_data is True:
-                    p = Process(
-                        target=task_function,
-                        args=(task_index, task_info, q)
-                    )
+                    showlog.warning(f'达到子进程数限制：{subprocess_limit}')
+                break
+            else:
+                # 未达到进程数限制
+                if task_index in active_process.keys():
+                    # 进程已存在，不重复创建，跳过
+                    continue
                 else:
-                    p = Process(
-                        target=task_function,
-                        args=(task_index, task_info)
-                    )
-                p.start()
-                # ---------- 开启进程 ----------
-                active_process[task_index] = {
-                    'task_index': task_index,  # 任务序号
-                    'task_info': task_info,  # 任务详情
-                    'process': p,  # 进程对象
-                }  # 记录开启的进程
-                if silence is False:
-                    showlog.info(f'进程：{task_index} 已开启')
-                task_index_start += 1  # 记录累计开启进程数
+                    # 进程不存在，待定
+                    # 不存在子进程限制规则/当前活跃进程数量未达到进程数限制，将开启一个新进程
+                    if silence is False:
+                        showlog.info(f'发现需要开启的子进程：{task_index}/{total_task_num}')
+                    task_info = inner_task_list[task_index]  # 提取将开启的进程的任务内容
+                    # ---------- 开启进程 ----------
+                    if return_data is True:
+                        p = Process(
+                            target=task_function,
+                            args=(task_index, task_info, q)
+                        )
+                    else:
+                        p = Process(
+                            target=task_function,
+                            args=(task_index, task_info)
+                        )
+                    p.start()
+                    # ---------- 开启进程 ----------
+                    active_process[task_index] = {
+                        'task_index': task_index,  # 任务序号
+                        'task_info': task_info,  # 任务详情
+                        'process': p,  # 进程对象
+                    }  # 记录开启的进程
+                    if silence is False:
+                        showlog.info(f'子进程：{task_index}/{total_task_num} 已开启')
+                    task_index_start += 1  # 记录累计开启进程数
 
         # 检测非活跃进程，并从active_process中剔除非活跃进程，以便开启新的进程
         inactive_process_temp = list()  # 非活跃进程
@@ -109,19 +115,29 @@ def run(
             # print(q.qsize())
             # print(q.get())
             # print(q.get_nowait())
-            if process_info['process'].is_alive() is False:
+            if process_info['process'].is_alive() is True:
+                # 该子进程仍然运行
+                continue
+            else:
+                # 该子进程停止运行
                 if silence is False:
                     showlog.warning(f'进程 {process_index} 不活跃，将被剔除...')
                 inactive_process_temp.append(process_index)
+                # 尝试终止进程
+                process_info['process'].terminate()
+                process_info['process'].join()
+
         if inactive_process_temp:
+            # 存在需要剔除的子进程
             for each_dead_process in inactive_process_temp:
                 active_process.pop(each_dead_process)
         else:
+            # 不存在需要剔除的子进程
             pass
 
         showlog.info(f'>> 当前活跃进程数：{len(active_process.keys())}')
 
-        if task_index_start >= len(task_list) and len(active_process.keys()) == 0:
+        if task_index_start >= len(inner_task_list) and len(active_process.keys()) == 0:
             if silence is False:
                 showlog.info('全部任务执行完成')
             if subprocess_keep is True:
@@ -131,3 +147,34 @@ def run(
         else:
             pass
         time.sleep(master_process_delay)
+
+
+def task_function_demo(
+        task_index,
+        task_info,
+        q=None
+):
+    print(task_index, task_info)
+    time.sleep(5)
+    q.put(task_index)
+
+
+if __name__ == '__main__':
+    task_list_demo = [
+        {'task_id': 1},
+        {'task_id': 2},
+        {'task_id': 3},
+        {'task_id': 4},
+        {'task_id': 5},
+        {'task_id': 6},
+        {'task_id': 7},
+        {'task_id': 8},
+        {'task_id': 9},
+        {'task_id': 10},
+        {'task_id': 11},
+    ]
+    run(
+        task_list=task_list_demo,
+        task_function=task_function_demo,
+        return_data=True
+    )

lazysdk/lazyurl.py

@@ -1,14 +1,35 @@
 import urllib.parse as urlparse
 
 
-def get_url_params(url):
+def get_url_params(
+        url: str
+):
     """
     获取url的params参数，返回dict形式
     """
     params_str = urlparse.urlsplit(url).query
-    params_str_split = params_str.split('&')
-    params_dict = dict()
-    for each in params_str_split:
-        each_split = each.split('=', maxsplit=1)
-        params_dict[each_split[0]] = each_split[1]
-    return params_dict
+    if params_str:
+        params_str_split = params_str.split('&')
+        params_dict = dict()
+        for each in params_str_split:
+            each_split = each.split('=', maxsplit=1)
+            params_dict[each_split[0]] = each_split[1]
+        return params_dict
+    else:
+        return
+
+
+def url_info(
+        url: str
+):
+    url_info_dict = dict()
+    url_info_dict['url'] = url
+    if url:
+        urlparse_obj = urlparse.urlsplit(url)
+        url_info_dict['host'] = urlparse_obj.hostname  # 域名
+        url_info_dict['path'] = urlparse_obj.path  # 路径
+        url_info_dict['scheme'] = urlparse_obj.scheme  # 协议
+        url_info_dict['params'] = get_url_params(url)
+    else:
+        pass
+    return url_info_dict

{lazysdk-0.1.32.dist-info → lazysdk-0.1.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lazysdk
-Version: 0.1.32
+Version: 0.1.34
 Summary: 基于Python的懒人包
 Home-page: https://gitee.com/ZeroSeeker/lazysdk
 Author: ZeroSeeker

{lazysdk-0.1.32.dist-info → lazysdk-0.1.34.dist-info}/RECORD

@@ -5,14 +5,14 @@ lazysdk/lazybase64.py,sha256=KUes0YYmC29y6wF79KWMLV4a7r89Kh3mc10JQvoIY7E,555
 lazysdk/lazydict.py,sha256=uPc0YtLrFtBskLooChUbjapfjJ27HNG92cFRqGiTxdQ,7636
 lazysdk/lazyenv.py,sha256=P6HjO9cbBPzHD_Vs-Ww_GThDq-h9Wn31VSiu9OzTNWI,3024
 lazysdk/lazyexcel.py,sha256=r8_VbCzB1dRhMbK4Dwk-cr5keJCo6AV4r8SYe4O5y6k,15129
-lazysdk/lazyfile.py,sha256=1Hps7G8q5qHq65v2iQPMo7JGOI-deSN4VbkZPDIaOYw,20805
+lazysdk/lazyfile.py,sha256=VgecarqYVii0fqTdjkmHXrdhN_-9HYT0kdU17XGBwAI,20806
 lazysdk/lazyflask.py,sha256=VUhJqtNz_evVyGOS40FKkkQWWrqGEUZ26cv1RQxS388,727
 lazysdk/lazyid.py,sha256=pEmRpN78-6vdz4hzLxgoCSHJw6gvnOVQyNjqusZZLQE,607
 lazysdk/lazyip.py,sha256=ZM4N0nCimW4r4z1_Bymb-ZAfabR_3L-hB5Biyyz77-I,451
 lazysdk/lazym3u8.py,sha256=Gd40ZFRrEj2HAAvGzfC5g390Jk7o6pnptDfsNRV0Vjc,31939
 lazysdk/lazymd5.py,sha256=JSKIcUmH2zwPDDJAd9XajTYd59eKidoKAkLLNU8OHM4,1112
 lazysdk/lazypath.py,sha256=Y21B4yHePdclSnp5HghKhlJv-YcCCbz2HMmzP4ZcRUY,6622
-lazysdk/lazyprocess.py,sha256=MatyUWr4Vr-egOwG2szLxsV4MdfiqKbWL-du8dA5dKE,5645
+lazysdk/lazyprocess.py,sha256=ZVCWOcRYYlUm5NJVb4h8g49xra9omY9oWEQtb8W_RuE,7203
 lazysdk/lazyproxies.py,sha256=bljLKkP3WksZ6jOHhjS5-jSHdUBRH6gx-BxEZTPJeXc,546
 lazysdk/lazyrandom.py,sha256=XZ8hTcNvc68FAI-BfaIt7QpCmzZ7m4RGDrfpy_U7zTw,1420
 lazysdk/lazyredis.py,sha256=78jM_tXNgUKULcp08CQEHmfAlzwlWuOsl6SuLwoLDdM,37918
@@ -20,9 +20,9 @@ lazysdk/lazyrequests.py,sha256=hLVbzoaT10DJhpNwrZfGfCioEy02CAUqD7EGkb3V_Eo,2848
 lazysdk/lazytext.py,sha256=dKnRZau2VTn6up0krOhmMSPAF5Uf7IEMsTH3D5w9Hac,1828
 lazysdk/lazytime.py,sha256=lOQ5U8iWa9dp1ZJ8SjfZ2FRBj9t43SsteYbxDcXkFzo,23199
 lazysdk/lazyua.py,sha256=QI4EgZAMF9AqXJJvaDEzsMSTbsR47_y8CkJkfPbX-kM,3783
-lazysdk/lazyurl.py,sha256=5sCdVkaQ7_OsQ9AIwjTkBLXQR9SLP27hBCn-J6v04RY,396
+lazysdk/lazyurl.py,sha256=YexBUKue1w7pl22RCcHPZPOfYeE6z4yB69pIenM52Xw,921
 lazysdk/lazywebhook.py,sha256=HwJIZ5j3wUpnEfPkFY-Xq9T_9xdhCJ71dYsWNX22_3Y,8386
-lazysdk-0.1.32.dist-info/METADATA,sha256=vI_Cwtr1ZRYt5Op2d2l6yXU-HG8mn5joNUxCbzrhiPw,1733
-lazysdk-0.1.32.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-lazysdk-0.1.32.dist-info/top_level.txt,sha256=--bGS42ZHUhVeO83y1wfvKFg6OjkTLxQ4V4riqPQljY,8
-lazysdk-0.1.32.dist-info/RECORD,,
+lazysdk-0.1.34.dist-info/METADATA,sha256=28S0w8RHV0wFd5fQA_4IfHxWFfhFFma2rr41G7HUlS8,1733
+lazysdk-0.1.34.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+lazysdk-0.1.34.dist-info/top_level.txt,sha256=--bGS42ZHUhVeO83y1wfvKFg6OjkTLxQ4V4riqPQljY,8
+lazysdk-0.1.34.dist-info/RECORD,,