ezKit 1.11.4__tar.gz → 1.11.5__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {ezkit-1.11.4/ezKit.egg-info → ezkit-1.11.5}/PKG-INFO +1 -1
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/utils.py +11 -13
- {ezkit-1.11.4 → ezkit-1.11.5/ezKit.egg-info}/PKG-INFO +1 -1
- {ezkit-1.11.4 → ezkit-1.11.5}/setup.py +1 -1
- {ezkit-1.11.4 → ezkit-1.11.5}/LICENSE +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/MANIFEST.in +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/README.md +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/__init__.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/_file.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/bottle.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/bottle_extensions.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/cipher.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/database.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/http.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/mongo.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/qywx.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/redis.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/sendemail.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/token.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit/xftp.py +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit.egg-info/SOURCES.txt +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit.egg-info/dependency_links.txt +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit.egg-info/requires.txt +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/ezKit.egg-info/top_level.txt +0 -0
- {ezkit-1.11.4 → ezkit-1.11.5}/setup.cfg +0 -0
@@ -7,14 +7,12 @@ import os
|
|
7
7
|
import subprocess
|
8
8
|
import time
|
9
9
|
import tomllib
|
10
|
+
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
|
10
11
|
from copy import deepcopy
|
11
12
|
from itertools import islice
|
12
|
-
from multiprocessing import Pool
|
13
|
-
from multiprocessing.pool import ThreadPool
|
14
13
|
from pathlib import Path
|
15
14
|
from shutil import rmtree
|
16
|
-
from
|
17
|
-
from typing import Any
|
15
|
+
from typing import Any, Callable
|
18
16
|
from urllib.parse import ParseResult, urlparse
|
19
17
|
from uuid import uuid4
|
20
18
|
|
@@ -742,7 +740,7 @@ def parent_dir(
|
|
742
740
|
# --------------------------------------------------------------------------------------------------
|
743
741
|
|
744
742
|
|
745
|
-
def retry(func:
|
743
|
+
def retry(func: Callable, times: int = 3, **kwargs) -> Any:
|
746
744
|
"""重试"""
|
747
745
|
|
748
746
|
# 函数传递参数: https://stackoverflow.com/a/803632
|
@@ -1191,12 +1189,12 @@ def delete_directory(
|
|
1191
1189
|
|
1192
1190
|
|
1193
1191
|
def processor(
|
1194
|
-
process_func:
|
1195
|
-
process_data: list
|
1192
|
+
process_func: Callable,
|
1193
|
+
process_data: list,
|
1196
1194
|
process_num: int = 2,
|
1197
1195
|
thread: bool = False,
|
1198
1196
|
**kwargs
|
1199
|
-
) ->
|
1197
|
+
) -> Any:
|
1200
1198
|
"""使用多线程或多进程对数据进行并行处理"""
|
1201
1199
|
|
1202
1200
|
# :param process_func: 处理函数
|
@@ -1226,13 +1224,13 @@ def processor(
|
|
1226
1224
|
|
1227
1225
|
# 确保并行数不超过数据量
|
1228
1226
|
process_num = min(len(process_data), process_num)
|
1229
|
-
|
1227
|
+
data_chunks = (
|
1230
1228
|
list_split(process_data, process_num, equally=True)
|
1231
1229
|
if process_num > 1
|
1232
1230
|
else [process_data]
|
1233
1231
|
)
|
1234
1232
|
|
1235
|
-
if not
|
1233
|
+
if not data_chunks:
|
1236
1234
|
logger.error("data chunks error")
|
1237
1235
|
return False
|
1238
1236
|
|
@@ -1241,9 +1239,9 @@ def processor(
|
|
1241
1239
|
)
|
1242
1240
|
|
1243
1241
|
# 执行多线程或多进程任务
|
1244
|
-
|
1245
|
-
with
|
1246
|
-
return
|
1242
|
+
pool = ThreadPoolExecutor if thread else ProcessPoolExecutor
|
1243
|
+
with pool(process_num, **kwargs) as executor:
|
1244
|
+
return executor.map(process_func, data_chunks)
|
1247
1245
|
|
1248
1246
|
except Exception as e:
|
1249
1247
|
logger.exception(e)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|