kevin-toolbox-dev 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kevin_toolbox/__init__.py +2 -2
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/__init__.py +1 -0
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/multi_process_execute.py +109 -0
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/multi_thread_execute.py +50 -29
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/__init__.py +15 -0
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_1.py +69 -0
- kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_2.py +76 -0
- kevin_toolbox/computer_science/data_structure/executor.py +2 -2
- kevin_toolbox/data_flow/file/json_/write_json.py +36 -3
- kevin_toolbox/nested_dict_list/serializer/backends/_json_.py +2 -2
- kevin_toolbox/patches/for_matplotlib/common_charts/plot_lines.py +5 -1
- {kevin_toolbox_dev-1.4.6.dist-info → kevin_toolbox_dev-1.4.7.dist-info}/METADATA +12 -19
- {kevin_toolbox_dev-1.4.6.dist-info → kevin_toolbox_dev-1.4.7.dist-info}/RECORD +15 -11
- {kevin_toolbox_dev-1.4.6.dist-info → kevin_toolbox_dev-1.4.7.dist-info}/WHEEL +0 -0
- {kevin_toolbox_dev-1.4.6.dist-info → kevin_toolbox_dev-1.4.7.dist-info}/top_level.txt +0 -0
kevin_toolbox/__init__.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
__version__ = "1.4.
|
1
|
+
__version__ = "1.4.7"
|
2
2
|
|
3
3
|
|
4
4
|
import os
|
@@ -12,5 +12,5 @@ os.system(
|
|
12
12
|
os.system(
|
13
13
|
f'python {os.path.split(__file__)[0]}/env_info/check_validity_and_uninstall.py '
|
14
14
|
f'--package_name kevin-toolbox-dev '
|
15
|
-
f'--expiration_timestamp
|
15
|
+
f'--expiration_timestamp 1755525740 --verbose 0'
|
16
16
|
)
|
@@ -0,0 +1,109 @@
|
|
1
|
+
import pickle
|
2
|
+
import concurrent.futures
|
3
|
+
from multiprocessing import Manager
|
4
|
+
from kevin_toolbox.computer_science.data_structure import Executor
|
5
|
+
from kevin_toolbox.computer_science.algorithm.parallel_and_concurrent.utils import wrapper_for_mp as wrapper
|
6
|
+
from kevin_toolbox.computer_science.algorithm.parallel_and_concurrent.utils import DEFAULT_PROCESS_NUMS
|
7
|
+
|
8
|
+
|
9
|
+
def multi_process_execute(executors, worker_nums=DEFAULT_PROCESS_NUMS, b_display_progress=True, timeout=None,
|
10
|
+
_hook_for_debug=None):
|
11
|
+
"""
|
12
|
+
多进程执行
|
13
|
+
|
14
|
+
参数:
|
15
|
+
executors: <list/generator/iterator of Executor> 执行器序列
|
16
|
+
worker_nums: <int> 进程数
|
17
|
+
b_display_progress: <boolean> 是否显示进度条
|
18
|
+
timeout: <int/float> 每个进程的最大等待时间,单位是s
|
19
|
+
默认为 None,表示允许等待无限长的时间
|
20
|
+
_hook_for_debug: <dict/None> 当设置为非 None 值时,将保存中间的执行信息。
|
21
|
+
包括:
|
22
|
+
- "execution_orders": 执行顺序
|
23
|
+
- "completion_orders": 完成顺序
|
24
|
+
这些信息与最终结果无关,仅面向更底层的调试需求,任何人都不应依赖该特性
|
25
|
+
返回:
|
26
|
+
res_ls, failed_idx_ls
|
27
|
+
执行结果列表,以及执行失败的执行器索引列表
|
28
|
+
"""
|
29
|
+
executor_ls = []
|
30
|
+
for i in executors:
|
31
|
+
assert isinstance(i, (Executor,))
|
32
|
+
try:
|
33
|
+
pickle.dumps(i)
|
34
|
+
except:
|
35
|
+
raise AttributeError(
|
36
|
+
f'非法任务。因为进程池中的任务必须要能被pickle化。\n对象 {i} 无法被 pickle,请检查其中是否使用了闭包内定义的函数')
|
37
|
+
executor_ls.append(i)
|
38
|
+
if b_display_progress:
|
39
|
+
from tqdm import tqdm
|
40
|
+
p_bar = tqdm(total=len(executor_ls))
|
41
|
+
else:
|
42
|
+
p_bar = None
|
43
|
+
|
44
|
+
if isinstance(_hook_for_debug, dict):
|
45
|
+
_execution_orders, _completion_orders = Manager().list(), Manager().list()
|
46
|
+
else:
|
47
|
+
_execution_orders, _completion_orders = None, None
|
48
|
+
|
49
|
+
res_ls = [None] * len(executor_ls)
|
50
|
+
failed_idx_ls = []
|
51
|
+
with concurrent.futures.ProcessPoolExecutor(max_workers=worker_nums) as process_pool:
|
52
|
+
# 提交任务并添加进度回调
|
53
|
+
futures = []
|
54
|
+
for i, executor in enumerate(executor_ls):
|
55
|
+
future = process_pool.submit(wrapper, executor, timeout, i, _execution_orders, _completion_orders)
|
56
|
+
if b_display_progress:
|
57
|
+
future.add_done_callback(lambda _: p_bar.update())
|
58
|
+
futures.append(future)
|
59
|
+
|
60
|
+
# 收集结果
|
61
|
+
for i, future in enumerate(futures):
|
62
|
+
try:
|
63
|
+
res, b_success = future.result()
|
64
|
+
except:
|
65
|
+
b_success = False
|
66
|
+
if b_success:
|
67
|
+
res_ls[i] = res
|
68
|
+
else:
|
69
|
+
failed_idx_ls.append(i)
|
70
|
+
|
71
|
+
if b_display_progress:
|
72
|
+
p_bar.close()
|
73
|
+
|
74
|
+
if isinstance(_hook_for_debug, (dict,)):
|
75
|
+
_hook_for_debug.update({
|
76
|
+
"execution_orders": list(_execution_orders),
|
77
|
+
"completion_orders": list(_completion_orders)
|
78
|
+
})
|
79
|
+
|
80
|
+
return res_ls, failed_idx_ls
|
81
|
+
|
82
|
+
|
83
|
+
if __name__ == '__main__':
|
84
|
+
import time
|
85
|
+
|
86
|
+
|
87
|
+
def func_(i):
|
88
|
+
# 模拟部分任务长时间运行,部分任务正常结束
|
89
|
+
if i in [2, 3, 7]:
|
90
|
+
time.sleep(100)
|
91
|
+
else:
|
92
|
+
time.sleep(0.01)
|
93
|
+
print(f"任务 {i} 执行完成")
|
94
|
+
return i * 2
|
95
|
+
|
96
|
+
|
97
|
+
hook_for_debug = dict()
|
98
|
+
a = time.time()
|
99
|
+
results, failed = multi_process_execute(
|
100
|
+
executors=[Executor(func=func_, args=(i,)) for i in range(10)],
|
101
|
+
worker_nums=10,
|
102
|
+
timeout=0.2,
|
103
|
+
_hook_for_debug=hook_for_debug
|
104
|
+
)
|
105
|
+
gap = time.time() - a
|
106
|
+
print("执行结果:", results)
|
107
|
+
print("超时失败的任务索引:", failed)
|
108
|
+
print("调试信息:", hook_for_debug)
|
109
|
+
print("总耗时:", gap)
|
@@ -1,14 +1,18 @@
|
|
1
1
|
import concurrent.futures
|
2
|
+
from multiprocessing import Manager
|
2
3
|
from kevin_toolbox.computer_science.data_structure import Executor
|
4
|
+
from kevin_toolbox.computer_science.algorithm.parallel_and_concurrent.utils import wrapper_for_mt as wrapper
|
5
|
+
from kevin_toolbox.computer_science.algorithm.parallel_and_concurrent.utils import DEFAULT_THREAD_NUMS
|
3
6
|
|
4
7
|
|
5
|
-
def multi_thread_execute(executors,
|
8
|
+
def multi_thread_execute(executors, worker_nums=DEFAULT_THREAD_NUMS, b_display_progress=True, timeout=None,
|
9
|
+
_hook_for_debug=None):
|
6
10
|
"""
|
7
11
|
多线程执行
|
8
12
|
|
9
13
|
参数:
|
10
14
|
executors: <list/generator/iterator of Executor> 执行器序列
|
11
|
-
|
15
|
+
worker_nums: <int> 线程数
|
12
16
|
b_display_progress: <boolean> 是否显示进度条
|
13
17
|
timeout: <int> 每个线程的最大等待时间,单位是s
|
14
18
|
默认为 None,表示允许等待无限长的时间
|
@@ -30,37 +34,42 @@ def multi_thread_execute(executors, thread_nums=50, b_display_progress=True, tim
|
|
30
34
|
p_bar = tqdm(total=len(executor_ls))
|
31
35
|
else:
|
32
36
|
p_bar = None
|
33
|
-
_execution_orders, _completion_orders = [], []
|
34
37
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
_completion_orders.append(idx)
|
40
|
-
if p_bar is not None:
|
41
|
-
p_bar.update()
|
42
|
-
|
43
|
-
return res
|
38
|
+
if isinstance(_hook_for_debug, dict):
|
39
|
+
_execution_orders, _completion_orders = Manager().list(), Manager().list()
|
40
|
+
else:
|
41
|
+
_execution_orders, _completion_orders = None, None
|
44
42
|
|
45
|
-
res_ls
|
46
|
-
|
43
|
+
res_ls = [None] * len(executor_ls)
|
44
|
+
failed_idx_ls = []
|
45
|
+
with concurrent.futures.ThreadPoolExecutor(max_workers=worker_nums) as thread_pool:
|
47
46
|
# 提交任务
|
48
|
-
futures = [
|
49
|
-
|
50
|
-
|
51
|
-
|
47
|
+
futures = []
|
48
|
+
for i, executor in enumerate(executor_ls):
|
49
|
+
future = thread_pool.submit(wrapper, executor, timeout, i, _execution_orders, _completion_orders)
|
50
|
+
if b_display_progress:
|
51
|
+
future.add_done_callback(lambda _: p_bar.update())
|
52
|
+
futures.append(future)
|
53
|
+
|
54
|
+
# 收集结果
|
52
55
|
for i, future in enumerate(futures):
|
53
|
-
|
54
|
-
|
56
|
+
try:
|
57
|
+
res, b_success = future.result()
|
58
|
+
except:
|
59
|
+
b_success = False
|
60
|
+
if b_success:
|
61
|
+
res_ls[i] = res
|
55
62
|
else:
|
56
|
-
res_ls.append(None)
|
57
63
|
failed_idx_ls.append(i)
|
64
|
+
|
58
65
|
if b_display_progress:
|
59
66
|
p_bar.close()
|
60
67
|
|
61
|
-
#
|
62
68
|
if isinstance(_hook_for_debug, (dict,)):
|
63
|
-
_hook_for_debug.update(
|
69
|
+
_hook_for_debug.update({
|
70
|
+
"execution_orders": list(_execution_orders),
|
71
|
+
"completion_orders": list(_completion_orders)
|
72
|
+
})
|
64
73
|
|
65
74
|
return res_ls, failed_idx_ls
|
66
75
|
|
@@ -70,15 +79,27 @@ if __name__ == '__main__':
|
|
70
79
|
|
71
80
|
|
72
81
|
def func_(i):
|
82
|
+
# 模拟部分任务长时间运行,部分任务正常结束
|
73
83
|
if i in [2, 3, 7]:
|
74
|
-
time.sleep(
|
84
|
+
time.sleep(100)
|
85
|
+
elif i in [4, 5, 6]:
|
86
|
+
time.sleep(0.01)
|
75
87
|
else:
|
76
|
-
time.sleep(
|
77
|
-
print(i)
|
88
|
+
time.sleep(0.05)
|
89
|
+
print(f"任务 {i} 执行完成")
|
78
90
|
return i * 2
|
79
91
|
|
80
92
|
|
81
93
|
hook_for_debug = dict()
|
82
|
-
|
83
|
-
|
84
|
-
|
94
|
+
a = time.time()
|
95
|
+
results, failed = multi_thread_execute(
|
96
|
+
executors=[Executor(func=func_, args=(i,)) for i in range(10)],
|
97
|
+
worker_nums=5,
|
98
|
+
timeout=0.2,
|
99
|
+
_hook_for_debug=hook_for_debug
|
100
|
+
)
|
101
|
+
gap = time.time() - a
|
102
|
+
print("执行结果:", results)
|
103
|
+
print("超时失败的任务索引:", failed)
|
104
|
+
print("调试信息:", hook_for_debug)
|
105
|
+
print("总耗时:", gap)
|
@@ -0,0 +1,15 @@
|
|
1
|
+
from .wrapper_with_timeout_1 import wrapper_with_timeout_1
|
2
|
+
from .wrapper_with_timeout_2 import wrapper_with_timeout_2
|
3
|
+
|
4
|
+
import signal
|
5
|
+
import multiprocessing
|
6
|
+
|
7
|
+
if callable(getattr(signal, "setitimer", None)):
|
8
|
+
wrapper_for_mp = wrapper_with_timeout_1 # 效率更高,优先选择
|
9
|
+
else:
|
10
|
+
wrapper_for_mp = wrapper_with_timeout_2
|
11
|
+
|
12
|
+
wrapper_for_mt = wrapper_with_timeout_2
|
13
|
+
|
14
|
+
DEFAULT_PROCESS_NUMS = multiprocessing.cpu_count() + 2
|
15
|
+
DEFAULT_THREAD_NUMS = DEFAULT_PROCESS_NUMS * 2
|
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_1.py
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
import signal
|
2
|
+
|
3
|
+
|
4
|
+
# 定义超时异常
|
5
|
+
class TimeoutException(Exception):
|
6
|
+
pass
|
7
|
+
|
8
|
+
|
9
|
+
# 定时器信号处理函数
|
10
|
+
def __alarm_handler(*args, **kwargs):
|
11
|
+
raise TimeoutException("任务超时")
|
12
|
+
|
13
|
+
|
14
|
+
def wrapper_with_timeout_1(executor, timeout=None, idx=-1, _execution_orders=None, _completion_orders=None):
|
15
|
+
"""
|
16
|
+
限制执行时间,使用 multiprocessing.Process 强制终止超时任务
|
17
|
+
该函数仅适用于多进程以及 unix 操作系统
|
18
|
+
|
19
|
+
参数:
|
20
|
+
executor: <Executor>执行器,需实现 run() 方法
|
21
|
+
idx: <int> 任务索引(用于调试)
|
22
|
+
timeout: <int/float>最大等待时间(单位:秒,支持 float)
|
23
|
+
_execution_orders, _completion_orders: 用于记录调试信息的 Manager.list
|
24
|
+
返回:
|
25
|
+
(result, b_success) 若超时或异常则 b_success 为 False
|
26
|
+
"""
|
27
|
+
if _execution_orders is not None:
|
28
|
+
_execution_orders.append(idx)
|
29
|
+
|
30
|
+
# 定时器
|
31
|
+
if timeout is not None:
|
32
|
+
signal.signal(signal.SIGALRM, __alarm_handler)
|
33
|
+
signal.setitimer(signal.ITIMER_REAL, timeout)
|
34
|
+
|
35
|
+
# 执行
|
36
|
+
res, b_success = None, True
|
37
|
+
try:
|
38
|
+
res = executor.run()
|
39
|
+
if _completion_orders is not None:
|
40
|
+
_completion_orders.append(idx)
|
41
|
+
except TimeoutException:
|
42
|
+
b_success = False
|
43
|
+
finally:
|
44
|
+
signal.alarm(0) # 取消定时器
|
45
|
+
return res, b_success
|
46
|
+
|
47
|
+
|
48
|
+
if __name__ == '__main__':
|
49
|
+
import time
|
50
|
+
|
51
|
+
|
52
|
+
def func_(i):
|
53
|
+
if i in [2, 3, 7]:
|
54
|
+
time.sleep(300)
|
55
|
+
else:
|
56
|
+
time.sleep(0.5)
|
57
|
+
return i * 2
|
58
|
+
|
59
|
+
|
60
|
+
from kevin_toolbox.computer_science.data_structure import Executor
|
61
|
+
|
62
|
+
print(wrapper_with_timeout_1(Executor(func=func_, args=(2,)), timeout=1))
|
63
|
+
print(wrapper_with_timeout_1(Executor(func=func_, args=(1,)), timeout=1))
|
64
|
+
|
65
|
+
execution_orders = []
|
66
|
+
completion_orders = []
|
67
|
+
print(wrapper_with_timeout_1(Executor(func=func_, args=(2,)), timeout=1, _execution_orders=execution_orders,
|
68
|
+
_completion_orders=completion_orders))
|
69
|
+
print(execution_orders, completion_orders)
|
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_2.py
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
from multiprocessing import Process, Queue
|
2
|
+
|
3
|
+
|
4
|
+
def __inner_wrapper(q, executor):
|
5
|
+
try:
|
6
|
+
res = executor.run()
|
7
|
+
q.put((res, True))
|
8
|
+
except:
|
9
|
+
q.put((None, False))
|
10
|
+
|
11
|
+
|
12
|
+
def wrapper_with_timeout_2(executor, timeout=None, idx=-1, _execution_orders=None, _completion_orders=None):
|
13
|
+
"""
|
14
|
+
限制执行时间,使用 multiprocessing.Process 强制终止超时任务
|
15
|
+
该函数适用于多线程、多进程以及所有操作系统,但是效率相较于 wrapper_with_timeout_1 较差
|
16
|
+
|
17
|
+
参数:
|
18
|
+
executor: <Executor>执行器,需实现 run() 方法
|
19
|
+
idx: <int> 任务索引(用于调试)
|
20
|
+
timeout: <int/float>最大等待时间(单位:秒,支持 float)
|
21
|
+
_execution_orders, _completion_orders: 用于记录调试信息的 Manager.list
|
22
|
+
返回:
|
23
|
+
(result, b_success) 若超时或异常则 b_success 为 False
|
24
|
+
"""
|
25
|
+
if _execution_orders is not None:
|
26
|
+
_execution_orders.append(idx)
|
27
|
+
|
28
|
+
res, b_success = None, False
|
29
|
+
if timeout is not None:
|
30
|
+
q = Queue()
|
31
|
+
p = Process(target=__inner_wrapper, args=(q, executor))
|
32
|
+
p.start()
|
33
|
+
p.join(timeout) # 最多等待 timeout 秒
|
34
|
+
|
35
|
+
if q.qsize():
|
36
|
+
try:
|
37
|
+
res, b_success = q.get_nowait()
|
38
|
+
except:
|
39
|
+
pass
|
40
|
+
if p.is_alive():
|
41
|
+
p.terminate()
|
42
|
+
p.join()
|
43
|
+
else:
|
44
|
+
try:
|
45
|
+
res, b_success = executor.run(), True
|
46
|
+
except:
|
47
|
+
pass
|
48
|
+
|
49
|
+
if b_success:
|
50
|
+
if _completion_orders is not None:
|
51
|
+
_completion_orders.append(idx)
|
52
|
+
return res, b_success
|
53
|
+
|
54
|
+
|
55
|
+
if __name__ == '__main__':
|
56
|
+
import time
|
57
|
+
|
58
|
+
|
59
|
+
def func_(i):
|
60
|
+
if i in [2, 3, 7]:
|
61
|
+
time.sleep(300)
|
62
|
+
else:
|
63
|
+
time.sleep(0.5)
|
64
|
+
return i * 2
|
65
|
+
|
66
|
+
|
67
|
+
from kevin_toolbox.computer_science.data_structure import Executor
|
68
|
+
|
69
|
+
print(wrapper_with_timeout_2(Executor(func=func_, args=(2,)), timeout=1))
|
70
|
+
print(wrapper_with_timeout_2(Executor(func=func_, args=(1,)), timeout=1))
|
71
|
+
|
72
|
+
execution_orders = []
|
73
|
+
completion_orders = []
|
74
|
+
print(wrapper_with_timeout_2(Executor(func=func_, args=(2,)), timeout=1, _execution_orders=execution_orders,
|
75
|
+
_completion_orders=completion_orders))
|
76
|
+
print(execution_orders, completion_orders)
|
@@ -83,7 +83,7 @@ class Executor:
|
|
83
83
|
# 校验参数
|
84
84
|
# func
|
85
85
|
assert paras["func"] is None or callable(paras["func"]), \
|
86
|
-
f
|
86
|
+
f'func should be callable, but get a {type(paras["func"])}'
|
87
87
|
# args
|
88
88
|
assert isinstance(paras["args"], (list, tuple,)) and isinstance(paras["f_args"], (list, tuple,))
|
89
89
|
for i, f in enumerate(paras["f_args"]):
|
@@ -93,7 +93,7 @@ class Executor:
|
|
93
93
|
assert isinstance(paras["kwargs"], (dict,)) and isinstance(paras["f_kwargs"], (dict,))
|
94
94
|
for k, v in paras["f_kwargs"].items():
|
95
95
|
assert callable(v) and isinstance(k, (str,)), \
|
96
|
-
f"item {
|
96
|
+
f"item {k} in f_kwargs should be (str, callable) pairs, but get a ({type(k)}, {type(v)})"
|
97
97
|
|
98
98
|
# update paras
|
99
99
|
self.paras = paras
|
@@ -4,8 +4,13 @@ import copy
|
|
4
4
|
from kevin_toolbox.data_flow.file.json_.converter import integrate, escape_tuple_and_set, escape_non_str_dict_key
|
5
5
|
from kevin_toolbox.nested_dict_list import traverse
|
6
6
|
|
7
|
+
format_s = {
|
8
|
+
"pretty_printed": dict(indent=4, ensure_ascii=False, sort_keys=False),
|
9
|
+
"minified": dict(indent=None, ensure_ascii=False, sort_keys=False, separators=(',', ':'))
|
10
|
+
}
|
7
11
|
|
8
|
-
|
12
|
+
|
13
|
+
def write_json(content, file_path, converters=None, b_use_suggested_converter=False, output_format="pretty_printed"):
|
9
14
|
"""
|
10
15
|
写入 json file
|
11
16
|
|
@@ -13,7 +18,6 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
|
|
13
18
|
content: 待写入内容
|
14
19
|
file_path: <path or None> 写入路径
|
15
20
|
当设置为 None 时,将直接把(经converters处理后的)待写入内容作为结果返回,而不进行实际的写入
|
16
|
-
sort_keys
|
17
21
|
converters: <list of converters> 对写入内容中每个节点的处理方式
|
18
22
|
转换器 converter 应该是一个形如 def(x): ... ; return x 的函数,具体可以参考
|
19
23
|
json_.converter 中已实现的转换器
|
@@ -22,8 +26,27 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
|
|
22
26
|
可以避免因 json 的读取/写入而丢失部分信息。
|
23
27
|
默认为 False。
|
24
28
|
注意:当 converters 非 None,此参数失效,以 converters 中的具体设置为准
|
29
|
+
output_format: <str/dict/tuple> json的输出格式
|
30
|
+
对于 str 目前支持以下取值:
|
31
|
+
- "pretty_printed": 通过添加大量的空格和换行符来格式化输出,使输出更易读
|
32
|
+
- "minified": 删除所有空格和换行符,使输出更紧凑
|
33
|
+
默认为 pretty_printed。
|
34
|
+
对于 dict,将允许使用更加细致的格式设定,比如:
|
35
|
+
{"indent": 2, ensure_ascii=True}
|
36
|
+
如果需要基于已有格式进行微调可以使用以下方式:
|
37
|
+
("pretty_printed", {"indent": 2, ensure_ascii=True})
|
25
38
|
"""
|
39
|
+
global format_s
|
26
40
|
assert isinstance(file_path, (str, type(None)))
|
41
|
+
if isinstance(output_format, (str,)):
|
42
|
+
output_format = format_s[output_format]
|
43
|
+
elif isinstance(output_format, (tuple,)):
|
44
|
+
output_format = format_s[output_format[0]]
|
45
|
+
output_format.update(output_format[1])
|
46
|
+
elif isinstance(output_format, (dict,)):
|
47
|
+
pass
|
48
|
+
else:
|
49
|
+
raise ValueError(f'Unsupported output_format: {output_format}.')
|
27
50
|
|
28
51
|
if converters is None and b_use_suggested_converter:
|
29
52
|
converters = [escape_tuple_and_set, escape_non_str_dict_key]
|
@@ -35,7 +58,7 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
|
|
35
58
|
converter=lambda _, x: converter(x),
|
36
59
|
b_traverse_matched_element=True)[0]
|
37
60
|
|
38
|
-
content = json.dumps(content,
|
61
|
+
content = json.dumps(content, **output_format)
|
39
62
|
|
40
63
|
if file_path is not None:
|
41
64
|
file_path = os.path.abspath(os.path.expanduser(file_path))
|
@@ -44,3 +67,13 @@ def write_json(content, file_path, sort_keys=False, converters=None, b_use_sugge
|
|
44
67
|
f.write(content)
|
45
68
|
else:
|
46
69
|
return content
|
70
|
+
|
71
|
+
|
72
|
+
if __name__ == "__main__":
|
73
|
+
a = {'rect': {'l:eft': [0, 1, 2], 'top': 67, 'right': 286, 'bottom': 332}}
|
74
|
+
res_0 = write_json(a, file_path=None, output_format="pretty_printed")
|
75
|
+
print(res_0)
|
76
|
+
print(len(res_0))
|
77
|
+
res_1 = write_json(a, file_path=None, output_format="minified")
|
78
|
+
print(res_1)
|
79
|
+
print(len(res_1))
|
@@ -35,8 +35,8 @@ class Json_(Backend_Base):
|
|
35
35
|
"""
|
36
36
|
if id(var) != self.w_cache_s["id_"]:
|
37
37
|
try:
|
38
|
-
self.w_cache_s["content"] = json_.write(content=var, file_path=None,
|
39
|
-
|
38
|
+
self.w_cache_s["content"] = json_.write(content=var, file_path=None, b_use_suggested_converter=True,
|
39
|
+
output_format=kwargs.get("output_format", "pretty_printed"))
|
40
40
|
except:
|
41
41
|
self.w_cache_s["content"], self.w_cache_s["id_"] = None, None
|
42
42
|
self.w_cache_s["able"] = False
|
@@ -5,7 +5,7 @@ from kevin_toolbox.patches.for_os.path import replace_illegal_chars
|
|
5
5
|
from kevin_toolbox.patches.for_matplotlib.color import generate_color_list
|
6
6
|
|
7
7
|
|
8
|
-
def plot_lines(data_s, title, x_name, output_dir=None, **kwargs):
|
8
|
+
def plot_lines(data_s, title, x_name, x_ticklabels_name=None, output_dir=None, **kwargs):
|
9
9
|
data_s = copy.copy(data_s)
|
10
10
|
line_nums = len(data_s) - 1
|
11
11
|
paras = {
|
@@ -23,6 +23,10 @@ def plot_lines(data_s, title, x_name, output_dir=None, **kwargs):
|
|
23
23
|
plt.clf()
|
24
24
|
#
|
25
25
|
x_all_ls = data_s.pop(x_name)
|
26
|
+
if x_ticklabels_name is not None:
|
27
|
+
x_ticklabels = data_s.pop(x_ticklabels_name)
|
28
|
+
assert len(x_all_ls) == len(x_ticklabels)
|
29
|
+
plt.xticks(x_all_ls, x_ticklabels)
|
26
30
|
data_s, temp = dict(), data_s
|
27
31
|
for k, v_ls in temp.items():
|
28
32
|
y_ls, x_ls = [], []
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: kevin-toolbox-dev
|
3
|
-
Version: 1.4.
|
3
|
+
Version: 1.4.7
|
4
4
|
Summary: 一个常用的工具代码包集合
|
5
5
|
Home-page: https://github.com/cantbeblank96/kevin_toolbox
|
6
6
|
Download-URL: https://github.com/username/your-package/archive/refs/tags/v1.0.0.tar.gz
|
@@ -51,26 +51,19 @@ pip install kevin-toolbox --no-dependencies
|
|
51
51
|
|
52
52
|
[版本更新记录](./notes/Release_Record.md):
|
53
53
|
|
54
|
-
- v 1.4.
|
54
|
+
- v 1.4.7 (2025-02-19)【new feature】【bug fix】【incompatible change】
|
55
55
|
|
56
56
|
- data_flow.file
|
57
|
-
- modify json_.
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
- 【
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
- 【new feature】add check_validity_and_uninstall(),检查当前机器时间是否超过 expiration_timestamp 指定的有效期,若超过则卸载 package_name 对应的库。
|
68
|
-
- 【new feature】add check_version_and_update(),检查当前库的版本,并尝试更新。
|
69
|
-
- 以上函数均系从同名脚本中抽取出来。
|
57
|
+
- 【new feature】【incompatible change】modify json_.write(),支持使用参数 output_format 设置更复杂的输出格式。同时废弃原来的sort_keys参数。
|
58
|
+
- output_format 支持以下输入:
|
59
|
+
- "pretty_printed": 通过添加大量的空格和换行符来格式化输出,使输出更易读
|
60
|
+
- "minified": 删除所有空格和换行符,使输出更紧凑
|
61
|
+
- `<dict/tuple>`: 更加细致的格式设定,比如 `{"indent": 2, ensure_ascii=True}`,如果需要基于已有格式进行微调可以使用以下方式:`("pretty_printed", {"indent": 2, ensure_ascii=True})`
|
62
|
+
- computer_science.algorithm.parallel_and_concurrent
|
63
|
+
- 【bug fix】【incompatible change】fix bug in multi_thread_execute(),修正了参数timeout无法对每个任务起效的bug,将参数thread_nums更名为worker_nums。
|
64
|
+
- 【new feature】add multi_process_execute(),用于多进程执行任务。同样支持timeout设定和进度条显示。
|
65
|
+
- patches.for_matplotlib.common_charts
|
66
|
+
- modify plot_lines(),添加了 x_ticklabels_name 参数用于自定义x轴的坐标值
|
70
67
|
- 以上修改,均已添加了对应的测试用例。
|
71
|
-
- developing
|
72
|
-
- 【new feature】add photo_album_organization,该模块包含一系列整理相册相关的脚本。
|
73
|
-
- 0_group_by_timestamp.py :按照时间戳分组
|
74
|
-
- 1_merge_folders.py :将文件数量较少的目录合并
|
75
68
|
|
76
69
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
kevin_toolbox/__init__.py,sha256
|
1
|
+
kevin_toolbox/__init__.py,sha256=-Tz-COP35PmSQrmLdDYK9Eq_i7wCXKv3hJp2_MzZHRw,410
|
2
2
|
kevin_toolbox/computer_science/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
3
|
kevin_toolbox/computer_science/algorithm/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
4
4
|
kevin_toolbox/computer_science/algorithm/cache_manager/__init__.py,sha256=p2hddkZ1HfYF9-m2Hx-o9IotwQHd4QwDCePy2ADpTDA,41
|
@@ -35,9 +35,13 @@ kevin_toolbox/computer_science/algorithm/for_seq/flatten_list.py,sha256=XnDq-_nQ
|
|
35
35
|
kevin_toolbox/computer_science/algorithm/for_seq/get_subsets.py,sha256=uVc2pf9cBjX9sWd9VJ3w6nbsRPaeFT1fXRFfGl1zk6Q,309
|
36
36
|
kevin_toolbox/computer_science/algorithm/locks/__init__.py,sha256=ZjZjqGsQB-z9MoYfOPYlI0H7nfZI1hNgxtmJaDdrynI,35
|
37
37
|
kevin_toolbox/computer_science/algorithm/locks/mutex_lock.py,sha256=81cCw3oTXCZxFNhUzFTB5cPKmvbcLM_Gg-5DlT2nuRQ,2492
|
38
|
-
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/__init__.py,sha256=
|
38
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/__init__.py,sha256=bIyMaB3v6JFjkrY7-sg_-yAnFH8I53E2qxIhGprcLzo,112
|
39
39
|
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/async_executor.py,sha256=yWHpD_1XrC8hG3PWXUZEDj5rnfM0-d-NptRE856tcmY,896
|
40
|
-
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/
|
40
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/multi_process_execute.py,sha256=00QSeECaglANNyfeAeqWrP83WAhoixpg_zyy6k1ioyY,4176
|
41
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/multi_thread_execute.py,sha256=psvezDjaPAwK05GYCJno7GhiW-Zt_Swy5nuyS_xOgUU,3901
|
42
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/__init__.py,sha256=zdYxhuhJk3m7b1MMaXh2frCpOh6vbK3N6u3t0UEC3_w,475
|
43
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_1.py,sha256=8vO26zBOq12BXJc0wudyGeWObIAfDkxWU-pg5VOOZMA,2165
|
44
|
+
kevin_toolbox/computer_science/algorithm/parallel_and_concurrent/utils/wrapper_with_timeout_2.py,sha256=LcDBFjPY77tyZGjk2g3iCEj7bNeQS4usaqgoUQAZxSY,2412
|
41
45
|
kevin_toolbox/computer_science/algorithm/pareto_front/__init__.py,sha256=F1uD0ZT2Ukb708_Eay96SEhaDfCq9YEheJRust33P6w,111
|
42
46
|
kevin_toolbox/computer_science/algorithm/pareto_front/get_pareto_points_idx.py,sha256=WR-_9BruqAWH0QECa40b1Iz1_k6uesBrUYrn2s9ALBM,3008
|
43
47
|
kevin_toolbox/computer_science/algorithm/pareto_front/optimum_picker.py,sha256=wnYN2s9r2g1z5wF0FvFLawRYITUJbMXbBs4TPsdvhlE,9923
|
@@ -58,7 +62,7 @@ kevin_toolbox/computer_science/algorithm/statistician/init_var/init_by_data_form
|
|
58
62
|
kevin_toolbox/computer_science/algorithm/statistician/init_var/init_by_like.py,sha256=8QfvltiNDqZUYiNW6Ebt0UIuYvyqhSpsCYn99T2q70c,572
|
59
63
|
kevin_toolbox/computer_science/algorithm/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
60
64
|
kevin_toolbox/computer_science/data_structure/__init__.py,sha256=_esL73v9Gi40xb5N7UGxslIk8yHM6idQlXbzELR7XhA,31
|
61
|
-
kevin_toolbox/computer_science/data_structure/executor.py,sha256=
|
65
|
+
kevin_toolbox/computer_science/data_structure/executor.py,sha256=ogcozUbd2QW6AruS50CkMdXWc9rE_UJd-KcHO_RMXQg,6211
|
62
66
|
kevin_toolbox/dangerous/__init__.py,sha256=7TqcyVO0IUUZnFw6vFybvdY7UCg-Bv1Moh95IIVMT2c,93
|
63
67
|
kevin_toolbox/dangerous/dump_into_pickle_with_executor_attached.py,sha256=oaPlXyMsG57XbahVqrZENvGSZy2EefoRCmfpm_6tNEQ,2898
|
64
68
|
kevin_toolbox/data_flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -76,7 +80,7 @@ kevin_toolbox/data_flow/file/excel/__init__.py,sha256=5m_rmklI6n6yk4rSAEW39pxzYT
|
|
76
80
|
kevin_toolbox/data_flow/file/excel/write_excel_with_matrix.py,sha256=zrY_l0xCBpjqxm_9MoGpEXaZ4V_UwRMRgShessJ1sxA,5121
|
77
81
|
kevin_toolbox/data_flow/file/json_/__init__.py,sha256=VAt8COS2tO3PJRuhSc43i35fEOlArFM_YahdTmEBaHE,85
|
78
82
|
kevin_toolbox/data_flow/file/json_/read_json.py,sha256=RyCeNONMmvVOeX_F3kSSmED_nx4opipLe8OHJzXKZvQ,2151
|
79
|
-
kevin_toolbox/data_flow/file/json_/write_json.py,sha256=
|
83
|
+
kevin_toolbox/data_flow/file/json_/write_json.py,sha256=ZiNrVUK3ofFmtA-5mU_W1N7bDdReC_yi6CtG2I47B4A,4198
|
80
84
|
kevin_toolbox/data_flow/file/json_/converter/__init__.py,sha256=oQMgAgzELLq_f4LIIfz5E6l_E7g4lFsXqfmnJ3tPZTY,401
|
81
85
|
kevin_toolbox/data_flow/file/json_/converter/convert_dict_key_to_number.py,sha256=SuSZj_HCqKZutHAJ5AttABnGBRZplPGQhMxJBt2Wlgc,559
|
82
86
|
kevin_toolbox/data_flow/file/json_/converter/convert_ndarray_to_list.py,sha256=GALpC1MFJ4aMzs0FZIfJScYznfCP-gmhPeM8sWXGSWg,391
|
@@ -256,7 +260,7 @@ kevin_toolbox/nested_dict_list/serializer/saved_node_name_builder.py,sha256=qsD-
|
|
256
260
|
kevin_toolbox/nested_dict_list/serializer/variable.py,sha256=ZywG6obipRBCGY1cY42gdvsuWk8GLZXr6eCYcW7ZJ9c,392
|
257
261
|
kevin_toolbox/nested_dict_list/serializer/write.py,sha256=ZUYJlBXQbCkMW2UN3d29obskGGbTA-gm3dmuLLltxLI,24101
|
258
262
|
kevin_toolbox/nested_dict_list/serializer/backends/__init__.py,sha256=8g7y-L3cmctxao616dVkGiot00FJzKNmNl_69V2bSmE,39
|
259
|
-
kevin_toolbox/nested_dict_list/serializer/backends/_json_.py,sha256=
|
263
|
+
kevin_toolbox/nested_dict_list/serializer/backends/_json_.py,sha256=yu3604KvzU8dKyECBQ3v127dsEr-VaRz0Mm4dBJgNfc,2189
|
260
264
|
kevin_toolbox/nested_dict_list/serializer/backends/_ndl.py,sha256=3YkAq_Bqzehnw0kGxqxwtF6uUz0EV37tLI-1ROHjixY,1794
|
261
265
|
kevin_toolbox/nested_dict_list/serializer/backends/_numpy_bin.py,sha256=xiPFmPUTjy0X0R1E0N8mrByENhNb69QalHnbYQXFvTo,1470
|
262
266
|
kevin_toolbox/nested_dict_list/serializer/backends/_numpy_npy.py,sha256=CF6R7ie68zA0TqAXBdxUgHKVDYtEPHfVXR9rMFBbsdw,1384
|
@@ -289,7 +293,7 @@ kevin_toolbox/patches/for_matplotlib/common_charts/__init__.py,sha256=etey2r0LO4
|
|
289
293
|
kevin_toolbox/patches/for_matplotlib/common_charts/plot_bars.py,sha256=crS1h79Dz6gGOnqhjuuN2o5pl8CekhCenx9lRz5KPiI,1887
|
290
294
|
kevin_toolbox/patches/for_matplotlib/common_charts/plot_confusion_matrix.py,sha256=KtmUAlKs3_ALFRKAEi0OAXj6SyG5L7LMmoSgOxKvvVs,3213
|
291
295
|
kevin_toolbox/patches/for_matplotlib/common_charts/plot_distribution.py,sha256=stuyaULWM_vVW3r9WrpzGqA8rohQrdNKT3Agsbobqck,2396
|
292
|
-
kevin_toolbox/patches/for_matplotlib/common_charts/plot_lines.py,sha256=
|
296
|
+
kevin_toolbox/patches/for_matplotlib/common_charts/plot_lines.py,sha256=R7mbfmXIOj52BrGuQjf9uDXJ2etwCRCN7ZWX_5cQtBs,2242
|
293
297
|
kevin_toolbox/patches/for_matplotlib/common_charts/plot_scatters.py,sha256=whO36bmixjwtsjCS6Ah6zEGJAlJyGcD-wmV3dA6u7mk,1658
|
294
298
|
kevin_toolbox/patches/for_matplotlib/common_charts/plot_scatters_matrix.py,sha256=bf2EfGlPW9dtDfRse1gk8RVxvC8CJ0NeMdrpSw43wFg,1989
|
295
299
|
kevin_toolbox/patches/for_numpy/__init__.py,sha256=SNjZGxTRBn-uzkyZi6Jcz-9juhhZKT8TI70qH-fhGGc,21
|
@@ -354,7 +358,7 @@ kevin_toolbox/patches/for_torch/nn/__init__.py,sha256=aJs3RMqRzQmd8KKDmQW9FxwCqS
|
|
354
358
|
kevin_toolbox/patches/for_torch/nn/lambda_layer.py,sha256=KUuLiX_Dr4bvRmpAaCW5QTDWDcnMPRnw0jg4NNXTFhM,223
|
355
359
|
kevin_toolbox/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
356
360
|
kevin_toolbox/utils/variable.py,sha256=PxUmp9w4CKKcKHjgdVNF_Iaw5gwPPOd4aY_Oe5F9U1M,133
|
357
|
-
kevin_toolbox_dev-1.4.
|
358
|
-
kevin_toolbox_dev-1.4.
|
359
|
-
kevin_toolbox_dev-1.4.
|
360
|
-
kevin_toolbox_dev-1.4.
|
361
|
+
kevin_toolbox_dev-1.4.7.dist-info/METADATA,sha256=WBubN5isDdTd6IK1yTNeqFgU7ZfnrE4LbjU57x3Q0x4,2586
|
362
|
+
kevin_toolbox_dev-1.4.7.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
363
|
+
kevin_toolbox_dev-1.4.7.dist-info/top_level.txt,sha256=S5TeRGF-PwlhsaUEPTI-f2vWrpLmh3axpyI6v-Fi75o,14
|
364
|
+
kevin_toolbox_dev-1.4.7.dist-info/RECORD,,
|
File without changes
|
File without changes
|