pyxllib 0.3.197__py3-none-any.whl → 3.201.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pyxllib/__init__.py +14 -21
- pyxllib/algo/__init__.py +8 -8
- pyxllib/algo/disjoint.py +54 -54
- pyxllib/algo/geo.py +537 -541
- pyxllib/algo/intervals.py +964 -964
- pyxllib/algo/matcher.py +389 -389
- pyxllib/algo/newbie.py +166 -166
- pyxllib/algo/pupil.py +629 -629
- pyxllib/algo/shapelylib.py +67 -67
- pyxllib/algo/specialist.py +241 -241
- pyxllib/algo/stat.py +494 -494
- pyxllib/algo/treelib.py +145 -149
- pyxllib/algo/unitlib.py +62 -66
- pyxllib/autogui/__init__.py +5 -5
- pyxllib/autogui/activewin.py +246 -246
- pyxllib/autogui/all.py +9 -9
- pyxllib/autogui/autogui.py +846 -852
- pyxllib/autogui/uiautolib.py +362 -362
- pyxllib/autogui/virtualkey.py +102 -102
- pyxllib/autogui/wechat.py +827 -827
- pyxllib/autogui/wechat_msg.py +421 -421
- pyxllib/autogui/wxautolib.py +84 -84
- pyxllib/cv/__init__.py +5 -5
- pyxllib/cv/expert.py +267 -267
- pyxllib/cv/imfile.py +159 -159
- pyxllib/cv/imhash.py +39 -39
- pyxllib/cv/pupil.py +9 -9
- pyxllib/cv/rgbfmt.py +1525 -1525
- pyxllib/cv/slidercaptcha.py +137 -137
- pyxllib/cv/trackbartools.py +251 -251
- pyxllib/cv/xlcvlib.py +1040 -1040
- pyxllib/cv/xlpillib.py +423 -423
- pyxllib/data/echarts.py +236 -240
- pyxllib/data/jsonlib.py +85 -89
- pyxllib/data/oss.py +72 -72
- pyxllib/data/pglib.py +1111 -1127
- pyxllib/data/sqlite.py +568 -568
- pyxllib/data/sqllib.py +297 -297
- pyxllib/ext/JLineViewer.py +505 -505
- pyxllib/ext/__init__.py +6 -6
- pyxllib/ext/demolib.py +251 -246
- pyxllib/ext/drissionlib.py +277 -277
- pyxllib/ext/kq5034lib.py +12 -12
- pyxllib/ext/qt.py +449 -449
- pyxllib/ext/robustprocfile.py +493 -497
- pyxllib/ext/seleniumlib.py +76 -76
- pyxllib/ext/tk.py +173 -173
- pyxllib/ext/unixlib.py +821 -827
- pyxllib/ext/utools.py +345 -351
- pyxllib/ext/webhook.py +124 -119
- pyxllib/ext/win32lib.py +40 -40
- pyxllib/ext/wjxlib.py +91 -88
- pyxllib/ext/wpsapi.py +124 -124
- pyxllib/ext/xlwork.py +9 -9
- pyxllib/ext/yuquelib.py +1110 -1105
- pyxllib/file/__init__.py +17 -17
- pyxllib/file/docxlib.py +757 -761
- pyxllib/file/gitlib.py +309 -309
- pyxllib/file/libreoffice.py +165 -165
- pyxllib/file/movielib.py +144 -148
- pyxllib/file/newbie.py +10 -10
- pyxllib/file/onenotelib.py +1469 -1469
- pyxllib/file/packlib/__init__.py +330 -330
- pyxllib/file/packlib/zipfile.py +2441 -2441
- pyxllib/file/pdflib.py +422 -426
- pyxllib/file/pupil.py +185 -185
- pyxllib/file/specialist/__init__.py +681 -685
- pyxllib/file/specialist/dirlib.py +799 -799
- pyxllib/file/specialist/download.py +193 -193
- pyxllib/file/specialist/filelib.py +2825 -2829
- pyxllib/file/xlsxlib.py +3122 -3131
- pyxllib/file/xlsyncfile.py +341 -341
- pyxllib/prog/__init__.py +5 -5
- pyxllib/prog/cachetools.py +58 -64
- pyxllib/prog/deprecatedlib.py +233 -233
- pyxllib/prog/filelock.py +42 -42
- pyxllib/prog/ipyexec.py +253 -253
- pyxllib/prog/multiprogs.py +940 -940
- pyxllib/prog/newbie.py +451 -451
- pyxllib/prog/pupil.py +1208 -1197
- pyxllib/prog/sitepackages.py +33 -33
- pyxllib/prog/specialist/__init__.py +348 -391
- pyxllib/prog/specialist/bc.py +203 -203
- pyxllib/prog/specialist/browser.py +497 -497
- pyxllib/prog/specialist/common.py +347 -347
- pyxllib/prog/specialist/datetime.py +198 -198
- pyxllib/prog/specialist/tictoc.py +240 -240
- pyxllib/prog/specialist/xllog.py +180 -180
- pyxllib/prog/xlosenv.py +110 -108
- pyxllib/stdlib/__init__.py +17 -17
- pyxllib/stdlib/tablepyxl/__init__.py +10 -10
- pyxllib/stdlib/tablepyxl/style.py +303 -303
- pyxllib/stdlib/tablepyxl/tablepyxl.py +130 -130
- pyxllib/text/__init__.py +8 -8
- pyxllib/text/ahocorasick.py +36 -39
- pyxllib/text/airscript.js +754 -744
- pyxllib/text/charclasslib.py +121 -121
- pyxllib/text/jiebalib.py +267 -267
- pyxllib/text/jinjalib.py +27 -32
- pyxllib/text/jsa_ai_prompt.md +271 -271
- pyxllib/text/jscode.py +922 -922
- pyxllib/text/latex/__init__.py +158 -158
- pyxllib/text/levenshtein.py +303 -303
- pyxllib/text/nestenv.py +1215 -1215
- pyxllib/text/newbie.py +300 -300
- pyxllib/text/pupil/__init__.py +8 -8
- pyxllib/text/pupil/common.py +1121 -1121
- pyxllib/text/pupil/xlalign.py +326 -326
- pyxllib/text/pycode.py +47 -47
- pyxllib/text/specialist/__init__.py +8 -8
- pyxllib/text/specialist/common.py +112 -112
- pyxllib/text/specialist/ptag.py +186 -186
- pyxllib/text/spellchecker.py +172 -172
- pyxllib/text/templates/echart_base.html +10 -10
- pyxllib/text/templates/highlight_code.html +16 -16
- pyxllib/text/templates/latex_editor.html +102 -102
- pyxllib/text/vbacode.py +17 -17
- pyxllib/text/xmllib.py +741 -747
- pyxllib/xl.py +42 -39
- pyxllib/xlcv.py +17 -17
- pyxllib-3.201.1.dist-info/METADATA +296 -0
- pyxllib-3.201.1.dist-info/RECORD +125 -0
- {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/licenses/LICENSE +190 -190
- pyxllib/ext/old.py +0 -663
- pyxllib-0.3.197.dist-info/METADATA +0 -48
- pyxllib-0.3.197.dist-info/RECORD +0 -126
- {pyxllib-0.3.197.dist-info → pyxllib-3.201.1.dist-info}/WHEEL +0 -0
pyxllib/prog/specialist/xllog.py
CHANGED
@@ -1,180 +1,180 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# @Author : 陈坤泽
|
4
|
-
# @Email : 877362867@qq.com
|
5
|
-
# @Date : 2020/09/18 22:16
|
6
|
-
|
7
|
-
import os
|
8
|
-
import concurrent.futures
|
9
|
-
import math
|
10
|
-
import time
|
11
|
-
import sys
|
12
|
-
|
13
|
-
from pyxllib.prog.pupil import EmptyPoolExecutor, format_exception
|
14
|
-
from pyxllib.text.pupil import shorten
|
15
|
-
|
16
|
-
XLLOG_CONF_FILE = 'xllog.yaml'
|
17
|
-
|
18
|
-
____xllog = """
|
19
|
-
"""
|
20
|
-
|
21
|
-
|
22
|
-
def get_xllog(name='xllog', *, log_file=None):
|
23
|
-
""" 获得pyxllib库的日志类
|
24
|
-
|
25
|
-
:param log_file: 增加输出到一个日志文件,该功能仅在首次初始化时有效
|
26
|
-
注意这个是'w'机制,会删除之前的日志文件
|
27
|
-
# TODO 这样的功能设计问题挺大的,工程逻辑很莫名其妙,有空要把日志功能修缮下
|
28
|
-
# 例如一个通用的初始化类,然后xllog只是一个特定的实例日志类
|
29
|
-
|
30
|
-
TODO 增加输出到钉钉机器人、邮箱的Handler?
|
31
|
-
"""
|
32
|
-
# import logging, coloredlogs
|
33
|
-
import logging
|
34
|
-
|
35
|
-
# 1 判断是否已存在,直接返回
|
36
|
-
if ('pyxllib.' + name) in logging.root.manager.loggerDict:
|
37
|
-
return logging.getLogger('pyxllib.' + name)
|
38
|
-
|
39
|
-
# 2 初次构建
|
40
|
-
if name == 'xllog': # 附带运行时间信息
|
41
|
-
if os.path.isfile(XLLOG_CONF_FILE):
|
42
|
-
# 尝试在默认位置是否有自定义配置文件,读取配置文件来创建
|
43
|
-
import logging.config
|
44
|
-
from pyxllib.file.specialist import File
|
45
|
-
data = File(XLLOG_CONF_FILE).read()
|
46
|
-
if isinstance(data, dict):
|
47
|
-
# 推荐使用yaml的字典结构,格式更简洁清晰
|
48
|
-
logging.config.dictConfig(data)
|
49
|
-
else:
|
50
|
-
# 但是普通的conf配置文件也支持
|
51
|
-
logging.config.fileConfig(XLLOG_CONF_FILE)
|
52
|
-
else:
|
53
|
-
# 3 否则生成一个非常简易版的xllog
|
54
|
-
xllog = logging.getLogger('pyxllib.xllog')
|
55
|
-
fmt = '%(asctime)s %(message)s'
|
56
|
-
if log_file:
|
57
|
-
file_handler = logging.FileHandler(f'{log_file}', 'a')
|
58
|
-
file_handler.setLevel(logging.DEBUG)
|
59
|
-
file_handler.setFormatter(logging.Formatter(fmt))
|
60
|
-
xllog.addHandler(file_handler)
|
61
|
-
# coloredlogs.install(level='DEBUG', logger=xllog, fmt=fmt)
|
62
|
-
elif name == 'location': # 附带代码所处位置信息
|
63
|
-
loclog = logging.getLogger('pyxllib.location')
|
64
|
-
# coloredlogs.install(level='DEBUG', logger=loclog, fmt='%(filename)s/%(lineno)d: %(message)s')
|
65
|
-
return logging.getLogger('pyxllib.' + name)
|
66
|
-
|
67
|
-
|
68
|
-
class Iterate:
|
69
|
-
""" 迭代器类,用来封装一些特定模式的for循环操作
|
70
|
-
|
71
|
-
TODO 双循环,需要内部两两对比的迭代功能
|
72
|
-
|
73
|
-
200920周日18:20,最初设计的时候,是提供run_pair、run_pair2的功能的
|
74
|
-
不过后来想想,这个其实就是排列组合,在itertools里有combinations, permutations可以代替
|
75
|
-
甚至有放回的组合也有combinations_with_replacement,我实在是不需要再这里写这些冗余的功能
|
76
|
-
所以就移除了
|
77
|
-
"""
|
78
|
-
|
79
|
-
def __init__(self, items):
|
80
|
-
# 没有总长度倒也能接受,关键是可能要用start、end切片,所以还是先转成tuple更方便操作
|
81
|
-
self.items = tuple(items)
|
82
|
-
self.n_items = len(self.items)
|
83
|
-
self.format_width = math.ceil(math.log10(self.n_items + 1))
|
84
|
-
self.xllog = get_xllog()
|
85
|
-
|
86
|
-
def _format_pinterval(self, pinterval=None):
|
87
|
-
if isinstance(pinterval, str) and pinterval.endswith('%'):
|
88
|
-
# 百分比的情况,重算出间隔元素数
|
89
|
-
return int(round(self.n_items * float(pinterval[:-1]) / 100))
|
90
|
-
else: # 其他格式暂不解析,按原格式处理
|
91
|
-
return pinterval
|
92
|
-
|
93
|
-
def _step1_check_number(self, pinterval, func):
|
94
|
-
if pinterval:
|
95
|
-
sys.stdout.flush() # 让逻辑在前的标准输出先print出来,但其实这句也不一定能让print及时输出的~~可能会被日志提前抢输出了
|
96
|
-
self.xllog.info(f"使用 {func.__name__} 处理 {self.n_items} 个数据 {shorten(str(self.items), 30)}")
|
97
|
-
|
98
|
-
def _step2_check_range(self, start, end):
|
99
|
-
if start:
|
100
|
-
self.xllog.info(f"使用start参数,只处理≥{start}的条目")
|
101
|
-
else:
|
102
|
-
start = 0
|
103
|
-
if end:
|
104
|
-
# 这里空格是为了对齐,别删
|
105
|
-
self.xllog.info(f"使用 end 参数,只处理<{end}的条目")
|
106
|
-
else:
|
107
|
-
end = len(self.items)
|
108
|
-
return start, end
|
109
|
-
|
110
|
-
def _step3_executor(self, pinterval, max_workers):
|
111
|
-
if max_workers == 1:
|
112
|
-
# workers=1,实际上并不用多线程,用一个假的多线程类代替,能大大提速
|
113
|
-
executor = EmptyPoolExecutor()
|
114
|
-
# executor = concurrent.futures.ThreadPoolExecutor(max_workers)
|
115
|
-
else:
|
116
|
-
executor = concurrent.futures.ThreadPoolExecutor(max_workers)
|
117
|
-
if pinterval:
|
118
|
-
self.xllog.info(f'多线程执行,当前迭代所用线程数:{executor._max_workers}')
|
119
|
-
return executor
|
120
|
-
|
121
|
-
def _step4_iter(self, i, pinterval, executor):
|
122
|
-
# 队列中没有新任务时,才放入新任务,这样能确保pinterval的输出能反应实时情况,而不是一下全部进入队列,把for循环跑完了
|
123
|
-
while executor._work_queue.qsize(): pass
|
124
|
-
if pinterval and (i or pinterval == 1) and i % pinterval == 0:
|
125
|
-
message = f' {self.items[i]}' if pinterval == 1 else ''
|
126
|
-
self.xllog.info(f'{i:{self.format_width}d}/{self.n_items}={i / self.n_items:6.2%}{message}')
|
127
|
-
|
128
|
-
def _step5_finish(self, pinterval, interrupt, start_time):
|
129
|
-
from humanfriendly import format_timespan
|
130
|
-
end_time = time.time()
|
131
|
-
span = end_time - start_time
|
132
|
-
if span:
|
133
|
-
speed = self.n_items / span
|
134
|
-
msg = f'总用时:{format_timespan(span)},速度:{speed:.2f}it/s'
|
135
|
-
else:
|
136
|
-
msg = f'总用时:{format_timespan(span)}'
|
137
|
-
if not interrupt and pinterval:
|
138
|
-
self.xllog.info(f'{self.n_items / self.n_items:6.2%} 完成迭代,{msg}')
|
139
|
-
sys.stderr.flush()
|
140
|
-
|
141
|
-
def run(self, func, start=0, end=None, pinterval=None, max_workers=1, interrupt=True):
|
142
|
-
"""
|
143
|
-
:param func: 对每个item执行的功能
|
144
|
-
:param start: 跳过<start的数据,只处理>=start编号以上
|
145
|
-
:param end: 只处理 < end 的数据
|
146
|
-
:param pinterval: 每隔多少条目输出进度日志,默认不输出进度日志(但是错误日志依然会输出)
|
147
|
-
支持按百分比进度显示,例如每20%,pinterval='20%',不过一些底层实现机制原因,会有些许误差
|
148
|
-
TODO 支持按指定时间间隔显示? 例如每15秒,pinterval='15s' 感觉这种功能太花哨了,没必要搞
|
149
|
-
:param max_workers: 默认线程数,默认1,即串行
|
150
|
-
:type max_workers: int, None
|
151
|
-
:param interrupt: 出现错误时是否中断,默认True会终止程序,否则只会输出错误日志
|
152
|
-
:return:
|
153
|
-
"""
|
154
|
-
|
155
|
-
# 1 统一的参数处理部分
|
156
|
-
pinterval = self._format_pinterval(pinterval)
|
157
|
-
self._step1_check_number(pinterval, func)
|
158
|
-
start, end = self._step2_check_range(start, end)
|
159
|
-
error = False
|
160
|
-
executor = self._step3_executor(pinterval, max_workers)
|
161
|
-
|
162
|
-
# 2 封装的子处理部分
|
163
|
-
def wrap_func(func, i):
|
164
|
-
nonlocal error
|
165
|
-
item = self.items[i]
|
166
|
-
try:
|
167
|
-
func(item)
|
168
|
-
except Exception as e:
|
169
|
-
error = e
|
170
|
-
self.xllog.error(f'💔idx={i}运行出错:{item}\n{format_exception(e)}')
|
171
|
-
|
172
|
-
# 3 执行迭代
|
173
|
-
start_time = time.time()
|
174
|
-
for i in range(start, end):
|
175
|
-
self._step4_iter(i, pinterval, executor)
|
176
|
-
executor.submit(wrap_func, func, i)
|
177
|
-
if interrupt and error:
|
178
|
-
raise error
|
179
|
-
executor.shutdown() # 必须等executor结束,error才是准确的
|
180
|
-
self._step5_finish(pinterval, interrupt and error, start_time)
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Author : 陈坤泽
|
4
|
+
# @Email : 877362867@qq.com
|
5
|
+
# @Date : 2020/09/18 22:16
|
6
|
+
|
7
|
+
import os
|
8
|
+
import concurrent.futures
|
9
|
+
import math
|
10
|
+
import time
|
11
|
+
import sys
|
12
|
+
|
13
|
+
from pyxllib.prog.pupil import EmptyPoolExecutor, format_exception
|
14
|
+
from pyxllib.text.pupil import shorten
|
15
|
+
|
16
|
+
XLLOG_CONF_FILE = 'xllog.yaml'
|
17
|
+
|
18
|
+
____xllog = """
|
19
|
+
"""
|
20
|
+
|
21
|
+
|
22
|
+
def get_xllog(name='xllog', *, log_file=None):
|
23
|
+
""" 获得pyxllib库的日志类
|
24
|
+
|
25
|
+
:param log_file: 增加输出到一个日志文件,该功能仅在首次初始化时有效
|
26
|
+
注意这个是'w'机制,会删除之前的日志文件
|
27
|
+
# TODO 这样的功能设计问题挺大的,工程逻辑很莫名其妙,有空要把日志功能修缮下
|
28
|
+
# 例如一个通用的初始化类,然后xllog只是一个特定的实例日志类
|
29
|
+
|
30
|
+
TODO 增加输出到钉钉机器人、邮箱的Handler?
|
31
|
+
"""
|
32
|
+
# import logging, coloredlogs
|
33
|
+
import logging
|
34
|
+
|
35
|
+
# 1 判断是否已存在,直接返回
|
36
|
+
if ('pyxllib.' + name) in logging.root.manager.loggerDict:
|
37
|
+
return logging.getLogger('pyxllib.' + name)
|
38
|
+
|
39
|
+
# 2 初次构建
|
40
|
+
if name == 'xllog': # 附带运行时间信息
|
41
|
+
if os.path.isfile(XLLOG_CONF_FILE):
|
42
|
+
# 尝试在默认位置是否有自定义配置文件,读取配置文件来创建
|
43
|
+
import logging.config
|
44
|
+
from pyxllib.file.specialist import File
|
45
|
+
data = File(XLLOG_CONF_FILE).read()
|
46
|
+
if isinstance(data, dict):
|
47
|
+
# 推荐使用yaml的字典结构,格式更简洁清晰
|
48
|
+
logging.config.dictConfig(data)
|
49
|
+
else:
|
50
|
+
# 但是普通的conf配置文件也支持
|
51
|
+
logging.config.fileConfig(XLLOG_CONF_FILE)
|
52
|
+
else:
|
53
|
+
# 3 否则生成一个非常简易版的xllog
|
54
|
+
xllog = logging.getLogger('pyxllib.xllog')
|
55
|
+
fmt = '%(asctime)s %(message)s'
|
56
|
+
if log_file:
|
57
|
+
file_handler = logging.FileHandler(f'{log_file}', 'a')
|
58
|
+
file_handler.setLevel(logging.DEBUG)
|
59
|
+
file_handler.setFormatter(logging.Formatter(fmt))
|
60
|
+
xllog.addHandler(file_handler)
|
61
|
+
# coloredlogs.install(level='DEBUG', logger=xllog, fmt=fmt)
|
62
|
+
elif name == 'location': # 附带代码所处位置信息
|
63
|
+
loclog = logging.getLogger('pyxllib.location')
|
64
|
+
# coloredlogs.install(level='DEBUG', logger=loclog, fmt='%(filename)s/%(lineno)d: %(message)s')
|
65
|
+
return logging.getLogger('pyxllib.' + name)
|
66
|
+
|
67
|
+
|
68
|
+
class Iterate:
|
69
|
+
""" 迭代器类,用来封装一些特定模式的for循环操作
|
70
|
+
|
71
|
+
TODO 双循环,需要内部两两对比的迭代功能
|
72
|
+
|
73
|
+
200920周日18:20,最初设计的时候,是提供run_pair、run_pair2的功能的
|
74
|
+
不过后来想想,这个其实就是排列组合,在itertools里有combinations, permutations可以代替
|
75
|
+
甚至有放回的组合也有combinations_with_replacement,我实在是不需要再这里写这些冗余的功能
|
76
|
+
所以就移除了
|
77
|
+
"""
|
78
|
+
|
79
|
+
def __init__(self, items):
|
80
|
+
# 没有总长度倒也能接受,关键是可能要用start、end切片,所以还是先转成tuple更方便操作
|
81
|
+
self.items = tuple(items)
|
82
|
+
self.n_items = len(self.items)
|
83
|
+
self.format_width = math.ceil(math.log10(self.n_items + 1))
|
84
|
+
self.xllog = get_xllog()
|
85
|
+
|
86
|
+
def _format_pinterval(self, pinterval=None):
|
87
|
+
if isinstance(pinterval, str) and pinterval.endswith('%'):
|
88
|
+
# 百分比的情况,重算出间隔元素数
|
89
|
+
return int(round(self.n_items * float(pinterval[:-1]) / 100))
|
90
|
+
else: # 其他格式暂不解析,按原格式处理
|
91
|
+
return pinterval
|
92
|
+
|
93
|
+
def _step1_check_number(self, pinterval, func):
|
94
|
+
if pinterval:
|
95
|
+
sys.stdout.flush() # 让逻辑在前的标准输出先print出来,但其实这句也不一定能让print及时输出的~~可能会被日志提前抢输出了
|
96
|
+
self.xllog.info(f"使用 {func.__name__} 处理 {self.n_items} 个数据 {shorten(str(self.items), 30)}")
|
97
|
+
|
98
|
+
def _step2_check_range(self, start, end):
|
99
|
+
if start:
|
100
|
+
self.xllog.info(f"使用start参数,只处理≥{start}的条目")
|
101
|
+
else:
|
102
|
+
start = 0
|
103
|
+
if end:
|
104
|
+
# 这里空格是为了对齐,别删
|
105
|
+
self.xllog.info(f"使用 end 参数,只处理<{end}的条目")
|
106
|
+
else:
|
107
|
+
end = len(self.items)
|
108
|
+
return start, end
|
109
|
+
|
110
|
+
def _step3_executor(self, pinterval, max_workers):
|
111
|
+
if max_workers == 1:
|
112
|
+
# workers=1,实际上并不用多线程,用一个假的多线程类代替,能大大提速
|
113
|
+
executor = EmptyPoolExecutor()
|
114
|
+
# executor = concurrent.futures.ThreadPoolExecutor(max_workers)
|
115
|
+
else:
|
116
|
+
executor = concurrent.futures.ThreadPoolExecutor(max_workers)
|
117
|
+
if pinterval:
|
118
|
+
self.xllog.info(f'多线程执行,当前迭代所用线程数:{executor._max_workers}')
|
119
|
+
return executor
|
120
|
+
|
121
|
+
def _step4_iter(self, i, pinterval, executor):
|
122
|
+
# 队列中没有新任务时,才放入新任务,这样能确保pinterval的输出能反应实时情况,而不是一下全部进入队列,把for循环跑完了
|
123
|
+
while executor._work_queue.qsize(): pass
|
124
|
+
if pinterval and (i or pinterval == 1) and i % pinterval == 0:
|
125
|
+
message = f' {self.items[i]}' if pinterval == 1 else ''
|
126
|
+
self.xllog.info(f'{i:{self.format_width}d}/{self.n_items}={i / self.n_items:6.2%}{message}')
|
127
|
+
|
128
|
+
def _step5_finish(self, pinterval, interrupt, start_time):
|
129
|
+
from humanfriendly import format_timespan
|
130
|
+
end_time = time.time()
|
131
|
+
span = end_time - start_time
|
132
|
+
if span:
|
133
|
+
speed = self.n_items / span
|
134
|
+
msg = f'总用时:{format_timespan(span)},速度:{speed:.2f}it/s'
|
135
|
+
else:
|
136
|
+
msg = f'总用时:{format_timespan(span)}'
|
137
|
+
if not interrupt and pinterval:
|
138
|
+
self.xllog.info(f'{self.n_items / self.n_items:6.2%} 完成迭代,{msg}')
|
139
|
+
sys.stderr.flush()
|
140
|
+
|
141
|
+
def run(self, func, start=0, end=None, pinterval=None, max_workers=1, interrupt=True):
|
142
|
+
"""
|
143
|
+
:param func: 对每个item执行的功能
|
144
|
+
:param start: 跳过<start的数据,只处理>=start编号以上
|
145
|
+
:param end: 只处理 < end 的数据
|
146
|
+
:param pinterval: 每隔多少条目输出进度日志,默认不输出进度日志(但是错误日志依然会输出)
|
147
|
+
支持按百分比进度显示,例如每20%,pinterval='20%',不过一些底层实现机制原因,会有些许误差
|
148
|
+
TODO 支持按指定时间间隔显示? 例如每15秒,pinterval='15s' 感觉这种功能太花哨了,没必要搞
|
149
|
+
:param max_workers: 默认线程数,默认1,即串行
|
150
|
+
:type max_workers: int, None
|
151
|
+
:param interrupt: 出现错误时是否中断,默认True会终止程序,否则只会输出错误日志
|
152
|
+
:return:
|
153
|
+
"""
|
154
|
+
|
155
|
+
# 1 统一的参数处理部分
|
156
|
+
pinterval = self._format_pinterval(pinterval)
|
157
|
+
self._step1_check_number(pinterval, func)
|
158
|
+
start, end = self._step2_check_range(start, end)
|
159
|
+
error = False
|
160
|
+
executor = self._step3_executor(pinterval, max_workers)
|
161
|
+
|
162
|
+
# 2 封装的子处理部分
|
163
|
+
def wrap_func(func, i):
|
164
|
+
nonlocal error
|
165
|
+
item = self.items[i]
|
166
|
+
try:
|
167
|
+
func(item)
|
168
|
+
except Exception as e:
|
169
|
+
error = e
|
170
|
+
self.xllog.error(f'💔idx={i}运行出错:{item}\n{format_exception(e)}')
|
171
|
+
|
172
|
+
# 3 执行迭代
|
173
|
+
start_time = time.time()
|
174
|
+
for i in range(start, end):
|
175
|
+
self._step4_iter(i, pinterval, executor)
|
176
|
+
executor.submit(wrap_func, func, i)
|
177
|
+
if interrupt and error:
|
178
|
+
raise error
|
179
|
+
executor.shutdown() # 必须等executor结束,error才是准确的
|
180
|
+
self._step5_finish(pinterval, interrupt and error, start_time)
|
pyxllib/prog/xlosenv.py
CHANGED
@@ -1,108 +1,110 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# @Author : 陈坤泽
|
4
|
-
# @Email : 877362867@qq.com
|
5
|
-
# @Date : 2024/10/30
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
import
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
>> XlOsEnv.
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
:
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
return value
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
:param
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
value =
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Author : 陈坤泽
|
4
|
+
# @Email : 877362867@qq.com
|
5
|
+
# @Date : 2024/10/30
|
6
|
+
|
7
|
+
# from pyxllib.prog.pupil import check_package
|
8
|
+
# check_package('envariable')
|
9
|
+
|
10
|
+
import os
|
11
|
+
import json
|
12
|
+
import base64
|
13
|
+
|
14
|
+
from pyxllib.text.newbie import add_quote
|
15
|
+
|
16
|
+
|
17
|
+
class XlOsEnv:
|
18
|
+
""" pyxllib库自带的一套环境变量数据解析类
|
19
|
+
|
20
|
+
会将json的字符串值,或者普通str,存储到环境变量中
|
21
|
+
|
22
|
+
环境变量也可以用来实现全局变量的信息传递,虽然不太建议这样做
|
23
|
+
|
24
|
+
>> XlOsEnv.persist_set('TP10_ACCOUNT',
|
25
|
+
{'server': '172.16.250.250', 'port': 22, 'user': 'ckz', 'passwd': '123456'},
|
26
|
+
True)
|
27
|
+
>> print(XlOsEnv.get('TP10_ACCOUNT'), True) # 展示存储的账号信息
|
28
|
+
eyJzZXJ2ZXIiOiAiMTcyLjE2LjE3MC4xMzQiLCAicG9ydCI6IDIyLCAidXNlciI6ICJjaGVua3VuemUiLCAicGFzc3dkIjogImNvZGV4bHByIn0=
|
29
|
+
>> XlOsEnv.unset('TP10_ACCOUNT')
|
30
|
+
"""
|
31
|
+
|
32
|
+
@classmethod
|
33
|
+
def get(cls, name, *, decoding=False):
|
34
|
+
""" 获取环境变量值
|
35
|
+
|
36
|
+
:param name: 环境变量名
|
37
|
+
:param decoding: 是否需要先进行base64解码
|
38
|
+
:return:
|
39
|
+
返回json解析后的数据
|
40
|
+
或者普通的字符串值
|
41
|
+
"""
|
42
|
+
value = os.getenv(name, None)
|
43
|
+
if value is None:
|
44
|
+
return value
|
45
|
+
|
46
|
+
if decoding:
|
47
|
+
value = base64.b64decode(value.encode())
|
48
|
+
|
49
|
+
try:
|
50
|
+
return json.loads(value)
|
51
|
+
except json.decoder.JSONDecodeError:
|
52
|
+
if isinstance(value, bytes):
|
53
|
+
return value.decode()
|
54
|
+
else:
|
55
|
+
return value
|
56
|
+
|
57
|
+
@classmethod
|
58
|
+
def set(cls, name, value, encoding=False):
|
59
|
+
""" 临时改变环境变量
|
60
|
+
|
61
|
+
:param name: 环境变量名
|
62
|
+
:param value: 要存储的值
|
63
|
+
:param encoding: 是否将内容转成base64后,再存储环境变量
|
64
|
+
防止一些密码信息,明文写出来太容易泄露
|
65
|
+
不过这个策略也很容易被破解;只防君子,难防小人
|
66
|
+
|
67
|
+
当然,谁看到这有闲情功夫的话,可以考虑做一套更复杂的加密系统
|
68
|
+
并且encoding支持多种不同的解加密策略,这样单看环境变量值就很难破译了
|
69
|
+
:return: str, 最终存储的字符串内容
|
70
|
+
"""
|
71
|
+
# 1 打包
|
72
|
+
if isinstance(value, str):
|
73
|
+
value = add_quote(value)
|
74
|
+
else:
|
75
|
+
value = json.dumps(value)
|
76
|
+
|
77
|
+
# 2 编码
|
78
|
+
if encoding:
|
79
|
+
value = base64.b64encode(value.encode()).decode()
|
80
|
+
|
81
|
+
# 3 存储到环境变量
|
82
|
+
os.environ[name] = value
|
83
|
+
|
84
|
+
return value
|
85
|
+
|
86
|
+
@classmethod
|
87
|
+
def persist_set(cls, name, value, encoding=False, *, cfgfile=None):
|
88
|
+
""" python里默认是改不了系统变量的,需要使用一些特殊手段
|
89
|
+
https://stackoverflow.com/questions/17657686/is-it-possible-to-set-an-environment-variable-from-python-permanently/17657905
|
90
|
+
|
91
|
+
:param cfgfile: 在linux系统时,可以使用该参数
|
92
|
+
默认是把环境变量写入 ~/.bashrc,可以考虑写到
|
93
|
+
TODO 有这个设想,但很不好实现,不是很关键的功能,所以还未开发
|
94
|
+
|
95
|
+
"""
|
96
|
+
# 写入环境变量这里是有点小麻烦的,要考虑unix和windows不同平台,以及怎么持久化存储的问题,这里直接调用一个三方库来解决
|
97
|
+
from envariable import setenv
|
98
|
+
|
99
|
+
value = cls.set(name, value, encoding)
|
100
|
+
if value[0] == value[-1] == '"':
|
101
|
+
value = '\\' + value + '\\'
|
102
|
+
setenv(name, value)
|
103
|
+
|
104
|
+
return value
|
105
|
+
|
106
|
+
@classmethod
|
107
|
+
def unset(cls, name):
|
108
|
+
""" 删除环境变量 """
|
109
|
+
from envariable import unsetenv
|
110
|
+
unsetenv(name)
|
pyxllib/stdlib/__init__.py
CHANGED
@@ -1,17 +1,17 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# @Author : 陈坤泽
|
4
|
-
# @Email : 877362867@qq.com
|
5
|
-
# @Date : 2021/01/13 15:29
|
6
|
-
|
7
|
-
"""
|
8
|
-
对标准库或一些第三方库,进行的功能扩展
|
9
|
-
也有可能对一些bug进行了修改
|
10
|
-
|
11
|
-
有些是小的库,直接把源码搬过来了
|
12
|
-
有些是较大的库,仍然要(会自动在需要使用时 pip install)安装
|
13
|
-
|
14
|
-
改了底层标准库一些功能,修复一些bug,或者提升功能兼容性、强度
|
15
|
-
|
16
|
-
onepy: 做了些中文注解,其他修改了啥我也忘了~~可能是有改源码功能的
|
17
|
-
"""
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Author : 陈坤泽
|
4
|
+
# @Email : 877362867@qq.com
|
5
|
+
# @Date : 2021/01/13 15:29
|
6
|
+
|
7
|
+
"""
|
8
|
+
对标准库或一些第三方库,进行的功能扩展
|
9
|
+
也有可能对一些bug进行了修改
|
10
|
+
|
11
|
+
有些是小的库,直接把源码搬过来了
|
12
|
+
有些是较大的库,仍然要(会自动在需要使用时 pip install)安装
|
13
|
+
|
14
|
+
改了底层标准库一些功能,修复一些bug,或者提升功能兼容性、强度
|
15
|
+
|
16
|
+
onepy: 做了些中文注解,其他修改了啥我也忘了~~可能是有改源码功能的
|
17
|
+
"""
|
@@ -1,10 +1,10 @@
|
|
1
|
-
#!/usr/bin/env python3
|
2
|
-
# -*- coding: utf-8 -*-
|
3
|
-
# @Author : 陈坤泽
|
4
|
-
# @Email : 877362867@qq.com
|
5
|
-
# @Date : 2020/06/02 20:00
|
6
|
-
|
7
|
-
|
8
|
-
"""
|
9
|
-
from https://github.com/martsberger/tablepyxl
|
10
|
-
"""
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# @Author : 陈坤泽
|
4
|
+
# @Email : 877362867@qq.com
|
5
|
+
# @Date : 2020/06/02 20:00
|
6
|
+
|
7
|
+
|
8
|
+
"""
|
9
|
+
from https://github.com/martsberger/tablepyxl
|
10
|
+
"""
|