re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. re_common/baselibrary/__init__.py +4 -4
  2. re_common/baselibrary/baseabs/__init__.py +6 -6
  3. re_common/baselibrary/baseabs/baseabs.py +26 -26
  4. re_common/baselibrary/database/mbuilder.py +132 -132
  5. re_common/baselibrary/database/moudle.py +93 -93
  6. re_common/baselibrary/database/msqlite3.py +194 -194
  7. re_common/baselibrary/database/mysql.py +169 -169
  8. re_common/baselibrary/database/sql_factory.py +26 -26
  9. re_common/baselibrary/mthread/MThreadingRun.py +486 -486
  10. re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
  11. re_common/baselibrary/mthread/__init__.py +2 -2
  12. re_common/baselibrary/mthread/mythreading.py +695 -695
  13. re_common/baselibrary/pakge_other/socks.py +404 -404
  14. re_common/baselibrary/readconfig/config_factory.py +18 -18
  15. re_common/baselibrary/readconfig/ini_config.py +317 -317
  16. re_common/baselibrary/readconfig/toml_config.py +49 -49
  17. re_common/baselibrary/temporary/envdata.py +36 -36
  18. re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
  19. re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
  20. re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
  21. re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
  22. re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
  23. re_common/baselibrary/tools/contrast_db3.py +123 -123
  24. re_common/baselibrary/tools/copy_file.py +39 -39
  25. re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
  26. re_common/baselibrary/tools/foreachgz.py +39 -39
  27. re_common/baselibrary/tools/get_attr.py +10 -10
  28. re_common/baselibrary/tools/image_to_pdf.py +61 -61
  29. re_common/baselibrary/tools/java_code_deal.py +139 -139
  30. re_common/baselibrary/tools/javacode.py +79 -79
  31. re_common/baselibrary/tools/mdb_db3.py +48 -48
  32. re_common/baselibrary/tools/merge_file.py +171 -171
  33. re_common/baselibrary/tools/merge_gz_file.py +165 -165
  34. re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
  35. re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
  36. re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
  37. re_common/baselibrary/tools/mongo_tools.py +50 -50
  38. re_common/baselibrary/tools/move_file.py +170 -170
  39. re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
  40. re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
  41. re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
  42. re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
  43. re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
  44. re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
  45. re_common/baselibrary/tools/myparsel.py +104 -104
  46. re_common/baselibrary/tools/rename_dir_file.py +37 -37
  47. re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
  48. re_common/baselibrary/tools/split_line_to_many.py +25 -25
  49. re_common/baselibrary/tools/stringtodicts.py +33 -33
  50. re_common/baselibrary/tools/workwechant_bot.py +84 -84
  51. re_common/baselibrary/utils/baseaiohttp.py +296 -296
  52. re_common/baselibrary/utils/baseaiomysql.py +87 -87
  53. re_common/baselibrary/utils/baseallstep.py +191 -191
  54. re_common/baselibrary/utils/baseavro.py +19 -19
  55. re_common/baselibrary/utils/baseboto3.py +291 -291
  56. re_common/baselibrary/utils/basecsv.py +32 -32
  57. re_common/baselibrary/utils/basedict.py +133 -133
  58. re_common/baselibrary/utils/basedir.py +241 -241
  59. re_common/baselibrary/utils/baseencode.py +351 -351
  60. re_common/baselibrary/utils/baseencoding.py +28 -28
  61. re_common/baselibrary/utils/baseesdsl.py +86 -86
  62. re_common/baselibrary/utils/baseexcel.py +264 -264
  63. re_common/baselibrary/utils/baseexcept.py +109 -109
  64. re_common/baselibrary/utils/basefile.py +654 -654
  65. re_common/baselibrary/utils/baseftp.py +214 -214
  66. re_common/baselibrary/utils/basegzip.py +60 -60
  67. re_common/baselibrary/utils/basehdfs.py +135 -135
  68. re_common/baselibrary/utils/basehttpx.py +268 -268
  69. re_common/baselibrary/utils/baseip.py +87 -87
  70. re_common/baselibrary/utils/basejson.py +2 -2
  71. re_common/baselibrary/utils/baselist.py +32 -32
  72. re_common/baselibrary/utils/basemotor.py +190 -190
  73. re_common/baselibrary/utils/basemssql.py +98 -98
  74. re_common/baselibrary/utils/baseodbc.py +113 -113
  75. re_common/baselibrary/utils/basepandas.py +302 -302
  76. re_common/baselibrary/utils/basepeewee.py +11 -11
  77. re_common/baselibrary/utils/basepika.py +180 -180
  78. re_common/baselibrary/utils/basepydash.py +143 -143
  79. re_common/baselibrary/utils/basepymongo.py +230 -230
  80. re_common/baselibrary/utils/basequeue.py +22 -22
  81. re_common/baselibrary/utils/baserar.py +57 -57
  82. re_common/baselibrary/utils/baserequest.py +279 -279
  83. re_common/baselibrary/utils/baseset.py +8 -8
  84. re_common/baselibrary/utils/basesmb.py +403 -403
  85. re_common/baselibrary/utils/basestring.py +382 -382
  86. re_common/baselibrary/utils/basetime.py +320 -320
  87. re_common/baselibrary/utils/baseurl.py +121 -121
  88. re_common/baselibrary/utils/basezip.py +57 -57
  89. re_common/baselibrary/utils/core/__init__.py +7 -7
  90. re_common/baselibrary/utils/core/bottomutils.py +18 -18
  91. re_common/baselibrary/utils/core/mdeprecated.py +327 -327
  92. re_common/baselibrary/utils/core/mlamada.py +16 -16
  93. re_common/baselibrary/utils/core/msginfo.py +25 -25
  94. re_common/baselibrary/utils/core/requests_core.py +103 -103
  95. re_common/baselibrary/utils/fateadm.py +429 -429
  96. re_common/baselibrary/utils/importfun.py +123 -123
  97. re_common/baselibrary/utils/mfaker.py +57 -57
  98. re_common/baselibrary/utils/my_abc/__init__.py +3 -3
  99. re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
  100. re_common/baselibrary/utils/mylogger.py +414 -414
  101. re_common/baselibrary/utils/myredisclient.py +861 -861
  102. re_common/baselibrary/utils/pipupgrade.py +21 -21
  103. re_common/baselibrary/utils/ringlist.py +85 -85
  104. re_common/baselibrary/utils/version_compare.py +36 -36
  105. re_common/baselibrary/utils/ydmhttp.py +126 -126
  106. re_common/facade/lazy_import.py +11 -11
  107. re_common/facade/loggerfacade.py +25 -25
  108. re_common/facade/mysqlfacade.py +467 -467
  109. re_common/facade/now.py +31 -31
  110. re_common/facade/sqlite3facade.py +257 -257
  111. re_common/facade/use/mq_use_facade.py +83 -83
  112. re_common/facade/use/proxy_use_facade.py +19 -19
  113. re_common/libtest/base_dict_test.py +19 -19
  114. re_common/libtest/baseavro_test.py +13 -13
  115. re_common/libtest/basefile_test.py +14 -14
  116. re_common/libtest/basemssql_test.py +77 -77
  117. re_common/libtest/baseodbc_test.py +7 -7
  118. re_common/libtest/basepandas_test.py +38 -38
  119. re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
  120. re_common/libtest/get_attr_test/settings.py +54 -54
  121. re_common/libtest/idencode_test.py +53 -53
  122. re_common/libtest/iniconfig_test.py +35 -35
  123. re_common/libtest/ip_test.py +34 -34
  124. re_common/libtest/merge_file_test.py +20 -20
  125. re_common/libtest/mfaker_test.py +8 -8
  126. re_common/libtest/mm3_test.py +31 -31
  127. re_common/libtest/mylogger_test.py +88 -88
  128. re_common/libtest/myparsel_test.py +27 -27
  129. re_common/libtest/mysql_test.py +151 -151
  130. re_common/libtest/pymongo_test.py +21 -21
  131. re_common/libtest/split_test.py +11 -11
  132. re_common/libtest/sqlite3_merge_test.py +5 -5
  133. re_common/libtest/sqlite3_test.py +34 -34
  134. re_common/libtest/tomlconfig_test.py +30 -30
  135. re_common/libtest/use_tools_test/__init__.py +2 -2
  136. re_common/libtest/user/__init__.py +4 -4
  137. re_common/studio/__init__.py +4 -4
  138. re_common/studio/assignment_expressions.py +36 -36
  139. re_common/studio/mydash/test1.py +18 -18
  140. re_common/studio/pydashstudio/first.py +9 -9
  141. re_common/studio/streamlitstudio/first_app.py +65 -65
  142. re_common/studio/streamlitstudio/uber_pickups.py +23 -23
  143. re_common/studio/test.py +18 -18
  144. re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
  145. re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
  146. re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
  147. re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
  148. re_common/v2/baselibrary/decorators/utils.py +59 -59
  149. re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
  150. re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
  151. re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
  152. re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
  153. re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
  154. re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
  155. re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
  156. re_common/v2/baselibrary/tools/concurrency.py +35 -35
  157. re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
  158. re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
  159. re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
  160. re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
  161. re_common/v2/baselibrary/tools/dict_tools.py +44 -44
  162. re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
  163. re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
  164. re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
  165. re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
  166. re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
  167. re_common/v2/baselibrary/tools/list_tools.py +69 -69
  168. re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
  169. re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
  170. re_common/v2/baselibrary/tools/text_matcher.py +326 -326
  171. re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
  172. re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
  173. re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
  174. re_common/v2/baselibrary/utils/author_smi.py +361 -361
  175. re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
  176. re_common/v2/baselibrary/utils/basedict.py +37 -37
  177. re_common/v2/baselibrary/utils/basehdfs.py +163 -163
  178. re_common/v2/baselibrary/utils/basepika.py +180 -180
  179. re_common/v2/baselibrary/utils/basetime.py +77 -77
  180. re_common/v2/baselibrary/utils/db.py +156 -156
  181. re_common/v2/baselibrary/utils/json_cls.py +16 -16
  182. re_common/v2/baselibrary/utils/mq.py +83 -83
  183. re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
  184. re_common/v2/baselibrary/utils/string_bool.py +186 -186
  185. re_common/v2/baselibrary/utils/string_clear.py +246 -246
  186. re_common/v2/baselibrary/utils/string_smi.py +18 -18
  187. re_common/v2/baselibrary/utils/stringutils.py +271 -278
  188. re_common/vip/base_step_process.py +11 -11
  189. re_common/vip/baseencodeid.py +90 -90
  190. re_common/vip/changetaskname.py +28 -28
  191. re_common/vip/core_var.py +24 -24
  192. re_common/vip/mmh3Hash.py +89 -89
  193. re_common/vip/proxy/allproxys.py +127 -127
  194. re_common/vip/proxy/allproxys_thread.py +159 -159
  195. re_common/vip/proxy/cnki_proxy.py +153 -153
  196. re_common/vip/proxy/kuaidaili.py +87 -87
  197. re_common/vip/proxy/proxy_all.py +113 -113
  198. re_common/vip/proxy/update_kuaidaili_0.py +42 -42
  199. re_common/vip/proxy/wanfang_proxy.py +152 -152
  200. re_common/vip/proxy/wp_proxy_all.py +181 -181
  201. re_common/vip/read_rawid_to_txt.py +91 -91
  202. re_common/vip/title/__init__.py +5 -5
  203. re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
  204. re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
  205. re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
  206. re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
  207. re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
  208. re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
  209. re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
  210. re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
  211. re_common/vip/title/transform/__init__.py +10 -10
  212. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/LICENSE +201 -201
  213. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/METADATA +16 -16
  214. re_common-10.0.39.dist-info/RECORD +248 -0
  215. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/WHEEL +1 -1
  216. re_common-10.0.37.dist-info/RECORD +0 -248
  217. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/top_level.txt +0 -0
@@ -1,327 +1,327 @@
1
- import asyncio
2
- import functools
3
- import sys
4
- import time
5
- import traceback
6
- import warnings
7
- from functools import wraps
8
-
9
- # https://www.jianshu.com/p/ee82b941772a
10
- import aiohttp.client_exceptions
11
- import requests
12
- from aiohttp import ClientProxyConnectionError
13
- from google.protobuf.message import DecodeError
14
-
15
- from re_common.baselibrary.utils.core.requests_core import MsgCode
16
-
17
-
18
- def InterVal(start_time, interval):
19
- '''
20
- 执行函数必须间隔多少时间,没有到指定的时间函数就不会被执行
21
- :param start_time: 传入开始时间
22
- :param interval: 秒数
23
- :return: 返回执行结果 否则返回 None
24
- '''
25
-
26
- def dewrapper(func):
27
- """
28
- 它能把原函数的元信息拷贝到装饰器里面的 func 函数中。
29
- 函数的元信息包括docstring、name、参数列表等等。可以尝试去除@functools.wraps(func),
30
- 你会发现test.__name__的输出变成了wrapper。
31
- """
32
-
33
- @wraps(func)
34
- def wrapper(*args, **kwargs):
35
- start = start_time
36
- time_now = int(time.time())
37
- if time_now - start > interval:
38
- result = func(*args, **kwargs)
39
- return result
40
- else:
41
- return None
42
-
43
- return wrapper
44
-
45
- return dewrapper
46
-
47
-
48
- def timethis(func):
49
- """
50
- 函数执行的时间差
51
- :param func:
52
- :return:
53
- """
54
-
55
- @wraps(func)
56
- def wrapper(*args, **kwargs):
57
- start = time.time()
58
- result = func(*args, **kwargs)
59
- end = time.time()
60
- print(func.__name__, end - start)
61
- return result
62
-
63
- return wrapper
64
-
65
-
66
- def func_time(callback=None, is_print=True):
67
- """
68
- 装饰器获取时间
69
- :param func:
70
- :return:
71
- """
72
-
73
- def dewrapper(func):
74
- @functools.wraps(func)
75
- async def wrapper(*args, **kwargs):
76
- start = time.time()
77
- result = await func(*args, **kwargs)
78
- end = time.time()
79
- if is_print:
80
- print(func.__name__, end - start)
81
- if callback is not None:
82
- await callback(start, end)
83
- return result
84
-
85
- return wrapper
86
-
87
- return dewrapper
88
-
89
-
90
- def try_except(callback=None, is_print=True):
91
- """
92
- 使用装饰器使用try_except,并用callback回收错误信息
93
- :param func:
94
- :return:
95
- """
96
-
97
- def dewrapper(func):
98
-
99
- @wraps(func)
100
- def wrapper(*args, **kwargs):
101
- try:
102
- return func(*args, **kwargs)
103
- except BaseException as e:
104
- if callback is not None:
105
- callback(*sys.exc_info(), *args, **kwargs)
106
- if is_print:
107
- print(traceback.format_exc())
108
- return traceback.format_exc()
109
-
110
- return wrapper
111
-
112
- return dewrapper
113
-
114
-
115
- def try_except2(callback=None, is_print=True):
116
- """
117
- 使用装饰器使用try_except,并用callback回收错误信息
118
- :param func:
119
- :return:
120
- """
121
-
122
- def dewrapper(func):
123
-
124
- @wraps(func)
125
- def wrapper(*args, **kwargs):
126
- try:
127
- return func(*args, **kwargs)
128
- except BaseException as e:
129
- if callback is not None:
130
- callback(*sys.exc_info(), *args, **kwargs)
131
- if is_print:
132
- print(traceback.format_exc())
133
- return False, {"traceback": traceback.format_exc()}
134
-
135
- return wrapper
136
-
137
- return dewrapper
138
-
139
-
140
- def try_except2_async(callback=None, is_print=True):
141
- """
142
- 使用装饰器使用try_except,并用callback回收错误信息
143
- 异步函数专用装饰器
144
- :param func:
145
- :return:
146
- """
147
-
148
- def dewrapper(func):
149
-
150
- @functools.wraps(func)
151
- async def wrapper(*args, **kwargs):
152
- try:
153
- return await func(*args, **kwargs)
154
- except BaseException as e:
155
- if callback is not None:
156
- if is_print:
157
- print("traceback=>", traceback.format_exc())
158
- bools, one_dic = await callback(*sys.exc_info(), *args, **kwargs)
159
- return bools, one_dic
160
-
161
- return False, {"traceback": traceback.format_exc()}
162
-
163
- return wrapper
164
-
165
- return dewrapper
166
-
167
-
168
- # https://blog.csdn.net/qq_39314099/article/details/83822593
169
- def deprecated(message):
170
- def deprecated_decorator(func):
171
- @wraps(func)
172
- def deprecated_func(*args, **kwargs):
173
- warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
174
- category=DeprecationWarning,
175
- stacklevel=2)
176
- warnings.simplefilter('default', DeprecationWarning)
177
- return func(*args, **kwargs)
178
-
179
- return deprecated_func
180
-
181
- return deprecated_decorator
182
-
183
-
184
- def request_try_except(func):
185
- """
186
- 拦截request请求错误
187
- :param func:
188
- :return:
189
- """
190
-
191
- @wraps(func)
192
- def wrapTheFunction(*args, **kwargs):
193
- try:
194
- return func(*args, **kwargs)
195
- except requests.exceptions.ReadTimeout as e:
196
- try:
197
- funcname = func.__name__
198
- except:
199
- funcname = "错误,获取func的name失败"
200
- dicts = {"code": MsgCode.TIME_OUT_ERROR,
201
- "msg": "time out, {},{}".format(repr(e), funcname)}
202
- return False, dicts
203
- except requests.exceptions.ProxyError as e:
204
- dicts = {"code": MsgCode.PROXY_ERROR,
205
- "msg": "proxy error, {}".format(repr(e))}
206
- return False, dicts
207
- except:
208
- dicts = {"code": MsgCode.ON_KNOW,
209
- "msg": traceback.format_exc()}
210
- return False, dicts
211
-
212
- return wrapTheFunction
213
-
214
-
215
- def aiohttp_try_except(func):
216
- """
217
- 拦截aiohttp 异步请求错误
218
- :param func:
219
- :return:
220
- """
221
-
222
- @wraps(func)
223
- async def wrapTheFunction(*args, **kwargs):
224
- try:
225
- return await func(*args, **kwargs)
226
- except ClientProxyConnectionError as e:
227
- dicts = {"code": MsgCode.PROXY_ERROR,
228
- "msg": "proxy error, {}".format(repr(e))}
229
- return False, dicts
230
- except asyncio.exceptions.TimeoutError as e:
231
- try:
232
- funcname = func.__name__
233
- except:
234
- funcname = "错误,获取func的name失败"
235
- dicts = {"code": MsgCode.TIME_OUT_ERROR,
236
- "msg": "time out, {},{}".format(repr(e), funcname)}
237
- return False, dicts
238
- except aiohttp.client_exceptions.ClientPayloadError as e:
239
- dicts = {"code": MsgCode.PAYLOAD_ERROR,
240
- "msg": "非法的压缩格式,错误的chunk编码,数据不足Content-length, {}".format(repr(e))}
241
- return False, dicts
242
- except DecodeError as e:
243
-
244
- dicts = {"code": MsgCode.GRPC_MESSAGE_DECODEERROR,
245
- "msg": "grpc message 消息结构错误 {}".format(repr(e))}
246
- return False, dicts
247
- except Exception as e:
248
- dicts = {"code": MsgCode.ON_KNOW,
249
- "msg": traceback.format_exc()}
250
- return False, dicts
251
-
252
- return wrapTheFunction
253
-
254
-
255
- def mongodb_try_except(func):
256
- """
257
- 拦截mongodb执行错误
258
- :param func:
259
- :return:
260
- """
261
-
262
- @wraps(func)
263
- async def wrapTheFunction(*args, **kwargs):
264
- try:
265
- return await func(*args, **kwargs)
266
- except Exception as e:
267
- dict_ = {
268
- "status": "FAILED",
269
- "msg_code": MsgCode.NO_RESOURCE,
270
- "msg": "保存到数据库失败",
271
- "data": {"err": repr(e)}
272
- }
273
- return False, dict_
274
-
275
- return wrapTheFunction
276
-
277
-
278
- def retry_func(retry_times=3, sleep_time=1):
279
- """函数重复执行次数"""
280
-
281
- def retry_decorator(func):
282
- @functools.wraps(func)
283
- def wrapper_func(*args, **kwargs):
284
- flag = 0
285
- while flag < retry_times:
286
- res = func(*args, **kwargs)
287
- res_bool = res
288
- if isinstance(res, tuple):
289
- res_bool = res[0]
290
- if not res_bool:
291
- flag += 1
292
- # logger.warning("{0} execute {1} times".format((func.__name__), flag))
293
- time.sleep(sleep_time)
294
- continue
295
- else:
296
- return res
297
- return res
298
- return wrapper_func
299
-
300
- return retry_decorator
301
-
302
-
303
- def retry_func_async(retry_times=3, sleep_time=1):
304
- """函数重复执行次数"""
305
-
306
- def retry_decorator(func):
307
- @functools.wraps(func)
308
- async def wrapper_func(*args, **kwargs):
309
- flag = 0
310
- res = (False, {})
311
- while flag < retry_times:
312
- res = await func(*args, **kwargs)
313
- res_bool = res
314
- if isinstance(res, tuple):
315
- res_bool = res[0]
316
- if not res_bool:
317
- flag += 1
318
- # logger.warning("{0} execute {1} times".format((func.__name__), flag))
319
- await asyncio.sleep(sleep_time)
320
- continue
321
- else:
322
- return res
323
- return res
324
-
325
- return wrapper_func
326
-
327
- return retry_decorator
1
+ import asyncio
2
+ import functools
3
+ import sys
4
+ import time
5
+ import traceback
6
+ import warnings
7
+ from functools import wraps
8
+
9
+ # https://www.jianshu.com/p/ee82b941772a
10
+ import aiohttp.client_exceptions
11
+ import requests
12
+ from aiohttp import ClientProxyConnectionError
13
+ from google.protobuf.message import DecodeError
14
+
15
+ from re_common.baselibrary.utils.core.requests_core import MsgCode
16
+
17
+
18
+ def InterVal(start_time, interval):
19
+ '''
20
+ 执行函数必须间隔多少时间,没有到指定的时间函数就不会被执行
21
+ :param start_time: 传入开始时间
22
+ :param interval: 秒数
23
+ :return: 返回执行结果 否则返回 None
24
+ '''
25
+
26
+ def dewrapper(func):
27
+ """
28
+ 它能把原函数的元信息拷贝到装饰器里面的 func 函数中。
29
+ 函数的元信息包括docstring、name、参数列表等等。可以尝试去除@functools.wraps(func),
30
+ 你会发现test.__name__的输出变成了wrapper。
31
+ """
32
+
33
+ @wraps(func)
34
+ def wrapper(*args, **kwargs):
35
+ start = start_time
36
+ time_now = int(time.time())
37
+ if time_now - start > interval:
38
+ result = func(*args, **kwargs)
39
+ return result
40
+ else:
41
+ return None
42
+
43
+ return wrapper
44
+
45
+ return dewrapper
46
+
47
+
48
+ def timethis(func):
49
+ """
50
+ 函数执行的时间差
51
+ :param func:
52
+ :return:
53
+ """
54
+
55
+ @wraps(func)
56
+ def wrapper(*args, **kwargs):
57
+ start = time.time()
58
+ result = func(*args, **kwargs)
59
+ end = time.time()
60
+ print(func.__name__, end - start)
61
+ return result
62
+
63
+ return wrapper
64
+
65
+
66
+ def func_time(callback=None, is_print=True):
67
+ """
68
+ 装饰器获取时间
69
+ :param func:
70
+ :return:
71
+ """
72
+
73
+ def dewrapper(func):
74
+ @functools.wraps(func)
75
+ async def wrapper(*args, **kwargs):
76
+ start = time.time()
77
+ result = await func(*args, **kwargs)
78
+ end = time.time()
79
+ if is_print:
80
+ print(func.__name__, end - start)
81
+ if callback is not None:
82
+ await callback(start, end)
83
+ return result
84
+
85
+ return wrapper
86
+
87
+ return dewrapper
88
+
89
+
90
+ def try_except(callback=None, is_print=True):
91
+ """
92
+ 使用装饰器使用try_except,并用callback回收错误信息
93
+ :param func:
94
+ :return:
95
+ """
96
+
97
+ def dewrapper(func):
98
+
99
+ @wraps(func)
100
+ def wrapper(*args, **kwargs):
101
+ try:
102
+ return func(*args, **kwargs)
103
+ except BaseException as e:
104
+ if callback is not None:
105
+ callback(*sys.exc_info(), *args, **kwargs)
106
+ if is_print:
107
+ print(traceback.format_exc())
108
+ return traceback.format_exc()
109
+
110
+ return wrapper
111
+
112
+ return dewrapper
113
+
114
+
115
+ def try_except2(callback=None, is_print=True):
116
+ """
117
+ 使用装饰器使用try_except,并用callback回收错误信息
118
+ :param func:
119
+ :return:
120
+ """
121
+
122
+ def dewrapper(func):
123
+
124
+ @wraps(func)
125
+ def wrapper(*args, **kwargs):
126
+ try:
127
+ return func(*args, **kwargs)
128
+ except BaseException as e:
129
+ if callback is not None:
130
+ callback(*sys.exc_info(), *args, **kwargs)
131
+ if is_print:
132
+ print(traceback.format_exc())
133
+ return False, {"traceback": traceback.format_exc()}
134
+
135
+ return wrapper
136
+
137
+ return dewrapper
138
+
139
+
140
+ def try_except2_async(callback=None, is_print=True):
141
+ """
142
+ 使用装饰器使用try_except,并用callback回收错误信息
143
+ 异步函数专用装饰器
144
+ :param func:
145
+ :return:
146
+ """
147
+
148
+ def dewrapper(func):
149
+
150
+ @functools.wraps(func)
151
+ async def wrapper(*args, **kwargs):
152
+ try:
153
+ return await func(*args, **kwargs)
154
+ except BaseException as e:
155
+ if callback is not None:
156
+ if is_print:
157
+ print("traceback=>", traceback.format_exc())
158
+ bools, one_dic = await callback(*sys.exc_info(), *args, **kwargs)
159
+ return bools, one_dic
160
+
161
+ return False, {"traceback": traceback.format_exc()}
162
+
163
+ return wrapper
164
+
165
+ return dewrapper
166
+
167
+
168
+ # https://blog.csdn.net/qq_39314099/article/details/83822593
169
+ def deprecated(message):
170
+ def deprecated_decorator(func):
171
+ @wraps(func)
172
+ def deprecated_func(*args, **kwargs):
173
+ warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
174
+ category=DeprecationWarning,
175
+ stacklevel=2)
176
+ warnings.simplefilter('default', DeprecationWarning)
177
+ return func(*args, **kwargs)
178
+
179
+ return deprecated_func
180
+
181
+ return deprecated_decorator
182
+
183
+
184
+ def request_try_except(func):
185
+ """
186
+ 拦截request请求错误
187
+ :param func:
188
+ :return:
189
+ """
190
+
191
+ @wraps(func)
192
+ def wrapTheFunction(*args, **kwargs):
193
+ try:
194
+ return func(*args, **kwargs)
195
+ except requests.exceptions.ReadTimeout as e:
196
+ try:
197
+ funcname = func.__name__
198
+ except:
199
+ funcname = "错误,获取func的name失败"
200
+ dicts = {"code": MsgCode.TIME_OUT_ERROR,
201
+ "msg": "time out, {},{}".format(repr(e), funcname)}
202
+ return False, dicts
203
+ except requests.exceptions.ProxyError as e:
204
+ dicts = {"code": MsgCode.PROXY_ERROR,
205
+ "msg": "proxy error, {}".format(repr(e))}
206
+ return False, dicts
207
+ except:
208
+ dicts = {"code": MsgCode.ON_KNOW,
209
+ "msg": traceback.format_exc()}
210
+ return False, dicts
211
+
212
+ return wrapTheFunction
213
+
214
+
215
+ def aiohttp_try_except(func):
216
+ """
217
+ 拦截aiohttp 异步请求错误
218
+ :param func:
219
+ :return:
220
+ """
221
+
222
+ @wraps(func)
223
+ async def wrapTheFunction(*args, **kwargs):
224
+ try:
225
+ return await func(*args, **kwargs)
226
+ except ClientProxyConnectionError as e:
227
+ dicts = {"code": MsgCode.PROXY_ERROR,
228
+ "msg": "proxy error, {}".format(repr(e))}
229
+ return False, dicts
230
+ except asyncio.exceptions.TimeoutError as e:
231
+ try:
232
+ funcname = func.__name__
233
+ except:
234
+ funcname = "错误,获取func的name失败"
235
+ dicts = {"code": MsgCode.TIME_OUT_ERROR,
236
+ "msg": "time out, {},{}".format(repr(e), funcname)}
237
+ return False, dicts
238
+ except aiohttp.client_exceptions.ClientPayloadError as e:
239
+ dicts = {"code": MsgCode.PAYLOAD_ERROR,
240
+ "msg": "非法的压缩格式,错误的chunk编码,数据不足Content-length, {}".format(repr(e))}
241
+ return False, dicts
242
+ except DecodeError as e:
243
+
244
+ dicts = {"code": MsgCode.GRPC_MESSAGE_DECODEERROR,
245
+ "msg": "grpc message 消息结构错误 {}".format(repr(e))}
246
+ return False, dicts
247
+ except Exception as e:
248
+ dicts = {"code": MsgCode.ON_KNOW,
249
+ "msg": traceback.format_exc()}
250
+ return False, dicts
251
+
252
+ return wrapTheFunction
253
+
254
+
255
+ def mongodb_try_except(func):
256
+ """
257
+ 拦截mongodb执行错误
258
+ :param func:
259
+ :return:
260
+ """
261
+
262
+ @wraps(func)
263
+ async def wrapTheFunction(*args, **kwargs):
264
+ try:
265
+ return await func(*args, **kwargs)
266
+ except Exception as e:
267
+ dict_ = {
268
+ "status": "FAILED",
269
+ "msg_code": MsgCode.NO_RESOURCE,
270
+ "msg": "保存到数据库失败",
271
+ "data": {"err": repr(e)}
272
+ }
273
+ return False, dict_
274
+
275
+ return wrapTheFunction
276
+
277
+
278
+ def retry_func(retry_times=3, sleep_time=1):
279
+ """函数重复执行次数"""
280
+
281
+ def retry_decorator(func):
282
+ @functools.wraps(func)
283
+ def wrapper_func(*args, **kwargs):
284
+ flag = 0
285
+ while flag < retry_times:
286
+ res = func(*args, **kwargs)
287
+ res_bool = res
288
+ if isinstance(res, tuple):
289
+ res_bool = res[0]
290
+ if not res_bool:
291
+ flag += 1
292
+ # logger.warning("{0} execute {1} times".format((func.__name__), flag))
293
+ time.sleep(sleep_time)
294
+ continue
295
+ else:
296
+ return res
297
+ return res
298
+ return wrapper_func
299
+
300
+ return retry_decorator
301
+
302
+
303
+ def retry_func_async(retry_times=3, sleep_time=1):
304
+ """函数重复执行次数"""
305
+
306
+ def retry_decorator(func):
307
+ @functools.wraps(func)
308
+ async def wrapper_func(*args, **kwargs):
309
+ flag = 0
310
+ res = (False, {})
311
+ while flag < retry_times:
312
+ res = await func(*args, **kwargs)
313
+ res_bool = res
314
+ if isinstance(res, tuple):
315
+ res_bool = res[0]
316
+ if not res_bool:
317
+ flag += 1
318
+ # logger.warning("{0} execute {1} times".format((func.__name__), flag))
319
+ await asyncio.sleep(sleep_time)
320
+ continue
321
+ else:
322
+ return res
323
+ return res
324
+
325
+ return wrapper_func
326
+
327
+ return retry_decorator