re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +77 -77
- re_common/v2/baselibrary/utils/db.py +156 -156
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +186 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +271 -278
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/LICENSE +201 -201
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/METADATA +16 -16
- re_common-10.0.39.dist-info/RECORD +248 -0
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/WHEEL +1 -1
- re_common-10.0.37.dist-info/RECORD +0 -248
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/top_level.txt +0 -0
|
@@ -1,327 +1,327 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import functools
|
|
3
|
-
import sys
|
|
4
|
-
import time
|
|
5
|
-
import traceback
|
|
6
|
-
import warnings
|
|
7
|
-
from functools import wraps
|
|
8
|
-
|
|
9
|
-
# https://www.jianshu.com/p/ee82b941772a
|
|
10
|
-
import aiohttp.client_exceptions
|
|
11
|
-
import requests
|
|
12
|
-
from aiohttp import ClientProxyConnectionError
|
|
13
|
-
from google.protobuf.message import DecodeError
|
|
14
|
-
|
|
15
|
-
from re_common.baselibrary.utils.core.requests_core import MsgCode
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def InterVal(start_time, interval):
|
|
19
|
-
'''
|
|
20
|
-
执行函数必须间隔多少时间,没有到指定的时间函数就不会被执行
|
|
21
|
-
:param start_time: 传入开始时间
|
|
22
|
-
:param interval: 秒数
|
|
23
|
-
:return: 返回执行结果 否则返回 None
|
|
24
|
-
'''
|
|
25
|
-
|
|
26
|
-
def dewrapper(func):
|
|
27
|
-
"""
|
|
28
|
-
它能把原函数的元信息拷贝到装饰器里面的 func 函数中。
|
|
29
|
-
函数的元信息包括docstring、name、参数列表等等。可以尝试去除@functools.wraps(func),
|
|
30
|
-
你会发现test.__name__的输出变成了wrapper。
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
@wraps(func)
|
|
34
|
-
def wrapper(*args, **kwargs):
|
|
35
|
-
start = start_time
|
|
36
|
-
time_now = int(time.time())
|
|
37
|
-
if time_now - start > interval:
|
|
38
|
-
result = func(*args, **kwargs)
|
|
39
|
-
return result
|
|
40
|
-
else:
|
|
41
|
-
return None
|
|
42
|
-
|
|
43
|
-
return wrapper
|
|
44
|
-
|
|
45
|
-
return dewrapper
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
def timethis(func):
|
|
49
|
-
"""
|
|
50
|
-
函数执行的时间差
|
|
51
|
-
:param func:
|
|
52
|
-
:return:
|
|
53
|
-
"""
|
|
54
|
-
|
|
55
|
-
@wraps(func)
|
|
56
|
-
def wrapper(*args, **kwargs):
|
|
57
|
-
start = time.time()
|
|
58
|
-
result = func(*args, **kwargs)
|
|
59
|
-
end = time.time()
|
|
60
|
-
print(func.__name__, end - start)
|
|
61
|
-
return result
|
|
62
|
-
|
|
63
|
-
return wrapper
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def func_time(callback=None, is_print=True):
|
|
67
|
-
"""
|
|
68
|
-
装饰器获取时间
|
|
69
|
-
:param func:
|
|
70
|
-
:return:
|
|
71
|
-
"""
|
|
72
|
-
|
|
73
|
-
def dewrapper(func):
|
|
74
|
-
@functools.wraps(func)
|
|
75
|
-
async def wrapper(*args, **kwargs):
|
|
76
|
-
start = time.time()
|
|
77
|
-
result = await func(*args, **kwargs)
|
|
78
|
-
end = time.time()
|
|
79
|
-
if is_print:
|
|
80
|
-
print(func.__name__, end - start)
|
|
81
|
-
if callback is not None:
|
|
82
|
-
await callback(start, end)
|
|
83
|
-
return result
|
|
84
|
-
|
|
85
|
-
return wrapper
|
|
86
|
-
|
|
87
|
-
return dewrapper
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
def try_except(callback=None, is_print=True):
|
|
91
|
-
"""
|
|
92
|
-
使用装饰器使用try_except,并用callback回收错误信息
|
|
93
|
-
:param func:
|
|
94
|
-
:return:
|
|
95
|
-
"""
|
|
96
|
-
|
|
97
|
-
def dewrapper(func):
|
|
98
|
-
|
|
99
|
-
@wraps(func)
|
|
100
|
-
def wrapper(*args, **kwargs):
|
|
101
|
-
try:
|
|
102
|
-
return func(*args, **kwargs)
|
|
103
|
-
except BaseException as e:
|
|
104
|
-
if callback is not None:
|
|
105
|
-
callback(*sys.exc_info(), *args, **kwargs)
|
|
106
|
-
if is_print:
|
|
107
|
-
print(traceback.format_exc())
|
|
108
|
-
return traceback.format_exc()
|
|
109
|
-
|
|
110
|
-
return wrapper
|
|
111
|
-
|
|
112
|
-
return dewrapper
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
def try_except2(callback=None, is_print=True):
|
|
116
|
-
"""
|
|
117
|
-
使用装饰器使用try_except,并用callback回收错误信息
|
|
118
|
-
:param func:
|
|
119
|
-
:return:
|
|
120
|
-
"""
|
|
121
|
-
|
|
122
|
-
def dewrapper(func):
|
|
123
|
-
|
|
124
|
-
@wraps(func)
|
|
125
|
-
def wrapper(*args, **kwargs):
|
|
126
|
-
try:
|
|
127
|
-
return func(*args, **kwargs)
|
|
128
|
-
except BaseException as e:
|
|
129
|
-
if callback is not None:
|
|
130
|
-
callback(*sys.exc_info(), *args, **kwargs)
|
|
131
|
-
if is_print:
|
|
132
|
-
print(traceback.format_exc())
|
|
133
|
-
return False, {"traceback": traceback.format_exc()}
|
|
134
|
-
|
|
135
|
-
return wrapper
|
|
136
|
-
|
|
137
|
-
return dewrapper
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
def try_except2_async(callback=None, is_print=True):
|
|
141
|
-
"""
|
|
142
|
-
使用装饰器使用try_except,并用callback回收错误信息
|
|
143
|
-
异步函数专用装饰器
|
|
144
|
-
:param func:
|
|
145
|
-
:return:
|
|
146
|
-
"""
|
|
147
|
-
|
|
148
|
-
def dewrapper(func):
|
|
149
|
-
|
|
150
|
-
@functools.wraps(func)
|
|
151
|
-
async def wrapper(*args, **kwargs):
|
|
152
|
-
try:
|
|
153
|
-
return await func(*args, **kwargs)
|
|
154
|
-
except BaseException as e:
|
|
155
|
-
if callback is not None:
|
|
156
|
-
if is_print:
|
|
157
|
-
print("traceback=>", traceback.format_exc())
|
|
158
|
-
bools, one_dic = await callback(*sys.exc_info(), *args, **kwargs)
|
|
159
|
-
return bools, one_dic
|
|
160
|
-
|
|
161
|
-
return False, {"traceback": traceback.format_exc()}
|
|
162
|
-
|
|
163
|
-
return wrapper
|
|
164
|
-
|
|
165
|
-
return dewrapper
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
# https://blog.csdn.net/qq_39314099/article/details/83822593
|
|
169
|
-
def deprecated(message):
|
|
170
|
-
def deprecated_decorator(func):
|
|
171
|
-
@wraps(func)
|
|
172
|
-
def deprecated_func(*args, **kwargs):
|
|
173
|
-
warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
|
|
174
|
-
category=DeprecationWarning,
|
|
175
|
-
stacklevel=2)
|
|
176
|
-
warnings.simplefilter('default', DeprecationWarning)
|
|
177
|
-
return func(*args, **kwargs)
|
|
178
|
-
|
|
179
|
-
return deprecated_func
|
|
180
|
-
|
|
181
|
-
return deprecated_decorator
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
def request_try_except(func):
|
|
185
|
-
"""
|
|
186
|
-
拦截request请求错误
|
|
187
|
-
:param func:
|
|
188
|
-
:return:
|
|
189
|
-
"""
|
|
190
|
-
|
|
191
|
-
@wraps(func)
|
|
192
|
-
def wrapTheFunction(*args, **kwargs):
|
|
193
|
-
try:
|
|
194
|
-
return func(*args, **kwargs)
|
|
195
|
-
except requests.exceptions.ReadTimeout as e:
|
|
196
|
-
try:
|
|
197
|
-
funcname = func.__name__
|
|
198
|
-
except:
|
|
199
|
-
funcname = "错误,获取func的name失败"
|
|
200
|
-
dicts = {"code": MsgCode.TIME_OUT_ERROR,
|
|
201
|
-
"msg": "time out, {},{}".format(repr(e), funcname)}
|
|
202
|
-
return False, dicts
|
|
203
|
-
except requests.exceptions.ProxyError as e:
|
|
204
|
-
dicts = {"code": MsgCode.PROXY_ERROR,
|
|
205
|
-
"msg": "proxy error, {}".format(repr(e))}
|
|
206
|
-
return False, dicts
|
|
207
|
-
except:
|
|
208
|
-
dicts = {"code": MsgCode.ON_KNOW,
|
|
209
|
-
"msg": traceback.format_exc()}
|
|
210
|
-
return False, dicts
|
|
211
|
-
|
|
212
|
-
return wrapTheFunction
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def aiohttp_try_except(func):
|
|
216
|
-
"""
|
|
217
|
-
拦截aiohttp 异步请求错误
|
|
218
|
-
:param func:
|
|
219
|
-
:return:
|
|
220
|
-
"""
|
|
221
|
-
|
|
222
|
-
@wraps(func)
|
|
223
|
-
async def wrapTheFunction(*args, **kwargs):
|
|
224
|
-
try:
|
|
225
|
-
return await func(*args, **kwargs)
|
|
226
|
-
except ClientProxyConnectionError as e:
|
|
227
|
-
dicts = {"code": MsgCode.PROXY_ERROR,
|
|
228
|
-
"msg": "proxy error, {}".format(repr(e))}
|
|
229
|
-
return False, dicts
|
|
230
|
-
except asyncio.exceptions.TimeoutError as e:
|
|
231
|
-
try:
|
|
232
|
-
funcname = func.__name__
|
|
233
|
-
except:
|
|
234
|
-
funcname = "错误,获取func的name失败"
|
|
235
|
-
dicts = {"code": MsgCode.TIME_OUT_ERROR,
|
|
236
|
-
"msg": "time out, {},{}".format(repr(e), funcname)}
|
|
237
|
-
return False, dicts
|
|
238
|
-
except aiohttp.client_exceptions.ClientPayloadError as e:
|
|
239
|
-
dicts = {"code": MsgCode.PAYLOAD_ERROR,
|
|
240
|
-
"msg": "非法的压缩格式,错误的chunk编码,数据不足Content-length, {}".format(repr(e))}
|
|
241
|
-
return False, dicts
|
|
242
|
-
except DecodeError as e:
|
|
243
|
-
|
|
244
|
-
dicts = {"code": MsgCode.GRPC_MESSAGE_DECODEERROR,
|
|
245
|
-
"msg": "grpc message 消息结构错误 {}".format(repr(e))}
|
|
246
|
-
return False, dicts
|
|
247
|
-
except Exception as e:
|
|
248
|
-
dicts = {"code": MsgCode.ON_KNOW,
|
|
249
|
-
"msg": traceback.format_exc()}
|
|
250
|
-
return False, dicts
|
|
251
|
-
|
|
252
|
-
return wrapTheFunction
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
def mongodb_try_except(func):
|
|
256
|
-
"""
|
|
257
|
-
拦截mongodb执行错误
|
|
258
|
-
:param func:
|
|
259
|
-
:return:
|
|
260
|
-
"""
|
|
261
|
-
|
|
262
|
-
@wraps(func)
|
|
263
|
-
async def wrapTheFunction(*args, **kwargs):
|
|
264
|
-
try:
|
|
265
|
-
return await func(*args, **kwargs)
|
|
266
|
-
except Exception as e:
|
|
267
|
-
dict_ = {
|
|
268
|
-
"status": "FAILED",
|
|
269
|
-
"msg_code": MsgCode.NO_RESOURCE,
|
|
270
|
-
"msg": "保存到数据库失败",
|
|
271
|
-
"data": {"err": repr(e)}
|
|
272
|
-
}
|
|
273
|
-
return False, dict_
|
|
274
|
-
|
|
275
|
-
return wrapTheFunction
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
def retry_func(retry_times=3, sleep_time=1):
|
|
279
|
-
"""函数重复执行次数"""
|
|
280
|
-
|
|
281
|
-
def retry_decorator(func):
|
|
282
|
-
@functools.wraps(func)
|
|
283
|
-
def wrapper_func(*args, **kwargs):
|
|
284
|
-
flag = 0
|
|
285
|
-
while flag < retry_times:
|
|
286
|
-
res = func(*args, **kwargs)
|
|
287
|
-
res_bool = res
|
|
288
|
-
if isinstance(res, tuple):
|
|
289
|
-
res_bool = res[0]
|
|
290
|
-
if not res_bool:
|
|
291
|
-
flag += 1
|
|
292
|
-
# logger.warning("{0} execute {1} times".format((func.__name__), flag))
|
|
293
|
-
time.sleep(sleep_time)
|
|
294
|
-
continue
|
|
295
|
-
else:
|
|
296
|
-
return res
|
|
297
|
-
return res
|
|
298
|
-
return wrapper_func
|
|
299
|
-
|
|
300
|
-
return retry_decorator
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
def retry_func_async(retry_times=3, sleep_time=1):
|
|
304
|
-
"""函数重复执行次数"""
|
|
305
|
-
|
|
306
|
-
def retry_decorator(func):
|
|
307
|
-
@functools.wraps(func)
|
|
308
|
-
async def wrapper_func(*args, **kwargs):
|
|
309
|
-
flag = 0
|
|
310
|
-
res = (False, {})
|
|
311
|
-
while flag < retry_times:
|
|
312
|
-
res = await func(*args, **kwargs)
|
|
313
|
-
res_bool = res
|
|
314
|
-
if isinstance(res, tuple):
|
|
315
|
-
res_bool = res[0]
|
|
316
|
-
if not res_bool:
|
|
317
|
-
flag += 1
|
|
318
|
-
# logger.warning("{0} execute {1} times".format((func.__name__), flag))
|
|
319
|
-
await asyncio.sleep(sleep_time)
|
|
320
|
-
continue
|
|
321
|
-
else:
|
|
322
|
-
return res
|
|
323
|
-
return res
|
|
324
|
-
|
|
325
|
-
return wrapper_func
|
|
326
|
-
|
|
327
|
-
return retry_decorator
|
|
1
|
+
import asyncio
|
|
2
|
+
import functools
|
|
3
|
+
import sys
|
|
4
|
+
import time
|
|
5
|
+
import traceback
|
|
6
|
+
import warnings
|
|
7
|
+
from functools import wraps
|
|
8
|
+
|
|
9
|
+
# https://www.jianshu.com/p/ee82b941772a
|
|
10
|
+
import aiohttp.client_exceptions
|
|
11
|
+
import requests
|
|
12
|
+
from aiohttp import ClientProxyConnectionError
|
|
13
|
+
from google.protobuf.message import DecodeError
|
|
14
|
+
|
|
15
|
+
from re_common.baselibrary.utils.core.requests_core import MsgCode
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def InterVal(start_time, interval):
|
|
19
|
+
'''
|
|
20
|
+
执行函数必须间隔多少时间,没有到指定的时间函数就不会被执行
|
|
21
|
+
:param start_time: 传入开始时间
|
|
22
|
+
:param interval: 秒数
|
|
23
|
+
:return: 返回执行结果 否则返回 None
|
|
24
|
+
'''
|
|
25
|
+
|
|
26
|
+
def dewrapper(func):
|
|
27
|
+
"""
|
|
28
|
+
它能把原函数的元信息拷贝到装饰器里面的 func 函数中。
|
|
29
|
+
函数的元信息包括docstring、name、参数列表等等。可以尝试去除@functools.wraps(func),
|
|
30
|
+
你会发现test.__name__的输出变成了wrapper。
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
@wraps(func)
|
|
34
|
+
def wrapper(*args, **kwargs):
|
|
35
|
+
start = start_time
|
|
36
|
+
time_now = int(time.time())
|
|
37
|
+
if time_now - start > interval:
|
|
38
|
+
result = func(*args, **kwargs)
|
|
39
|
+
return result
|
|
40
|
+
else:
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
return wrapper
|
|
44
|
+
|
|
45
|
+
return dewrapper
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def timethis(func):
|
|
49
|
+
"""
|
|
50
|
+
函数执行的时间差
|
|
51
|
+
:param func:
|
|
52
|
+
:return:
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
@wraps(func)
|
|
56
|
+
def wrapper(*args, **kwargs):
|
|
57
|
+
start = time.time()
|
|
58
|
+
result = func(*args, **kwargs)
|
|
59
|
+
end = time.time()
|
|
60
|
+
print(func.__name__, end - start)
|
|
61
|
+
return result
|
|
62
|
+
|
|
63
|
+
return wrapper
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def func_time(callback=None, is_print=True):
|
|
67
|
+
"""
|
|
68
|
+
装饰器获取时间
|
|
69
|
+
:param func:
|
|
70
|
+
:return:
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
def dewrapper(func):
|
|
74
|
+
@functools.wraps(func)
|
|
75
|
+
async def wrapper(*args, **kwargs):
|
|
76
|
+
start = time.time()
|
|
77
|
+
result = await func(*args, **kwargs)
|
|
78
|
+
end = time.time()
|
|
79
|
+
if is_print:
|
|
80
|
+
print(func.__name__, end - start)
|
|
81
|
+
if callback is not None:
|
|
82
|
+
await callback(start, end)
|
|
83
|
+
return result
|
|
84
|
+
|
|
85
|
+
return wrapper
|
|
86
|
+
|
|
87
|
+
return dewrapper
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def try_except(callback=None, is_print=True):
|
|
91
|
+
"""
|
|
92
|
+
使用装饰器使用try_except,并用callback回收错误信息
|
|
93
|
+
:param func:
|
|
94
|
+
:return:
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
def dewrapper(func):
|
|
98
|
+
|
|
99
|
+
@wraps(func)
|
|
100
|
+
def wrapper(*args, **kwargs):
|
|
101
|
+
try:
|
|
102
|
+
return func(*args, **kwargs)
|
|
103
|
+
except BaseException as e:
|
|
104
|
+
if callback is not None:
|
|
105
|
+
callback(*sys.exc_info(), *args, **kwargs)
|
|
106
|
+
if is_print:
|
|
107
|
+
print(traceback.format_exc())
|
|
108
|
+
return traceback.format_exc()
|
|
109
|
+
|
|
110
|
+
return wrapper
|
|
111
|
+
|
|
112
|
+
return dewrapper
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def try_except2(callback=None, is_print=True):
|
|
116
|
+
"""
|
|
117
|
+
使用装饰器使用try_except,并用callback回收错误信息
|
|
118
|
+
:param func:
|
|
119
|
+
:return:
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
def dewrapper(func):
|
|
123
|
+
|
|
124
|
+
@wraps(func)
|
|
125
|
+
def wrapper(*args, **kwargs):
|
|
126
|
+
try:
|
|
127
|
+
return func(*args, **kwargs)
|
|
128
|
+
except BaseException as e:
|
|
129
|
+
if callback is not None:
|
|
130
|
+
callback(*sys.exc_info(), *args, **kwargs)
|
|
131
|
+
if is_print:
|
|
132
|
+
print(traceback.format_exc())
|
|
133
|
+
return False, {"traceback": traceback.format_exc()}
|
|
134
|
+
|
|
135
|
+
return wrapper
|
|
136
|
+
|
|
137
|
+
return dewrapper
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def try_except2_async(callback=None, is_print=True):
|
|
141
|
+
"""
|
|
142
|
+
使用装饰器使用try_except,并用callback回收错误信息
|
|
143
|
+
异步函数专用装饰器
|
|
144
|
+
:param func:
|
|
145
|
+
:return:
|
|
146
|
+
"""
|
|
147
|
+
|
|
148
|
+
def dewrapper(func):
|
|
149
|
+
|
|
150
|
+
@functools.wraps(func)
|
|
151
|
+
async def wrapper(*args, **kwargs):
|
|
152
|
+
try:
|
|
153
|
+
return await func(*args, **kwargs)
|
|
154
|
+
except BaseException as e:
|
|
155
|
+
if callback is not None:
|
|
156
|
+
if is_print:
|
|
157
|
+
print("traceback=>", traceback.format_exc())
|
|
158
|
+
bools, one_dic = await callback(*sys.exc_info(), *args, **kwargs)
|
|
159
|
+
return bools, one_dic
|
|
160
|
+
|
|
161
|
+
return False, {"traceback": traceback.format_exc()}
|
|
162
|
+
|
|
163
|
+
return wrapper
|
|
164
|
+
|
|
165
|
+
return dewrapper
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# https://blog.csdn.net/qq_39314099/article/details/83822593
|
|
169
|
+
def deprecated(message):
|
|
170
|
+
def deprecated_decorator(func):
|
|
171
|
+
@wraps(func)
|
|
172
|
+
def deprecated_func(*args, **kwargs):
|
|
173
|
+
warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
|
|
174
|
+
category=DeprecationWarning,
|
|
175
|
+
stacklevel=2)
|
|
176
|
+
warnings.simplefilter('default', DeprecationWarning)
|
|
177
|
+
return func(*args, **kwargs)
|
|
178
|
+
|
|
179
|
+
return deprecated_func
|
|
180
|
+
|
|
181
|
+
return deprecated_decorator
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def request_try_except(func):
|
|
185
|
+
"""
|
|
186
|
+
拦截request请求错误
|
|
187
|
+
:param func:
|
|
188
|
+
:return:
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
@wraps(func)
|
|
192
|
+
def wrapTheFunction(*args, **kwargs):
|
|
193
|
+
try:
|
|
194
|
+
return func(*args, **kwargs)
|
|
195
|
+
except requests.exceptions.ReadTimeout as e:
|
|
196
|
+
try:
|
|
197
|
+
funcname = func.__name__
|
|
198
|
+
except:
|
|
199
|
+
funcname = "错误,获取func的name失败"
|
|
200
|
+
dicts = {"code": MsgCode.TIME_OUT_ERROR,
|
|
201
|
+
"msg": "time out, {},{}".format(repr(e), funcname)}
|
|
202
|
+
return False, dicts
|
|
203
|
+
except requests.exceptions.ProxyError as e:
|
|
204
|
+
dicts = {"code": MsgCode.PROXY_ERROR,
|
|
205
|
+
"msg": "proxy error, {}".format(repr(e))}
|
|
206
|
+
return False, dicts
|
|
207
|
+
except:
|
|
208
|
+
dicts = {"code": MsgCode.ON_KNOW,
|
|
209
|
+
"msg": traceback.format_exc()}
|
|
210
|
+
return False, dicts
|
|
211
|
+
|
|
212
|
+
return wrapTheFunction
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def aiohttp_try_except(func):
|
|
216
|
+
"""
|
|
217
|
+
拦截aiohttp 异步请求错误
|
|
218
|
+
:param func:
|
|
219
|
+
:return:
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
@wraps(func)
|
|
223
|
+
async def wrapTheFunction(*args, **kwargs):
|
|
224
|
+
try:
|
|
225
|
+
return await func(*args, **kwargs)
|
|
226
|
+
except ClientProxyConnectionError as e:
|
|
227
|
+
dicts = {"code": MsgCode.PROXY_ERROR,
|
|
228
|
+
"msg": "proxy error, {}".format(repr(e))}
|
|
229
|
+
return False, dicts
|
|
230
|
+
except asyncio.exceptions.TimeoutError as e:
|
|
231
|
+
try:
|
|
232
|
+
funcname = func.__name__
|
|
233
|
+
except:
|
|
234
|
+
funcname = "错误,获取func的name失败"
|
|
235
|
+
dicts = {"code": MsgCode.TIME_OUT_ERROR,
|
|
236
|
+
"msg": "time out, {},{}".format(repr(e), funcname)}
|
|
237
|
+
return False, dicts
|
|
238
|
+
except aiohttp.client_exceptions.ClientPayloadError as e:
|
|
239
|
+
dicts = {"code": MsgCode.PAYLOAD_ERROR,
|
|
240
|
+
"msg": "非法的压缩格式,错误的chunk编码,数据不足Content-length, {}".format(repr(e))}
|
|
241
|
+
return False, dicts
|
|
242
|
+
except DecodeError as e:
|
|
243
|
+
|
|
244
|
+
dicts = {"code": MsgCode.GRPC_MESSAGE_DECODEERROR,
|
|
245
|
+
"msg": "grpc message 消息结构错误 {}".format(repr(e))}
|
|
246
|
+
return False, dicts
|
|
247
|
+
except Exception as e:
|
|
248
|
+
dicts = {"code": MsgCode.ON_KNOW,
|
|
249
|
+
"msg": traceback.format_exc()}
|
|
250
|
+
return False, dicts
|
|
251
|
+
|
|
252
|
+
return wrapTheFunction
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def mongodb_try_except(func):
|
|
256
|
+
"""
|
|
257
|
+
拦截mongodb执行错误
|
|
258
|
+
:param func:
|
|
259
|
+
:return:
|
|
260
|
+
"""
|
|
261
|
+
|
|
262
|
+
@wraps(func)
|
|
263
|
+
async def wrapTheFunction(*args, **kwargs):
|
|
264
|
+
try:
|
|
265
|
+
return await func(*args, **kwargs)
|
|
266
|
+
except Exception as e:
|
|
267
|
+
dict_ = {
|
|
268
|
+
"status": "FAILED",
|
|
269
|
+
"msg_code": MsgCode.NO_RESOURCE,
|
|
270
|
+
"msg": "保存到数据库失败",
|
|
271
|
+
"data": {"err": repr(e)}
|
|
272
|
+
}
|
|
273
|
+
return False, dict_
|
|
274
|
+
|
|
275
|
+
return wrapTheFunction
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def retry_func(retry_times=3, sleep_time=1):
|
|
279
|
+
"""函数重复执行次数"""
|
|
280
|
+
|
|
281
|
+
def retry_decorator(func):
|
|
282
|
+
@functools.wraps(func)
|
|
283
|
+
def wrapper_func(*args, **kwargs):
|
|
284
|
+
flag = 0
|
|
285
|
+
while flag < retry_times:
|
|
286
|
+
res = func(*args, **kwargs)
|
|
287
|
+
res_bool = res
|
|
288
|
+
if isinstance(res, tuple):
|
|
289
|
+
res_bool = res[0]
|
|
290
|
+
if not res_bool:
|
|
291
|
+
flag += 1
|
|
292
|
+
# logger.warning("{0} execute {1} times".format((func.__name__), flag))
|
|
293
|
+
time.sleep(sleep_time)
|
|
294
|
+
continue
|
|
295
|
+
else:
|
|
296
|
+
return res
|
|
297
|
+
return res
|
|
298
|
+
return wrapper_func
|
|
299
|
+
|
|
300
|
+
return retry_decorator
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def retry_func_async(retry_times=3, sleep_time=1):
|
|
304
|
+
"""函数重复执行次数"""
|
|
305
|
+
|
|
306
|
+
def retry_decorator(func):
|
|
307
|
+
@functools.wraps(func)
|
|
308
|
+
async def wrapper_func(*args, **kwargs):
|
|
309
|
+
flag = 0
|
|
310
|
+
res = (False, {})
|
|
311
|
+
while flag < retry_times:
|
|
312
|
+
res = await func(*args, **kwargs)
|
|
313
|
+
res_bool = res
|
|
314
|
+
if isinstance(res, tuple):
|
|
315
|
+
res_bool = res[0]
|
|
316
|
+
if not res_bool:
|
|
317
|
+
flag += 1
|
|
318
|
+
# logger.warning("{0} execute {1} times".format((func.__name__), flag))
|
|
319
|
+
await asyncio.sleep(sleep_time)
|
|
320
|
+
continue
|
|
321
|
+
else:
|
|
322
|
+
return res
|
|
323
|
+
return res
|
|
324
|
+
|
|
325
|
+
return wrapper_func
|
|
326
|
+
|
|
327
|
+
return retry_decorator
|