re-common 2.0.1__py3-none-any.whl → 10.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. re_common/baselibrary/__init__.py +4 -0
  2. re_common/baselibrary/baseabs/__init__.py +7 -0
  3. re_common/baselibrary/baseabs/baseabs.py +26 -0
  4. re_common/baselibrary/database/__init__.py +0 -0
  5. re_common/baselibrary/database/mbuilder.py +132 -0
  6. re_common/baselibrary/database/moudle.py +93 -0
  7. re_common/baselibrary/database/msqlite3.py +194 -0
  8. re_common/baselibrary/database/mysql.py +169 -0
  9. re_common/baselibrary/database/sql_factory.py +26 -0
  10. re_common/baselibrary/mthread/MThreadingRun.py +486 -0
  11. re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -0
  12. re_common/baselibrary/mthread/__init__.py +3 -0
  13. re_common/baselibrary/mthread/mythreading.py +695 -0
  14. re_common/baselibrary/pakge_other/__init__.py +0 -0
  15. re_common/baselibrary/pakge_other/socks.py +404 -0
  16. re_common/baselibrary/readconfig/__init__.py +0 -0
  17. re_common/baselibrary/readconfig/config_factory.py +18 -0
  18. re_common/baselibrary/readconfig/ini_config.py +317 -0
  19. re_common/baselibrary/readconfig/toml_config.py +49 -0
  20. re_common/baselibrary/temporary/__init__.py +0 -0
  21. re_common/baselibrary/temporary/envdata.py +36 -0
  22. re_common/baselibrary/tools/__init__.py +0 -0
  23. re_common/baselibrary/tools/all_requests/__init__.py +0 -0
  24. re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -0
  25. re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -0
  26. re_common/baselibrary/tools/all_requests/mrequest.py +412 -0
  27. re_common/baselibrary/tools/all_requests/requests_request.py +81 -0
  28. re_common/baselibrary/tools/batch_compre/__init__.py +0 -0
  29. re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -0
  30. re_common/baselibrary/tools/contrast_db3.py +123 -0
  31. re_common/baselibrary/tools/copy_file.py +39 -0
  32. re_common/baselibrary/tools/db3_2_sizedb3.py +102 -0
  33. re_common/baselibrary/tools/foreachgz.py +40 -0
  34. re_common/baselibrary/tools/get_attr.py +11 -0
  35. re_common/baselibrary/tools/image_to_pdf.py +62 -0
  36. re_common/baselibrary/tools/java_code_deal.py +139 -0
  37. re_common/baselibrary/tools/javacode.py +79 -0
  38. re_common/baselibrary/tools/mdb_db3.py +48 -0
  39. re_common/baselibrary/tools/merge_file.py +171 -0
  40. re_common/baselibrary/tools/merge_gz_file.py +165 -0
  41. re_common/baselibrary/tools/mhdfstools/__init__.py +0 -0
  42. re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -0
  43. re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -0
  44. re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -0
  45. re_common/baselibrary/tools/mongo_tools.py +50 -0
  46. re_common/baselibrary/tools/move_file.py +170 -0
  47. re_common/baselibrary/tools/move_mongo/__init__.py +0 -0
  48. re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -0
  49. re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -0
  50. re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -0
  51. re_common/baselibrary/tools/move_mongo/use_mv.py +93 -0
  52. re_common/baselibrary/tools/mpandas/__init__.py +0 -0
  53. re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -0
  54. re_common/baselibrary/tools/mpandas/pandas_visualization.py +8 -0
  55. re_common/baselibrary/tools/myparsel.py +104 -0
  56. re_common/baselibrary/tools/rename_dir_file.py +37 -0
  57. re_common/baselibrary/tools/sequoiadb_utils.py +398 -0
  58. re_common/baselibrary/tools/split_line_to_many.py +25 -0
  59. re_common/baselibrary/tools/stringtodicts.py +33 -0
  60. re_common/baselibrary/tools/workwechant_bot.py +84 -0
  61. re_common/baselibrary/utils/__init__.py +0 -0
  62. re_common/baselibrary/utils/baseaiohttp.py +296 -0
  63. re_common/baselibrary/utils/baseaiomysql.py +87 -0
  64. re_common/baselibrary/utils/baseallstep.py +191 -0
  65. re_common/baselibrary/utils/baseavro.py +19 -0
  66. re_common/baselibrary/utils/baseboto3.py +291 -0
  67. re_common/baselibrary/utils/basecsv.py +32 -0
  68. re_common/baselibrary/utils/basedict.py +133 -0
  69. re_common/baselibrary/utils/basedir.py +241 -0
  70. re_common/baselibrary/utils/baseencode.py +351 -0
  71. re_common/baselibrary/utils/baseencoding.py +29 -0
  72. re_common/baselibrary/utils/baseesdsl.py +86 -0
  73. re_common/baselibrary/utils/baseexcel.py +264 -0
  74. re_common/baselibrary/utils/baseexcept.py +109 -0
  75. re_common/baselibrary/utils/basefile.py +654 -0
  76. re_common/baselibrary/utils/baseftp.py +214 -0
  77. re_common/baselibrary/utils/basegzip.py +60 -0
  78. re_common/baselibrary/utils/basehdfs.py +135 -0
  79. re_common/baselibrary/utils/basehttpx.py +268 -0
  80. re_common/baselibrary/utils/baseip.py +87 -0
  81. re_common/baselibrary/utils/basejson.py +2 -0
  82. re_common/baselibrary/utils/baselist.py +32 -0
  83. re_common/baselibrary/utils/basemotor.py +190 -0
  84. re_common/baselibrary/utils/basemssql.py +98 -0
  85. re_common/baselibrary/utils/baseodbc.py +113 -0
  86. re_common/baselibrary/utils/basepandas.py +302 -0
  87. re_common/baselibrary/utils/basepeewee.py +11 -0
  88. re_common/baselibrary/utils/basepika.py +180 -0
  89. re_common/baselibrary/utils/basepydash.py +143 -0
  90. re_common/baselibrary/utils/basepymongo.py +230 -0
  91. re_common/baselibrary/utils/basequeue.py +22 -0
  92. re_common/baselibrary/utils/baserar.py +57 -0
  93. re_common/baselibrary/utils/baserequest.py +279 -0
  94. re_common/baselibrary/utils/baseset.py +8 -0
  95. re_common/baselibrary/utils/basesmb.py +403 -0
  96. re_common/baselibrary/utils/basestring.py +382 -0
  97. re_common/baselibrary/utils/basetime.py +320 -0
  98. re_common/baselibrary/utils/basetuple.py +0 -0
  99. re_common/baselibrary/utils/baseurl.py +121 -0
  100. re_common/baselibrary/utils/basezip.py +57 -0
  101. re_common/baselibrary/utils/core/__init__.py +8 -0
  102. re_common/baselibrary/utils/core/bottomutils.py +18 -0
  103. re_common/baselibrary/utils/core/mdeprecated.py +327 -0
  104. re_common/baselibrary/utils/core/mlamada.py +16 -0
  105. re_common/baselibrary/utils/core/msginfo.py +25 -0
  106. re_common/baselibrary/utils/core/requests_core.py +103 -0
  107. re_common/baselibrary/utils/fateadm.py +429 -0
  108. re_common/baselibrary/utils/importfun.py +123 -0
  109. re_common/baselibrary/utils/mfaker.py +57 -0
  110. re_common/baselibrary/utils/my_abc/__init__.py +3 -0
  111. re_common/baselibrary/utils/my_abc/better_abc.py +32 -0
  112. re_common/baselibrary/utils/mylogger.py +414 -0
  113. re_common/baselibrary/utils/myredisclient.py +861 -0
  114. re_common/baselibrary/utils/pipupgrade.py +21 -0
  115. re_common/baselibrary/utils/ringlist.py +85 -0
  116. re_common/baselibrary/utils/version_compare.py +36 -0
  117. re_common/baselibrary/utils/ydmhttp.py +126 -0
  118. re_common/facade/__init__.py +1 -0
  119. re_common/facade/lazy_import.py +11 -0
  120. re_common/facade/loggerfacade.py +25 -0
  121. re_common/facade/mysqlfacade.py +467 -0
  122. re_common/facade/now.py +31 -0
  123. re_common/facade/sqlite3facade.py +257 -0
  124. re_common/facade/use/__init__.py +0 -0
  125. re_common/facade/use/mq_use_facade.py +83 -0
  126. re_common/facade/use/proxy_use_facade.py +20 -0
  127. re_common/libtest/__init__.py +0 -0
  128. re_common/libtest/base_dict_test.py +19 -0
  129. re_common/libtest/baseavro_test.py +13 -0
  130. re_common/libtest/basefile_test.py +14 -0
  131. re_common/libtest/basemssql_test.py +77 -0
  132. re_common/libtest/baseodbc_test.py +8 -0
  133. re_common/libtest/basepandas_test.py +38 -0
  134. re_common/libtest/get_attr_test/__init__.py +0 -0
  135. re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -0
  136. re_common/libtest/get_attr_test/settings.py +55 -0
  137. re_common/libtest/idencode_test.py +54 -0
  138. re_common/libtest/iniconfig_test.py +35 -0
  139. re_common/libtest/ip_test.py +35 -0
  140. re_common/libtest/merge_file_test.py +20 -0
  141. re_common/libtest/mfaker_test.py +9 -0
  142. re_common/libtest/mm3_test.py +32 -0
  143. re_common/libtest/mylogger_test.py +89 -0
  144. re_common/libtest/myparsel_test.py +28 -0
  145. re_common/libtest/mysql_test.py +151 -0
  146. re_common/libtest/pymongo_test.py +21 -0
  147. re_common/libtest/split_test.py +12 -0
  148. re_common/libtest/sqlite3_merge_test.py +6 -0
  149. re_common/libtest/sqlite3_test.py +34 -0
  150. re_common/libtest/tomlconfig_test.py +30 -0
  151. re_common/libtest/use_tools_test/__init__.py +3 -0
  152. re_common/libtest/user/__init__.py +5 -0
  153. re_common/studio/__init__.py +5 -0
  154. re_common/studio/assignment_expressions.py +37 -0
  155. re_common/studio/mydash/__init__.py +0 -0
  156. re_common/studio/mydash/test1.py +19 -0
  157. re_common/studio/pydashstudio/__init__.py +0 -0
  158. re_common/studio/pydashstudio/first.py +9 -0
  159. re_common/studio/streamlitstudio/__init__.py +0 -0
  160. re_common/studio/streamlitstudio/first_app.py +66 -0
  161. re_common/studio/streamlitstudio/uber_pickups.py +24 -0
  162. re_common/studio/test.py +19 -0
  163. re_common/v2/baselibrary/utils/author_smi.py +14 -3
  164. re_common/v2/baselibrary/utils/stringutils.py +1 -0
  165. re_common/vip/__init__.py +0 -0
  166. re_common/vip/base_step_process.py +11 -0
  167. re_common/vip/baseencodeid.py +91 -0
  168. re_common/vip/changetaskname.py +28 -0
  169. re_common/vip/core_var.py +24 -0
  170. re_common/vip/mmh3Hash.py +90 -0
  171. re_common/vip/proxy/__init__.py +0 -0
  172. re_common/vip/proxy/allproxys.py +127 -0
  173. re_common/vip/proxy/allproxys_thread.py +159 -0
  174. re_common/vip/proxy/cnki_proxy.py +153 -0
  175. re_common/vip/proxy/kuaidaili.py +87 -0
  176. re_common/vip/proxy/proxy_all.py +113 -0
  177. re_common/vip/proxy/update_kuaidaili_0.py +42 -0
  178. re_common/vip/proxy/wanfang_proxy.py +152 -0
  179. re_common/vip/proxy/wp_proxy_all.py +182 -0
  180. re_common/vip/read_rawid_to_txt.py +92 -0
  181. re_common/vip/title/__init__.py +5 -0
  182. re_common/vip/title/transform/TransformBookTitleToZt.py +125 -0
  183. re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -0
  184. re_common/vip/title/transform/TransformCstadTitleToZt.py +196 -0
  185. re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -0
  186. re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -0
  187. re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -0
  188. re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -0
  189. re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -0
  190. re_common/vip/title/transform/__init__.py +11 -0
  191. {re_common-2.0.1.dist-info → re_common-10.0.1.dist-info}/METADATA +1 -1
  192. re_common-10.0.1.dist-info/RECORD +213 -0
  193. re_common-2.0.1.dist-info/RECORD +0 -25
  194. {re_common-2.0.1.dist-info → re_common-10.0.1.dist-info}/LICENSE +0 -0
  195. {re_common-2.0.1.dist-info → re_common-10.0.1.dist-info}/WHEEL +0 -0
  196. {re_common-2.0.1.dist-info → re_common-10.0.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,121 @@
1
+ import urllib.parse as parse
2
+
3
+ from re_common.baselibrary.utils.basestring import BaseString
4
+
5
+
6
+ class BaseUrl(BaseString):
7
+
8
+ @classmethod
9
+ def urlQuery2Dict(cls, url):
10
+ """
11
+ url中的参数转换成dict 字典类型方便读取
12
+ h还有另一种 方式
13
+ dict(parse.parse_qsl(parse.urlparse(url).query))
14
+ :param url:
15
+ :return:
16
+ """
17
+ query = parse.urlparse(url).query
18
+ return dict([(k, v[0]) for k, v in parse.parse_qs(query).items()])
19
+
20
+ @classmethod
21
+ def urlPath2List(cls, url):
22
+ """
23
+ 获取路由并返回一个列表
24
+ :param url:
25
+ :return:
26
+ """
27
+ listPath = parse.urlparse(url).path.split("/")
28
+ try:
29
+ listPath.remove("")
30
+ except:
31
+ pass
32
+ return listPath
33
+
34
+ @classmethod
35
+ def urlreplaceend(cls, url, endstrins):
36
+ list_url = url.split("/")
37
+ list_url.pop()
38
+ list_url.append(endstrins)
39
+ url = "/".join(list_url)
40
+ return url
41
+
42
+ @classmethod
43
+ def urlNetloc(cls, url):
44
+ """
45
+ 获取网址
46
+ :param url:
47
+ :return:
48
+ """
49
+ return parse.urlparse(url).netloc
50
+
51
+ @classmethod
52
+ def urlScheme(cls, url):
53
+ """
54
+ 获取网站的请求方式 http or https
55
+ :param url:
56
+ :return:
57
+ """
58
+ return parse.urlparse(url).scheme
59
+
60
+ @classmethod
61
+ def urlQuery2List(cls, url):
62
+ """
63
+ urllib.parse.parse_qs 返回字典
64
+ {'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
65
+ urllib.parse.parse_qsl 返回列表
66
+ [('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
67
+ :param url:
68
+ :return:
69
+ """
70
+ return parse.parse_qsl(parse.urlparse(url).query)
71
+
72
+ @classmethod
73
+ def urlQuery2dic(cls, url):
74
+ """
75
+ urllib.parse.parse_qs 返回字典
76
+ {'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
77
+ urllib.parse.parse_qsl 返回列表
78
+ [('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
79
+ :param url:
80
+ :return:
81
+ """
82
+ return parse.parse_qs(parse.urlparse(url).query)
83
+
84
+ @classmethod
85
+ def urlencode(cls, dic):
86
+ """
87
+ 将dic型的url参数进行编码 如
88
+ {'CID': 'quickSearchCitationFormat', 'database': '1', 'SEARCHID': 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259', 'intialSearch': 'true', 'angularReq': 'true', 'usageOrigin': 'searchform', 'usageZone': 'quicksearch', 'implicit': 'false', 'isFullJsonResult': 'true', 'startTime': '1529473818439', 'endTime': '1529473818477'}
89
+ :param dic:
90
+ :return:
91
+ """
92
+ return parse.urlencode(dic)
93
+
94
+ @classmethod
95
+ def urldecode(self, strings):
96
+ """
97
+ url解码
98
+ :param strings:
99
+ :return:
100
+ """
101
+ return parse.unquote(strings)
102
+
103
+ @classmethod
104
+ def dicts_to_url(cls, dic, url=None):
105
+ """
106
+ 将字典转换成get的url参数部分
107
+ :param dic:
108
+ :return:
109
+ """
110
+ parastring = ""
111
+ for key, value in dic.items():
112
+ parastring += str(key) + "=" + str(value) + "&"
113
+ parastring = parastring[:-1]
114
+ if url:
115
+ return url + "?" + parastring
116
+ return parastring
117
+
118
+ # url = "https://www.engineeringvillage.com/search/results/quick.url?CID=quickSearchCitationFormat&database=1&SEARCHID=f84e4d9aMa3f4M4e2aMae22M912f5e06a259&intialSearch=true&angularReq=true&usageOrigin=searchform&usageZone=quicksearch&implicit=false&isFullJsonResult=true&startTime=1529473818439&endTime=1529473818477"
119
+ # dic = BaseUrl.urlQuery2Dict(url)
120
+ # print(dic)
121
+ # BaseUrl.urlencode(url)
@@ -0,0 +1,57 @@
1
+ import zipfile
2
+
3
+
4
+ class BaseZIP(object):
5
+
6
+ @classmethod
7
+ def get_zip_file(cls, filepath):
8
+ azip = zipfile.ZipFile(filepath)
9
+ return azip
10
+
11
+ @classmethod
12
+ def get_zip_namelist(cls, zip_obj):
13
+ """
14
+ 返回zip的文件列表,包括深层次目录
15
+ :return: list
16
+ """
17
+ zip_list = zip_obj.namelist()
18
+ result_list = []
19
+ for zip_file in zip_list:
20
+ try:
21
+ zip_file = zip_file.encode('cp437').decode('gbk')
22
+ except:
23
+ zip_file = zip_file.encode('utf-8').decode('utf-8')
24
+ result_list.append(zip_file)
25
+ return result_list
26
+
27
+ @classmethod
28
+ def get_zipname(cls,zipobj):
29
+ """
30
+
31
+ :param zipobj:
32
+ :return: zip文件位置
33
+ """
34
+ return zipobj.filename
35
+
36
+ @classmethod
37
+ def get_info(cls,zipobj,filepath):
38
+ """
39
+ 获取文件的基本信息,注意,不是所有格式的文件都能获取信息
40
+ zipobj.getinfo(filepath)后可以获取以下信息
41
+ # 原来文件大小
42
+ print(azip_info.file_size)
43
+ # 压缩后大小
44
+ print(azip_info.compress_size)
45
+
46
+ # 这样可以求得压缩率,保留小数点后两位
47
+ print('压缩率为{:.2f}'.format(azip_info.file_size/azip_info.compress_size))
48
+ :param zipobj:
49
+ :param filepath: 为压缩文件下的相对目录 比如 test/test1/第二次目录/2021因工作被隔离明细表_采集组.xlsx
50
+ :return:
51
+ """
52
+ return zipobj.getinfo(filepath)
53
+
54
+
55
+ # zipobj = BaseZIP.get_zip_file(r'C:\Users\xuzhu\Desktop\VIPPatents_20210514To20210518.rar')
56
+ # print(BaseZIP.get_zip_namelist(zipobj))
57
+ #
@@ -0,0 +1,8 @@
1
+ # true 确定环境是python2
2
+ import sys
3
+
4
+ PY2 = sys.version_info[0] == 2
5
+ # true 确定环境是否是windows
6
+ WIN = sys.platform.startswith('win')
7
+
8
+ _identity = lambda x: x
@@ -0,0 +1,18 @@
1
+ # 储存单位
2
+ import os
3
+ import platform
4
+ import re
5
+
6
+ # 文件大小单位
7
+
8
+ SUFFIX = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
9
+ # 系统的分割符
10
+ __os_sep__ = "\\" if platform.system() == 'Windows' else os.sep
11
+ # 标准文件名正着
12
+ _filename_ascii_strip_re = re.compile(r'[^A-Za-z0-9_.-]')
13
+ # windows的设备文件
14
+ _windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', 'COM4', 'LPT1',
15
+ 'LPT2', 'LPT3', 'PRN', 'NUL')
16
+
17
+
18
+
@@ -0,0 +1,327 @@
1
+ import asyncio
2
+ import functools
3
+ import sys
4
+ import time
5
+ import traceback
6
+ import warnings
7
+ from functools import wraps
8
+
9
+ # https://www.jianshu.com/p/ee82b941772a
10
+ import aiohttp.client_exceptions
11
+ import requests
12
+ from aiohttp import ClientProxyConnectionError
13
+ from google.protobuf.message import DecodeError
14
+
15
+ from re_common.baselibrary.utils.core.requests_core import MsgCode
16
+
17
+
18
+ def InterVal(start_time, interval):
19
+ '''
20
+ 执行函数必须间隔多少时间,没有到指定的时间函数就不会被执行
21
+ :param start_time: 传入开始时间
22
+ :param interval: 秒数
23
+ :return: 返回执行结果 否则返回 None
24
+ '''
25
+
26
+ def dewrapper(func):
27
+ """
28
+ 它能把原函数的元信息拷贝到装饰器里面的 func 函数中。
29
+ 函数的元信息包括docstring、name、参数列表等等。可以尝试去除@functools.wraps(func),
30
+ 你会发现test.__name__的输出变成了wrapper。
31
+ """
32
+
33
+ @wraps(func)
34
+ def wrapper(*args, **kwargs):
35
+ start = start_time
36
+ time_now = int(time.time())
37
+ if time_now - start > interval:
38
+ result = func(*args, **kwargs)
39
+ return result
40
+ else:
41
+ return None
42
+
43
+ return wrapper
44
+
45
+ return dewrapper
46
+
47
+
48
+ def timethis(func):
49
+ """
50
+ 函数执行的时间差
51
+ :param func:
52
+ :return:
53
+ """
54
+
55
+ @wraps(func)
56
+ def wrapper(*args, **kwargs):
57
+ start = time.time()
58
+ result = func(*args, **kwargs)
59
+ end = time.time()
60
+ print(func.__name__, end - start)
61
+ return result
62
+
63
+ return wrapper
64
+
65
+
66
+ def func_time(callback=None, is_print=True):
67
+ """
68
+ 装饰器获取时间
69
+ :param func:
70
+ :return:
71
+ """
72
+
73
+ def dewrapper(func):
74
+ @functools.wraps(func)
75
+ async def wrapper(*args, **kwargs):
76
+ start = time.time()
77
+ result = await func(*args, **kwargs)
78
+ end = time.time()
79
+ if is_print:
80
+ print(func.__name__, end - start)
81
+ if callback is not None:
82
+ await callback(start, end)
83
+ return result
84
+
85
+ return wrapper
86
+
87
+ return dewrapper
88
+
89
+
90
+ def try_except(callback=None, is_print=True):
91
+ """
92
+ 使用装饰器使用try_except,并用callback回收错误信息
93
+ :param func:
94
+ :return:
95
+ """
96
+
97
+ def dewrapper(func):
98
+
99
+ @wraps(func)
100
+ def wrapper(*args, **kwargs):
101
+ try:
102
+ return func(*args, **kwargs)
103
+ except BaseException as e:
104
+ if callback is not None:
105
+ callback(*sys.exc_info(), *args, **kwargs)
106
+ if is_print:
107
+ print(traceback.format_exc())
108
+ return traceback.format_exc()
109
+
110
+ return wrapper
111
+
112
+ return dewrapper
113
+
114
+
115
+ def try_except2(callback=None, is_print=True):
116
+ """
117
+ 使用装饰器使用try_except,并用callback回收错误信息
118
+ :param func:
119
+ :return:
120
+ """
121
+
122
+ def dewrapper(func):
123
+
124
+ @wraps(func)
125
+ def wrapper(*args, **kwargs):
126
+ try:
127
+ return func(*args, **kwargs)
128
+ except BaseException as e:
129
+ if callback is not None:
130
+ callback(*sys.exc_info(), *args, **kwargs)
131
+ if is_print:
132
+ print(traceback.format_exc())
133
+ return False, {"traceback": traceback.format_exc()}
134
+
135
+ return wrapper
136
+
137
+ return dewrapper
138
+
139
+
140
+ def try_except2_async(callback=None, is_print=True):
141
+ """
142
+ 使用装饰器使用try_except,并用callback回收错误信息
143
+ 异步函数专用装饰器
144
+ :param func:
145
+ :return:
146
+ """
147
+
148
+ def dewrapper(func):
149
+
150
+ @functools.wraps(func)
151
+ async def wrapper(*args, **kwargs):
152
+ try:
153
+ return await func(*args, **kwargs)
154
+ except BaseException as e:
155
+ if callback is not None:
156
+ if is_print:
157
+ print("traceback=>", traceback.format_exc())
158
+ bools, one_dic = await callback(*sys.exc_info(), *args, **kwargs)
159
+ return bools, one_dic
160
+
161
+ return False, {"traceback": traceback.format_exc()}
162
+
163
+ return wrapper
164
+
165
+ return dewrapper
166
+
167
+
168
+ # https://blog.csdn.net/qq_39314099/article/details/83822593
169
+ def deprecated(message):
170
+ def deprecated_decorator(func):
171
+ @wraps(func)
172
+ def deprecated_func(*args, **kwargs):
173
+ warnings.warn("{} is a deprecated function. {}".format(func.__name__, message),
174
+ category=DeprecationWarning,
175
+ stacklevel=2)
176
+ warnings.simplefilter('default', DeprecationWarning)
177
+ return func(*args, **kwargs)
178
+
179
+ return deprecated_func
180
+
181
+ return deprecated_decorator
182
+
183
+
184
+ def request_try_except(func):
185
+ """
186
+ 拦截request请求错误
187
+ :param func:
188
+ :return:
189
+ """
190
+
191
+ @wraps(func)
192
+ def wrapTheFunction(*args, **kwargs):
193
+ try:
194
+ return func(*args, **kwargs)
195
+ except requests.exceptions.ReadTimeout as e:
196
+ try:
197
+ funcname = func.__name__
198
+ except:
199
+ funcname = "错误,获取func的name失败"
200
+ dicts = {"code": MsgCode.TIME_OUT_ERROR,
201
+ "msg": "time out, {},{}".format(repr(e), funcname)}
202
+ return False, dicts
203
+ except requests.exceptions.ProxyError as e:
204
+ dicts = {"code": MsgCode.PROXY_ERROR,
205
+ "msg": "proxy error, {}".format(repr(e))}
206
+ return False, dicts
207
+ except:
208
+ dicts = {"code": MsgCode.ON_KNOW,
209
+ "msg": traceback.format_exc()}
210
+ return False, dicts
211
+
212
+ return wrapTheFunction
213
+
214
+
215
+ def aiohttp_try_except(func):
216
+ """
217
+ 拦截aiohttp 异步请求错误
218
+ :param func:
219
+ :return:
220
+ """
221
+
222
+ @wraps(func)
223
+ async def wrapTheFunction(*args, **kwargs):
224
+ try:
225
+ return await func(*args, **kwargs)
226
+ except ClientProxyConnectionError as e:
227
+ dicts = {"code": MsgCode.PROXY_ERROR,
228
+ "msg": "proxy error, {}".format(repr(e))}
229
+ return False, dicts
230
+ except asyncio.exceptions.TimeoutError as e:
231
+ try:
232
+ funcname = func.__name__
233
+ except:
234
+ funcname = "错误,获取func的name失败"
235
+ dicts = {"code": MsgCode.TIME_OUT_ERROR,
236
+ "msg": "time out, {},{}".format(repr(e), funcname)}
237
+ return False, dicts
238
+ except aiohttp.client_exceptions.ClientPayloadError as e:
239
+ dicts = {"code": MsgCode.PAYLOAD_ERROR,
240
+ "msg": "非法的压缩格式,错误的chunk编码,数据不足Content-length, {}".format(repr(e))}
241
+ return False, dicts
242
+ except DecodeError as e:
243
+
244
+ dicts = {"code": MsgCode.GRPC_MESSAGE_DECODEERROR,
245
+ "msg": "grpc message 消息结构错误 {}".format(repr(e))}
246
+ return False, dicts
247
+ except Exception as e:
248
+ dicts = {"code": MsgCode.ON_KNOW,
249
+ "msg": traceback.format_exc()}
250
+ return False, dicts
251
+
252
+ return wrapTheFunction
253
+
254
+
255
+ def mongodb_try_except(func):
256
+ """
257
+ 拦截mongodb执行错误
258
+ :param func:
259
+ :return:
260
+ """
261
+
262
+ @wraps(func)
263
+ async def wrapTheFunction(*args, **kwargs):
264
+ try:
265
+ return await func(*args, **kwargs)
266
+ except Exception as e:
267
+ dict_ = {
268
+ "status": "FAILED",
269
+ "msg_code": MsgCode.NO_RESOURCE,
270
+ "msg": "保存到数据库失败",
271
+ "data": {"err": repr(e)}
272
+ }
273
+ return False, dict_
274
+
275
+ return wrapTheFunction
276
+
277
+
278
+ def retry_func(retry_times=3, sleep_time=1):
279
+ """函数重复执行次数"""
280
+
281
+ def retry_decorator(func):
282
+ @functools.wraps(func)
283
+ def wrapper_func(*args, **kwargs):
284
+ flag = 0
285
+ while flag < retry_times:
286
+ res = func(*args, **kwargs)
287
+ res_bool = res
288
+ if isinstance(res, tuple):
289
+ res_bool = res[0]
290
+ if not res_bool:
291
+ flag += 1
292
+ # logger.warning("{0} execute {1} times".format((func.__name__), flag))
293
+ time.sleep(sleep_time)
294
+ continue
295
+ else:
296
+ return res
297
+ return res
298
+ return wrapper_func
299
+
300
+ return retry_decorator
301
+
302
+
303
+ def retry_func_async(retry_times=3, sleep_time=1):
304
+ """函数重复执行次数"""
305
+
306
+ def retry_decorator(func):
307
+ @functools.wraps(func)
308
+ async def wrapper_func(*args, **kwargs):
309
+ flag = 0
310
+ res = (False, {})
311
+ while flag < retry_times:
312
+ res = await func(*args, **kwargs)
313
+ res_bool = res
314
+ if isinstance(res, tuple):
315
+ res_bool = res[0]
316
+ if not res_bool:
317
+ flag += 1
318
+ # logger.warning("{0} execute {1} times".format((func.__name__), flag))
319
+ await asyncio.sleep(sleep_time)
320
+ continue
321
+ else:
322
+ return res
323
+ return res
324
+
325
+ return wrapper_func
326
+
327
+ return retry_decorator
@@ -0,0 +1,16 @@
1
+ # 通过传入的bool值确定是否打印string
2
+ from re_common.baselibrary.utils.core.requests_core import SUCCESS, FAILED
3
+
4
+ printfunc = lambda LogBool, string: print(string) if LogBool else print("")
5
+
6
+ # 关闭r对象
7
+ closeResult = lambda r: r.close() if 'r' in locals() and r else None
8
+
9
+ # bools 传入True or False 给出对应的字符串
10
+ bools_string = lambda bools: SUCCESS if bools else FAILED
11
+
12
+ # 判断是否存在并strip()
13
+ html_strip = lambda html: html.strip() if html else html
14
+
15
+ # 根据给的字符串确认是否空,如果x == "" 给y值,否则如果x=="delete" 给"",否则给x的值
16
+ # set_value = lambda x, y: y if x == "" else "" if x == "delete" else x
@@ -0,0 +1,25 @@
1
+ class MsgInfo(object):
2
+ """
3
+ 消息体结构,没有想好如何使用
4
+ """
5
+
6
+ def __init__(self, code, msg, traceback, info_dict=None):
7
+ self.code = code # 状态码
8
+ self.msg = msg
9
+ self.traceback = traceback
10
+ if info_dict is None:
11
+ info_dict = {}
12
+ self.info_dict = info_dict
13
+ # 多个消息历史的列表
14
+ self.msg_list = []
15
+
16
+ def msg_lists(self):
17
+ """
18
+ 将消息结构list存放,而不是嵌套
19
+ :return:
20
+ """
21
+ self.msg_list.append({"code": self.code,
22
+ "msg": self.msg,
23
+ "traceback": self.traceback,
24
+ "info_dict": self.info_dict})
25
+ return self
@@ -0,0 +1,103 @@
1
+ USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
2
+
3
+ USER_AGENTS = [
4
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
5
+ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
6
+ "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
7
+ "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
8
+ "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
9
+ "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
10
+ "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
11
+ "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
12
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
13
+ "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
14
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
15
+ "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
16
+ "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
17
+ "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
18
+ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
19
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
20
+ "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
21
+ ]
22
+
23
+ INSIDE_HEADERS = {
24
+ "accept": "application/json",
25
+ "content-type": "application/json"
26
+ }
27
+
28
+
29
+ def set_proxy(proxy):
30
+ Proxiesss = {
31
+ 'http': proxy,
32
+ 'https': proxy
33
+ }
34
+ return Proxiesss
35
+
36
+
37
+ def set_proxy_aio(proxy):
38
+ if proxy:
39
+ return r"http://" + proxy
40
+ return None
41
+
42
+
43
+ def set_proxy_httpx(proxy):
44
+ if proxy.find("socks") > -1:
45
+ return proxy
46
+ if proxy:
47
+ Proxiesss = {
48
+ 'http://': "http://" + proxy,
49
+ 'https://': "http://" + proxy
50
+ }
51
+ return Proxiesss
52
+ else:
53
+ return None
54
+
55
+
56
+ class MsgCode(object):
57
+ SUCCESS_CODE: int = 200 # 成功状态码
58
+
59
+ NO_RESOURCE: int = 421 # 网页明确表明无资源
60
+ PAGE_BLANK: int = 420 # 网页空白页
61
+ END_STRING_ERROR: int = 210 # html结尾验证错误
62
+ MARK_ERROR: int = 211 # 验证html错误,验证没有通过
63
+ NOT_IS_JSON: int = 212 # 验证返回是否是json
64
+ VER_CODE: int = 213 # 需要验证码
65
+ NONE_HTML: int = 214 # html为空 空的html 在使用某些代理时 如果https 写成了http,会有这种情况
66
+ STATUS_ERROR: int = 215 # API接口内部执行错误
67
+ CODE_ERROR: int = 216 # data 状态码错误
68
+ URL_TIMEOUT: int = 217
69
+
70
+ SETTING_CONFIG_ERROR: int = 251 # 特定错误 更新taskinfo_save_setting的任务控制配置失败(switch参数错误,只能为0或1)
71
+
72
+ API_FAIL_CODE: int = 400 # api失败,但具体原因未知的状态码
73
+ TIME_OUT_ERROR: int = 408 # 超时错误
74
+ PROXY_ERROR: int = 422 # 代理错误
75
+ PAYLOAD_ERROR: int = 423 # 非法的压缩格式,错误的chunk编码,数据不足Content-length的大小。通常为Accept-Encoding:gzip, deflate, br ,也就是三种编码格式。但其实,aiohttp默认没有br解码,或者html编码格式错误
76
+ SERVER_ERROR: int = 530 # 服务器异常错误,被捕获发送
77
+ CHAOXIN_COOKIE_ERROR: int = 556 # 超星期刊获取cookie失败
78
+
79
+ DATABASE_CONFIG_IN_CODE: int = 998 # 在代码中配置数据库无对应错误码
80
+
81
+ ON_KNOW: int = 1000 # 未知错误
82
+
83
+ MONGO_ERROR: int = 1001 # mongodb 錯誤系列由1000 到 1100
84
+ MONGO_NO_ID: int = 1002 # 操作时由于没有该id的状态码,比如更新时ID不存在
85
+
86
+ PARE_STRUCTURE_ERROR: int = 1101 # 解析系列由1100 到 1200,解析结构错误
87
+ PARE_NO_DATA: int = 1102 # 没有数据,且认为是正常的没有数据的标识
88
+ PARE_NO_DATA_ERR: int = 1103 # 没有数据,且认为是非正常的没有数据的标识
89
+ PARE_NO_DATA_1: int = 1104 # 数据不全,但可以接受
90
+ PARE_NO_DATA_2: int = 1105 # 预留几个无数据,在不同库不同意义
91
+ PARE_NO_DATA_3: int = 1106 # 预留几个无数据,在不同库不同意义
92
+ SQL_INSERT_ERROR: int = 1201 # sql错误 由 1200 到 1300 insert 错误
93
+ SQL_UPDATE_ERROR: int = 1202 # update 错误
94
+ SQL_SELECT_ALL_ERROR: int = 1203 # 执行fetchall 错误
95
+ SQL_EXECUTEMANY_ERROR: int = 1204 # executemany 执行错误
96
+ SQL_EXECUTE_ERROR: int = 1205 # execute 执行错误
97
+ SQL_REPLACE_ERROR: int = 1206 # replace 执行错误
98
+
99
+ GRPC_MESSAGE_DECODEERROR: int = 1301 # grpc服务编码错误
100
+
101
+
102
+ SUCCESS = "SUCCESS"
103
+ FAILED = "FAILED"