re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +235 -220
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +497 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/tree_processor/__init__.py +0 -0
- re_common/v2/baselibrary/tools/tree_processor/builder.py +25 -0
- re_common/v2/baselibrary/tools/tree_processor/node.py +13 -0
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +94 -77
- re_common/v2/baselibrary/utils/db.py +174 -156
- re_common/v2/baselibrary/utils/elasticsearch.py +46 -0
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +187 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +312 -271
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.39.dist-info → re_common-10.0.41.dist-info}/LICENSE +201 -201
- {re_common-10.0.39.dist-info → re_common-10.0.41.dist-info}/METADATA +16 -16
- re_common-10.0.41.dist-info/RECORD +252 -0
- {re_common-10.0.39.dist-info → re_common-10.0.41.dist-info}/WHEEL +1 -1
- re_common-10.0.39.dist-info/RECORD +0 -248
- {re_common-10.0.39.dist-info → re_common-10.0.41.dist-info}/top_level.txt +0 -0
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
# 通过传入的bool值确定是否打印string
|
|
2
|
-
from re_common.baselibrary.utils.core.requests_core import SUCCESS, FAILED
|
|
3
|
-
|
|
4
|
-
printfunc = lambda LogBool, string: print(string) if LogBool else print("")
|
|
5
|
-
|
|
6
|
-
# 关闭r对象
|
|
7
|
-
closeResult = lambda r: r.close() if 'r' in locals() and r else None
|
|
8
|
-
|
|
9
|
-
# bools 传入True or False 给出对应的字符串
|
|
10
|
-
bools_string = lambda bools: SUCCESS if bools else FAILED
|
|
11
|
-
|
|
12
|
-
# 判断是否存在并strip()
|
|
13
|
-
html_strip = lambda html: html.strip() if html else html
|
|
14
|
-
|
|
15
|
-
# 根据给的字符串确认是否空,如果x == "" 给y值,否则如果x=="delete" 给"",否则给x的值
|
|
16
|
-
# set_value = lambda x, y: y if x == "" else "" if x == "delete" else x
|
|
1
|
+
# 通过传入的bool值确定是否打印string
|
|
2
|
+
from re_common.baselibrary.utils.core.requests_core import SUCCESS, FAILED
|
|
3
|
+
|
|
4
|
+
printfunc = lambda LogBool, string: print(string) if LogBool else print("")
|
|
5
|
+
|
|
6
|
+
# 关闭r对象
|
|
7
|
+
closeResult = lambda r: r.close() if 'r' in locals() and r else None
|
|
8
|
+
|
|
9
|
+
# bools 传入True or False 给出对应的字符串
|
|
10
|
+
bools_string = lambda bools: SUCCESS if bools else FAILED
|
|
11
|
+
|
|
12
|
+
# 判断是否存在并strip()
|
|
13
|
+
html_strip = lambda html: html.strip() if html else html
|
|
14
|
+
|
|
15
|
+
# 根据给的字符串确认是否空,如果x == "" 给y值,否则如果x=="delete" 给"",否则给x的值
|
|
16
|
+
# set_value = lambda x, y: y if x == "" else "" if x == "delete" else x
|
|
@@ -1,25 +1,25 @@
|
|
|
1
|
-
class MsgInfo(object):
|
|
2
|
-
"""
|
|
3
|
-
消息体结构,没有想好如何使用
|
|
4
|
-
"""
|
|
5
|
-
|
|
6
|
-
def __init__(self, code, msg, traceback, info_dict=None):
|
|
7
|
-
self.code = code # 状态码
|
|
8
|
-
self.msg = msg
|
|
9
|
-
self.traceback = traceback
|
|
10
|
-
if info_dict is None:
|
|
11
|
-
info_dict = {}
|
|
12
|
-
self.info_dict = info_dict
|
|
13
|
-
# 多个消息历史的列表
|
|
14
|
-
self.msg_list = []
|
|
15
|
-
|
|
16
|
-
def msg_lists(self):
|
|
17
|
-
"""
|
|
18
|
-
将消息结构list存放,而不是嵌套
|
|
19
|
-
:return:
|
|
20
|
-
"""
|
|
21
|
-
self.msg_list.append({"code": self.code,
|
|
22
|
-
"msg": self.msg,
|
|
23
|
-
"traceback": self.traceback,
|
|
24
|
-
"info_dict": self.info_dict})
|
|
25
|
-
return self
|
|
1
|
+
class MsgInfo(object):
|
|
2
|
+
"""
|
|
3
|
+
消息体结构,没有想好如何使用
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
def __init__(self, code, msg, traceback, info_dict=None):
|
|
7
|
+
self.code = code # 状态码
|
|
8
|
+
self.msg = msg
|
|
9
|
+
self.traceback = traceback
|
|
10
|
+
if info_dict is None:
|
|
11
|
+
info_dict = {}
|
|
12
|
+
self.info_dict = info_dict
|
|
13
|
+
# 多个消息历史的列表
|
|
14
|
+
self.msg_list = []
|
|
15
|
+
|
|
16
|
+
def msg_lists(self):
|
|
17
|
+
"""
|
|
18
|
+
将消息结构list存放,而不是嵌套
|
|
19
|
+
:return:
|
|
20
|
+
"""
|
|
21
|
+
self.msg_list.append({"code": self.code,
|
|
22
|
+
"msg": self.msg,
|
|
23
|
+
"traceback": self.traceback,
|
|
24
|
+
"info_dict": self.info_dict})
|
|
25
|
+
return self
|
|
@@ -1,103 +1,103 @@
|
|
|
1
|
-
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
|
|
2
|
-
|
|
3
|
-
USER_AGENTS = [
|
|
4
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
|
|
5
|
-
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
|
6
|
-
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
|
7
|
-
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
|
8
|
-
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
|
9
|
-
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
|
10
|
-
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
|
11
|
-
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
|
12
|
-
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
|
13
|
-
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
|
14
|
-
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
|
15
|
-
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
|
16
|
-
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
|
17
|
-
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
|
18
|
-
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
|
19
|
-
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
|
20
|
-
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
|
21
|
-
]
|
|
22
|
-
|
|
23
|
-
INSIDE_HEADERS = {
|
|
24
|
-
"accept": "application/json",
|
|
25
|
-
"content-type": "application/json"
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def set_proxy(proxy):
|
|
30
|
-
Proxiesss = {
|
|
31
|
-
'http': proxy,
|
|
32
|
-
'https': proxy
|
|
33
|
-
}
|
|
34
|
-
return Proxiesss
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def set_proxy_aio(proxy):
|
|
38
|
-
if proxy:
|
|
39
|
-
return r"http://" + proxy
|
|
40
|
-
return None
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def set_proxy_httpx(proxy):
|
|
44
|
-
if proxy.find("socks") > -1:
|
|
45
|
-
return proxy
|
|
46
|
-
if proxy:
|
|
47
|
-
Proxiesss = {
|
|
48
|
-
'http://': "http://" + proxy,
|
|
49
|
-
'https://': "http://" + proxy
|
|
50
|
-
}
|
|
51
|
-
return Proxiesss
|
|
52
|
-
else:
|
|
53
|
-
return None
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
class MsgCode(object):
|
|
57
|
-
SUCCESS_CODE: int = 200 # 成功状态码
|
|
58
|
-
|
|
59
|
-
NO_RESOURCE: int = 421 # 网页明确表明无资源
|
|
60
|
-
PAGE_BLANK: int = 420 # 网页空白页
|
|
61
|
-
END_STRING_ERROR: int = 210 # html结尾验证错误
|
|
62
|
-
MARK_ERROR: int = 211 # 验证html错误,验证没有通过
|
|
63
|
-
NOT_IS_JSON: int = 212 # 验证返回是否是json
|
|
64
|
-
VER_CODE: int = 213 # 需要验证码
|
|
65
|
-
NONE_HTML: int = 214 # html为空 空的html 在使用某些代理时 如果https 写成了http,会有这种情况
|
|
66
|
-
STATUS_ERROR: int = 215 # API接口内部执行错误
|
|
67
|
-
CODE_ERROR: int = 216 # data 状态码错误
|
|
68
|
-
URL_TIMEOUT: int = 217
|
|
69
|
-
|
|
70
|
-
SETTING_CONFIG_ERROR: int = 251 # 特定错误 更新taskinfo_save_setting的任务控制配置失败(switch参数错误,只能为0或1)
|
|
71
|
-
|
|
72
|
-
API_FAIL_CODE: int = 400 # api失败,但具体原因未知的状态码
|
|
73
|
-
TIME_OUT_ERROR: int = 408 # 超时错误
|
|
74
|
-
PROXY_ERROR: int = 422 # 代理错误
|
|
75
|
-
PAYLOAD_ERROR: int = 423 # 非法的压缩格式,错误的chunk编码,数据不足Content-length的大小。通常为Accept-Encoding:gzip, deflate, br ,也就是三种编码格式。但其实,aiohttp默认没有br解码,或者html编码格式错误
|
|
76
|
-
SERVER_ERROR: int = 530 # 服务器异常错误,被捕获发送
|
|
77
|
-
CHAOXIN_COOKIE_ERROR: int = 556 # 超星期刊获取cookie失败
|
|
78
|
-
|
|
79
|
-
DATABASE_CONFIG_IN_CODE: int = 998 # 在代码中配置数据库无对应错误码
|
|
80
|
-
|
|
81
|
-
ON_KNOW: int = 1000 # 未知错误
|
|
82
|
-
|
|
83
|
-
MONGO_ERROR: int = 1001 # mongodb 錯誤系列由1000 到 1100
|
|
84
|
-
MONGO_NO_ID: int = 1002 # 操作时由于没有该id的状态码,比如更新时ID不存在
|
|
85
|
-
|
|
86
|
-
PARE_STRUCTURE_ERROR: int = 1101 # 解析系列由1100 到 1200,解析结构错误
|
|
87
|
-
PARE_NO_DATA: int = 1102 # 没有数据,且认为是正常的没有数据的标识
|
|
88
|
-
PARE_NO_DATA_ERR: int = 1103 # 没有数据,且认为是非正常的没有数据的标识
|
|
89
|
-
PARE_NO_DATA_1: int = 1104 # 数据不全,但可以接受
|
|
90
|
-
PARE_NO_DATA_2: int = 1105 # 预留几个无数据,在不同库不同意义
|
|
91
|
-
PARE_NO_DATA_3: int = 1106 # 预留几个无数据,在不同库不同意义
|
|
92
|
-
SQL_INSERT_ERROR: int = 1201 # sql错误 由 1200 到 1300 insert 错误
|
|
93
|
-
SQL_UPDATE_ERROR: int = 1202 # update 错误
|
|
94
|
-
SQL_SELECT_ALL_ERROR: int = 1203 # 执行fetchall 错误
|
|
95
|
-
SQL_EXECUTEMANY_ERROR: int = 1204 # executemany 执行错误
|
|
96
|
-
SQL_EXECUTE_ERROR: int = 1205 # execute 执行错误
|
|
97
|
-
SQL_REPLACE_ERROR: int = 1206 # replace 执行错误
|
|
98
|
-
|
|
99
|
-
GRPC_MESSAGE_DECODEERROR: int = 1301 # grpc服务编码错误
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
SUCCESS = "SUCCESS"
|
|
103
|
-
FAILED = "FAILED"
|
|
1
|
+
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36'
|
|
2
|
+
|
|
3
|
+
USER_AGENTS = [
|
|
4
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36",
|
|
5
|
+
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
|
6
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
|
|
7
|
+
"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
|
|
8
|
+
"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
|
|
9
|
+
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
|
|
10
|
+
"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
|
|
11
|
+
"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
|
|
12
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
|
|
13
|
+
"Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
|
|
14
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
|
|
15
|
+
"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
|
|
16
|
+
"Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
|
|
17
|
+
"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
|
|
18
|
+
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
|
|
19
|
+
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
|
|
20
|
+
"Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
INSIDE_HEADERS = {
|
|
24
|
+
"accept": "application/json",
|
|
25
|
+
"content-type": "application/json"
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def set_proxy(proxy):
|
|
30
|
+
Proxiesss = {
|
|
31
|
+
'http': proxy,
|
|
32
|
+
'https': proxy
|
|
33
|
+
}
|
|
34
|
+
return Proxiesss
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def set_proxy_aio(proxy):
|
|
38
|
+
if proxy:
|
|
39
|
+
return r"http://" + proxy
|
|
40
|
+
return None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def set_proxy_httpx(proxy):
|
|
44
|
+
if proxy.find("socks") > -1:
|
|
45
|
+
return proxy
|
|
46
|
+
if proxy:
|
|
47
|
+
Proxiesss = {
|
|
48
|
+
'http://': "http://" + proxy,
|
|
49
|
+
'https://': "http://" + proxy
|
|
50
|
+
}
|
|
51
|
+
return Proxiesss
|
|
52
|
+
else:
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class MsgCode(object):
|
|
57
|
+
SUCCESS_CODE: int = 200 # 成功状态码
|
|
58
|
+
|
|
59
|
+
NO_RESOURCE: int = 421 # 网页明确表明无资源
|
|
60
|
+
PAGE_BLANK: int = 420 # 网页空白页
|
|
61
|
+
END_STRING_ERROR: int = 210 # html结尾验证错误
|
|
62
|
+
MARK_ERROR: int = 211 # 验证html错误,验证没有通过
|
|
63
|
+
NOT_IS_JSON: int = 212 # 验证返回是否是json
|
|
64
|
+
VER_CODE: int = 213 # 需要验证码
|
|
65
|
+
NONE_HTML: int = 214 # html为空 空的html 在使用某些代理时 如果https 写成了http,会有这种情况
|
|
66
|
+
STATUS_ERROR: int = 215 # API接口内部执行错误
|
|
67
|
+
CODE_ERROR: int = 216 # data 状态码错误
|
|
68
|
+
URL_TIMEOUT: int = 217
|
|
69
|
+
|
|
70
|
+
SETTING_CONFIG_ERROR: int = 251 # 特定错误 更新taskinfo_save_setting的任务控制配置失败(switch参数错误,只能为0或1)
|
|
71
|
+
|
|
72
|
+
API_FAIL_CODE: int = 400 # api失败,但具体原因未知的状态码
|
|
73
|
+
TIME_OUT_ERROR: int = 408 # 超时错误
|
|
74
|
+
PROXY_ERROR: int = 422 # 代理错误
|
|
75
|
+
PAYLOAD_ERROR: int = 423 # 非法的压缩格式,错误的chunk编码,数据不足Content-length的大小。通常为Accept-Encoding:gzip, deflate, br ,也就是三种编码格式。但其实,aiohttp默认没有br解码,或者html编码格式错误
|
|
76
|
+
SERVER_ERROR: int = 530 # 服务器异常错误,被捕获发送
|
|
77
|
+
CHAOXIN_COOKIE_ERROR: int = 556 # 超星期刊获取cookie失败
|
|
78
|
+
|
|
79
|
+
DATABASE_CONFIG_IN_CODE: int = 998 # 在代码中配置数据库无对应错误码
|
|
80
|
+
|
|
81
|
+
ON_KNOW: int = 1000 # 未知错误
|
|
82
|
+
|
|
83
|
+
MONGO_ERROR: int = 1001 # mongodb 錯誤系列由1000 到 1100
|
|
84
|
+
MONGO_NO_ID: int = 1002 # 操作时由于没有该id的状态码,比如更新时ID不存在
|
|
85
|
+
|
|
86
|
+
PARE_STRUCTURE_ERROR: int = 1101 # 解析系列由1100 到 1200,解析结构错误
|
|
87
|
+
PARE_NO_DATA: int = 1102 # 没有数据,且认为是正常的没有数据的标识
|
|
88
|
+
PARE_NO_DATA_ERR: int = 1103 # 没有数据,且认为是非正常的没有数据的标识
|
|
89
|
+
PARE_NO_DATA_1: int = 1104 # 数据不全,但可以接受
|
|
90
|
+
PARE_NO_DATA_2: int = 1105 # 预留几个无数据,在不同库不同意义
|
|
91
|
+
PARE_NO_DATA_3: int = 1106 # 预留几个无数据,在不同库不同意义
|
|
92
|
+
SQL_INSERT_ERROR: int = 1201 # sql错误 由 1200 到 1300 insert 错误
|
|
93
|
+
SQL_UPDATE_ERROR: int = 1202 # update 错误
|
|
94
|
+
SQL_SELECT_ALL_ERROR: int = 1203 # 执行fetchall 错误
|
|
95
|
+
SQL_EXECUTEMANY_ERROR: int = 1204 # executemany 执行错误
|
|
96
|
+
SQL_EXECUTE_ERROR: int = 1205 # execute 执行错误
|
|
97
|
+
SQL_REPLACE_ERROR: int = 1206 # replace 执行错误
|
|
98
|
+
|
|
99
|
+
GRPC_MESSAGE_DECODEERROR: int = 1301 # grpc服务编码错误
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
SUCCESS = "SUCCESS"
|
|
103
|
+
FAILED = "FAILED"
|