re-common 10.0.39__py3-none-any.whl → 10.0.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +77 -77
- re_common/v2/baselibrary/utils/db.py +156 -156
- re_common/v2/baselibrary/utils/elasticsearch.py +46 -0
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +186 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +271 -271
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/LICENSE +201 -201
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/METADATA +24 -16
- re_common-10.0.40.dist-info/RECORD +249 -0
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/WHEEL +1 -1
- re_common-10.0.39.dist-info/RECORD +0 -248
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/top_level.txt +0 -0
|
@@ -1,121 +1,121 @@
|
|
|
1
|
-
import urllib.parse as parse
|
|
2
|
-
|
|
3
|
-
from re_common.baselibrary.utils.basestring import BaseString
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
class BaseUrl(BaseString):
|
|
7
|
-
|
|
8
|
-
@classmethod
|
|
9
|
-
def urlQuery2Dict(cls, url):
|
|
10
|
-
"""
|
|
11
|
-
url中的参数转换成dict 字典类型方便读取
|
|
12
|
-
h还有另一种 方式
|
|
13
|
-
dict(parse.parse_qsl(parse.urlparse(url).query))
|
|
14
|
-
:param url:
|
|
15
|
-
:return:
|
|
16
|
-
"""
|
|
17
|
-
query = parse.urlparse(url).query
|
|
18
|
-
return dict([(k, v[0]) for k, v in parse.parse_qs(query).items()])
|
|
19
|
-
|
|
20
|
-
@classmethod
|
|
21
|
-
def urlPath2List(cls, url):
|
|
22
|
-
"""
|
|
23
|
-
获取路由并返回一个列表
|
|
24
|
-
:param url:
|
|
25
|
-
:return:
|
|
26
|
-
"""
|
|
27
|
-
listPath = parse.urlparse(url).path.split("/")
|
|
28
|
-
try:
|
|
29
|
-
listPath.remove("")
|
|
30
|
-
except:
|
|
31
|
-
pass
|
|
32
|
-
return listPath
|
|
33
|
-
|
|
34
|
-
@classmethod
|
|
35
|
-
def urlreplaceend(cls, url, endstrins):
|
|
36
|
-
list_url = url.split("/")
|
|
37
|
-
list_url.pop()
|
|
38
|
-
list_url.append(endstrins)
|
|
39
|
-
url = "/".join(list_url)
|
|
40
|
-
return url
|
|
41
|
-
|
|
42
|
-
@classmethod
|
|
43
|
-
def urlNetloc(cls, url):
|
|
44
|
-
"""
|
|
45
|
-
获取网址
|
|
46
|
-
:param url:
|
|
47
|
-
:return:
|
|
48
|
-
"""
|
|
49
|
-
return parse.urlparse(url).netloc
|
|
50
|
-
|
|
51
|
-
@classmethod
|
|
52
|
-
def urlScheme(cls, url):
|
|
53
|
-
"""
|
|
54
|
-
获取网站的请求方式 http or https
|
|
55
|
-
:param url:
|
|
56
|
-
:return:
|
|
57
|
-
"""
|
|
58
|
-
return parse.urlparse(url).scheme
|
|
59
|
-
|
|
60
|
-
@classmethod
|
|
61
|
-
def urlQuery2List(cls, url):
|
|
62
|
-
"""
|
|
63
|
-
urllib.parse.parse_qs 返回字典
|
|
64
|
-
{'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
|
|
65
|
-
urllib.parse.parse_qsl 返回列表
|
|
66
|
-
[('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
|
|
67
|
-
:param url:
|
|
68
|
-
:return:
|
|
69
|
-
"""
|
|
70
|
-
return parse.parse_qsl(parse.urlparse(url).query)
|
|
71
|
-
|
|
72
|
-
@classmethod
|
|
73
|
-
def urlQuery2dic(cls, url):
|
|
74
|
-
"""
|
|
75
|
-
urllib.parse.parse_qs 返回字典
|
|
76
|
-
{'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
|
|
77
|
-
urllib.parse.parse_qsl 返回列表
|
|
78
|
-
[('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
|
|
79
|
-
:param url:
|
|
80
|
-
:return:
|
|
81
|
-
"""
|
|
82
|
-
return parse.parse_qs(parse.urlparse(url).query)
|
|
83
|
-
|
|
84
|
-
@classmethod
|
|
85
|
-
def urlencode(cls, dic):
|
|
86
|
-
"""
|
|
87
|
-
将dic型的url参数进行编码 如
|
|
88
|
-
{'CID': 'quickSearchCitationFormat', 'database': '1', 'SEARCHID': 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259', 'intialSearch': 'true', 'angularReq': 'true', 'usageOrigin': 'searchform', 'usageZone': 'quicksearch', 'implicit': 'false', 'isFullJsonResult': 'true', 'startTime': '1529473818439', 'endTime': '1529473818477'}
|
|
89
|
-
:param dic:
|
|
90
|
-
:return:
|
|
91
|
-
"""
|
|
92
|
-
return parse.urlencode(dic)
|
|
93
|
-
|
|
94
|
-
@classmethod
|
|
95
|
-
def urldecode(self, strings):
|
|
96
|
-
"""
|
|
97
|
-
url解码
|
|
98
|
-
:param strings:
|
|
99
|
-
:return:
|
|
100
|
-
"""
|
|
101
|
-
return parse.unquote(strings)
|
|
102
|
-
|
|
103
|
-
@classmethod
|
|
104
|
-
def dicts_to_url(cls, dic, url=None):
|
|
105
|
-
"""
|
|
106
|
-
将字典转换成get的url参数部分
|
|
107
|
-
:param dic:
|
|
108
|
-
:return:
|
|
109
|
-
"""
|
|
110
|
-
parastring = ""
|
|
111
|
-
for key, value in dic.items():
|
|
112
|
-
parastring += str(key) + "=" + str(value) + "&"
|
|
113
|
-
parastring = parastring[:-1]
|
|
114
|
-
if url:
|
|
115
|
-
return url + "?" + parastring
|
|
116
|
-
return parastring
|
|
117
|
-
|
|
118
|
-
# url = "https://www.engineeringvillage.com/search/results/quick.url?CID=quickSearchCitationFormat&database=1&SEARCHID=f84e4d9aMa3f4M4e2aMae22M912f5e06a259&intialSearch=true&angularReq=true&usageOrigin=searchform&usageZone=quicksearch&implicit=false&isFullJsonResult=true&startTime=1529473818439&endTime=1529473818477"
|
|
119
|
-
# dic = BaseUrl.urlQuery2Dict(url)
|
|
120
|
-
# print(dic)
|
|
121
|
-
# BaseUrl.urlencode(url)
|
|
1
|
+
import urllib.parse as parse
|
|
2
|
+
|
|
3
|
+
from re_common.baselibrary.utils.basestring import BaseString
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BaseUrl(BaseString):
|
|
7
|
+
|
|
8
|
+
@classmethod
|
|
9
|
+
def urlQuery2Dict(cls, url):
|
|
10
|
+
"""
|
|
11
|
+
url中的参数转换成dict 字典类型方便读取
|
|
12
|
+
h还有另一种 方式
|
|
13
|
+
dict(parse.parse_qsl(parse.urlparse(url).query))
|
|
14
|
+
:param url:
|
|
15
|
+
:return:
|
|
16
|
+
"""
|
|
17
|
+
query = parse.urlparse(url).query
|
|
18
|
+
return dict([(k, v[0]) for k, v in parse.parse_qs(query).items()])
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def urlPath2List(cls, url):
|
|
22
|
+
"""
|
|
23
|
+
获取路由并返回一个列表
|
|
24
|
+
:param url:
|
|
25
|
+
:return:
|
|
26
|
+
"""
|
|
27
|
+
listPath = parse.urlparse(url).path.split("/")
|
|
28
|
+
try:
|
|
29
|
+
listPath.remove("")
|
|
30
|
+
except:
|
|
31
|
+
pass
|
|
32
|
+
return listPath
|
|
33
|
+
|
|
34
|
+
@classmethod
|
|
35
|
+
def urlreplaceend(cls, url, endstrins):
|
|
36
|
+
list_url = url.split("/")
|
|
37
|
+
list_url.pop()
|
|
38
|
+
list_url.append(endstrins)
|
|
39
|
+
url = "/".join(list_url)
|
|
40
|
+
return url
|
|
41
|
+
|
|
42
|
+
@classmethod
|
|
43
|
+
def urlNetloc(cls, url):
|
|
44
|
+
"""
|
|
45
|
+
获取网址
|
|
46
|
+
:param url:
|
|
47
|
+
:return:
|
|
48
|
+
"""
|
|
49
|
+
return parse.urlparse(url).netloc
|
|
50
|
+
|
|
51
|
+
@classmethod
|
|
52
|
+
def urlScheme(cls, url):
|
|
53
|
+
"""
|
|
54
|
+
获取网站的请求方式 http or https
|
|
55
|
+
:param url:
|
|
56
|
+
:return:
|
|
57
|
+
"""
|
|
58
|
+
return parse.urlparse(url).scheme
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def urlQuery2List(cls, url):
|
|
62
|
+
"""
|
|
63
|
+
urllib.parse.parse_qs 返回字典
|
|
64
|
+
{'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
|
|
65
|
+
urllib.parse.parse_qsl 返回列表
|
|
66
|
+
[('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
|
|
67
|
+
:param url:
|
|
68
|
+
:return:
|
|
69
|
+
"""
|
|
70
|
+
return parse.parse_qsl(parse.urlparse(url).query)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def urlQuery2dic(cls, url):
|
|
74
|
+
"""
|
|
75
|
+
urllib.parse.parse_qs 返回字典
|
|
76
|
+
{'CID': ['quickSearchCitationFormat'], 'database': ['1'], 'SEARCHID': ['f84e4d9aMa3f4M4e2aMae22M912f5e06a259'], 'intialSearch': ['true'], 'angularReq': ['true'], 'usageOrigin': ['searchform'], 'usageZone': ['quicksearch'], 'implicit': ['false'], 'isFullJsonResult': ['true'], 'startTime': ['1529473818439'], 'endTime': ['1529473818477']}
|
|
77
|
+
urllib.parse.parse_qsl 返回列表
|
|
78
|
+
[('CID', 'quickSearchCitationFormat'), ('database', '1'), ('SEARCHID', 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259'), ('intialSearch', 'true'), ('angularReq', 'true'), ('usageOrigin', 'searchform'), ('usageZone', 'quicksearch'), ('implicit', 'false'), ('isFullJsonResult', 'true'), ('startTime', '1529473818439'), ('endTime', '1529473818477')]
|
|
79
|
+
:param url:
|
|
80
|
+
:return:
|
|
81
|
+
"""
|
|
82
|
+
return parse.parse_qs(parse.urlparse(url).query)
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def urlencode(cls, dic):
|
|
86
|
+
"""
|
|
87
|
+
将dic型的url参数进行编码 如
|
|
88
|
+
{'CID': 'quickSearchCitationFormat', 'database': '1', 'SEARCHID': 'f84e4d9aMa3f4M4e2aMae22M912f5e06a259', 'intialSearch': 'true', 'angularReq': 'true', 'usageOrigin': 'searchform', 'usageZone': 'quicksearch', 'implicit': 'false', 'isFullJsonResult': 'true', 'startTime': '1529473818439', 'endTime': '1529473818477'}
|
|
89
|
+
:param dic:
|
|
90
|
+
:return:
|
|
91
|
+
"""
|
|
92
|
+
return parse.urlencode(dic)
|
|
93
|
+
|
|
94
|
+
@classmethod
|
|
95
|
+
def urldecode(self, strings):
|
|
96
|
+
"""
|
|
97
|
+
url解码
|
|
98
|
+
:param strings:
|
|
99
|
+
:return:
|
|
100
|
+
"""
|
|
101
|
+
return parse.unquote(strings)
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def dicts_to_url(cls, dic, url=None):
|
|
105
|
+
"""
|
|
106
|
+
将字典转换成get的url参数部分
|
|
107
|
+
:param dic:
|
|
108
|
+
:return:
|
|
109
|
+
"""
|
|
110
|
+
parastring = ""
|
|
111
|
+
for key, value in dic.items():
|
|
112
|
+
parastring += str(key) + "=" + str(value) + "&"
|
|
113
|
+
parastring = parastring[:-1]
|
|
114
|
+
if url:
|
|
115
|
+
return url + "?" + parastring
|
|
116
|
+
return parastring
|
|
117
|
+
|
|
118
|
+
# url = "https://www.engineeringvillage.com/search/results/quick.url?CID=quickSearchCitationFormat&database=1&SEARCHID=f84e4d9aMa3f4M4e2aMae22M912f5e06a259&intialSearch=true&angularReq=true&usageOrigin=searchform&usageZone=quicksearch&implicit=false&isFullJsonResult=true&startTime=1529473818439&endTime=1529473818477"
|
|
119
|
+
# dic = BaseUrl.urlQuery2Dict(url)
|
|
120
|
+
# print(dic)
|
|
121
|
+
# BaseUrl.urlencode(url)
|
|
@@ -1,57 +1,57 @@
|
|
|
1
|
-
import zipfile
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
class BaseZIP(object):
|
|
5
|
-
|
|
6
|
-
@classmethod
|
|
7
|
-
def get_zip_file(cls, filepath):
|
|
8
|
-
azip = zipfile.ZipFile(filepath)
|
|
9
|
-
return azip
|
|
10
|
-
|
|
11
|
-
@classmethod
|
|
12
|
-
def get_zip_namelist(cls, zip_obj):
|
|
13
|
-
"""
|
|
14
|
-
返回zip的文件列表,包括深层次目录
|
|
15
|
-
:return: list
|
|
16
|
-
"""
|
|
17
|
-
zip_list = zip_obj.namelist()
|
|
18
|
-
result_list = []
|
|
19
|
-
for zip_file in zip_list:
|
|
20
|
-
try:
|
|
21
|
-
zip_file = zip_file.encode('cp437').decode('gbk')
|
|
22
|
-
except:
|
|
23
|
-
zip_file = zip_file.encode('utf-8').decode('utf-8')
|
|
24
|
-
result_list.append(zip_file)
|
|
25
|
-
return result_list
|
|
26
|
-
|
|
27
|
-
@classmethod
|
|
28
|
-
def get_zipname(cls,zipobj):
|
|
29
|
-
"""
|
|
30
|
-
|
|
31
|
-
:param zipobj:
|
|
32
|
-
:return: zip文件位置
|
|
33
|
-
"""
|
|
34
|
-
return zipobj.filename
|
|
35
|
-
|
|
36
|
-
@classmethod
|
|
37
|
-
def get_info(cls,zipobj,filepath):
|
|
38
|
-
"""
|
|
39
|
-
获取文件的基本信息,注意,不是所有格式的文件都能获取信息
|
|
40
|
-
zipobj.getinfo(filepath)后可以获取以下信息
|
|
41
|
-
# 原来文件大小
|
|
42
|
-
print(azip_info.file_size)
|
|
43
|
-
# 压缩后大小
|
|
44
|
-
print(azip_info.compress_size)
|
|
45
|
-
|
|
46
|
-
# 这样可以求得压缩率,保留小数点后两位
|
|
47
|
-
print('压缩率为{:.2f}'.format(azip_info.file_size/azip_info.compress_size))
|
|
48
|
-
:param zipobj:
|
|
49
|
-
:param filepath: 为压缩文件下的相对目录 比如 test/test1/第二次目录/2021因工作被隔离明细表_采集组.xlsx
|
|
50
|
-
:return:
|
|
51
|
-
"""
|
|
52
|
-
return zipobj.getinfo(filepath)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
# zipobj = BaseZIP.get_zip_file(r'C:\Users\xuzhu\Desktop\VIPPatents_20210514To20210518.rar')
|
|
56
|
-
# print(BaseZIP.get_zip_namelist(zipobj))
|
|
57
|
-
#
|
|
1
|
+
import zipfile
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class BaseZIP(object):
|
|
5
|
+
|
|
6
|
+
@classmethod
|
|
7
|
+
def get_zip_file(cls, filepath):
|
|
8
|
+
azip = zipfile.ZipFile(filepath)
|
|
9
|
+
return azip
|
|
10
|
+
|
|
11
|
+
@classmethod
|
|
12
|
+
def get_zip_namelist(cls, zip_obj):
|
|
13
|
+
"""
|
|
14
|
+
返回zip的文件列表,包括深层次目录
|
|
15
|
+
:return: list
|
|
16
|
+
"""
|
|
17
|
+
zip_list = zip_obj.namelist()
|
|
18
|
+
result_list = []
|
|
19
|
+
for zip_file in zip_list:
|
|
20
|
+
try:
|
|
21
|
+
zip_file = zip_file.encode('cp437').decode('gbk')
|
|
22
|
+
except:
|
|
23
|
+
zip_file = zip_file.encode('utf-8').decode('utf-8')
|
|
24
|
+
result_list.append(zip_file)
|
|
25
|
+
return result_list
|
|
26
|
+
|
|
27
|
+
@classmethod
|
|
28
|
+
def get_zipname(cls,zipobj):
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
:param zipobj:
|
|
32
|
+
:return: zip文件位置
|
|
33
|
+
"""
|
|
34
|
+
return zipobj.filename
|
|
35
|
+
|
|
36
|
+
@classmethod
|
|
37
|
+
def get_info(cls,zipobj,filepath):
|
|
38
|
+
"""
|
|
39
|
+
获取文件的基本信息,注意,不是所有格式的文件都能获取信息
|
|
40
|
+
zipobj.getinfo(filepath)后可以获取以下信息
|
|
41
|
+
# 原来文件大小
|
|
42
|
+
print(azip_info.file_size)
|
|
43
|
+
# 压缩后大小
|
|
44
|
+
print(azip_info.compress_size)
|
|
45
|
+
|
|
46
|
+
# 这样可以求得压缩率,保留小数点后两位
|
|
47
|
+
print('压缩率为{:.2f}'.format(azip_info.file_size/azip_info.compress_size))
|
|
48
|
+
:param zipobj:
|
|
49
|
+
:param filepath: 为压缩文件下的相对目录 比如 test/test1/第二次目录/2021因工作被隔离明细表_采集组.xlsx
|
|
50
|
+
:return:
|
|
51
|
+
"""
|
|
52
|
+
return zipobj.getinfo(filepath)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# zipobj = BaseZIP.get_zip_file(r'C:\Users\xuzhu\Desktop\VIPPatents_20210514To20210518.rar')
|
|
56
|
+
# print(BaseZIP.get_zip_namelist(zipobj))
|
|
57
|
+
#
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# true 确定环境是python2
|
|
2
|
-
import sys
|
|
3
|
-
|
|
4
|
-
PY2 = sys.version_info[0] == 2
|
|
5
|
-
# true 确定环境是否是windows
|
|
6
|
-
WIN = sys.platform.startswith('win')
|
|
7
|
-
|
|
1
|
+
# true 确定环境是python2
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
PY2 = sys.version_info[0] == 2
|
|
5
|
+
# true 确定环境是否是windows
|
|
6
|
+
WIN = sys.platform.startswith('win')
|
|
7
|
+
|
|
8
8
|
_identity = lambda x: x
|
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
# 储存单位
|
|
2
|
-
import os
|
|
3
|
-
import platform
|
|
4
|
-
import re
|
|
5
|
-
|
|
6
|
-
# 文件大小单位
|
|
7
|
-
|
|
8
|
-
SUFFIX = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
|
9
|
-
# 系统的分割符
|
|
10
|
-
__os_sep__ = "\\" if platform.system() == 'Windows' else os.sep
|
|
11
|
-
# 标准文件名正着
|
|
12
|
-
_filename_ascii_strip_re = re.compile(r'[^A-Za-z0-9_.-]')
|
|
13
|
-
# windows的设备文件
|
|
14
|
-
_windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', 'COM4', 'LPT1',
|
|
15
|
-
'LPT2', 'LPT3', 'PRN', 'NUL')
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
1
|
+
# 储存单位
|
|
2
|
+
import os
|
|
3
|
+
import platform
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
# 文件大小单位
|
|
7
|
+
|
|
8
|
+
SUFFIX = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
|
9
|
+
# 系统的分割符
|
|
10
|
+
__os_sep__ = "\\" if platform.system() == 'Windows' else os.sep
|
|
11
|
+
# 标准文件名正着
|
|
12
|
+
_filename_ascii_strip_re = re.compile(r'[^A-Za-z0-9_.-]')
|
|
13
|
+
# windows的设备文件
|
|
14
|
+
_windows_device_files = ('CON', 'AUX', 'COM1', 'COM2', 'COM3', 'COM4', 'LPT1',
|
|
15
|
+
'LPT2', 'LPT3', 'PRN', 'NUL')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|