re-common 10.0.39__py3-none-any.whl → 10.0.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +77 -77
- re_common/v2/baselibrary/utils/db.py +156 -156
- re_common/v2/baselibrary/utils/elasticsearch.py +46 -0
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +186 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +271 -271
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/LICENSE +201 -201
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/METADATA +24 -16
- re_common-10.0.40.dist-info/RECORD +249 -0
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/WHEEL +1 -1
- re_common-10.0.39.dist-info/RECORD +0 -248
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/top_level.txt +0 -0
|
@@ -1,270 +1,270 @@
|
|
|
1
|
-
import atexit
|
|
2
|
-
import sys
|
|
3
|
-
import asyncio
|
|
4
|
-
import aiohttp
|
|
5
|
-
from typing import Optional
|
|
6
|
-
|
|
7
|
-
from tenacity import retry, stop_after_attempt, wait_random
|
|
8
|
-
|
|
9
|
-
g_headers = {
|
|
10
|
-
'accept': 'application/json',
|
|
11
|
-
'Content-Type': 'application/json',
|
|
12
|
-
'Authorization': 'Bearer eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJ1c2VyX2lkIjotMSwidXNlcl9uYW1lIjoiXHU1ZTk0XHU3NTI4XHU0ZTJkXHU1ZmMzQ2xpZW50In0.'
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
"""
|
|
16
|
-
cls._conn = aiohttp.TCPConnector(
|
|
17
|
-
limit=50, # 最大连接数
|
|
18
|
-
ssl=False, # 禁用SSL验证(按需开启)
|
|
19
|
-
force_close=True, # 保持连接活跃
|
|
20
|
-
enable_cleanup_closed=True # 自动清理关闭的连接
|
|
21
|
-
)
|
|
22
|
-
# 由于网络上有重名,没有连接。如果加入域,请转到“控制面板”中的“系统”更改计算机名,然后重试。如果加入工作组,请选择其他工作组名。
|
|
23
|
-
有可能是
|
|
24
|
-
force_close=True, # 保持连接活跃
|
|
25
|
-
enable_cleanup_closed=True # 自动清理关闭的连接
|
|
26
|
-
照成的
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
class HttpError(Exception):
|
|
31
|
-
code = 0
|
|
32
|
-
message = ""
|
|
33
|
-
headers = None
|
|
34
|
-
|
|
35
|
-
def __init__(
|
|
36
|
-
self,
|
|
37
|
-
*,
|
|
38
|
-
code: Optional[int] = None,
|
|
39
|
-
message: str = "",
|
|
40
|
-
headers: Optional[dict] = None,
|
|
41
|
-
) -> None:
|
|
42
|
-
if code is not None:
|
|
43
|
-
self.code = code
|
|
44
|
-
self.headers = headers
|
|
45
|
-
self.message = message
|
|
46
|
-
|
|
47
|
-
def __str__(self) -> str:
|
|
48
|
-
return f"code: {self.code}, message:{self.message}"
|
|
49
|
-
|
|
50
|
-
def __repr__(self) -> str:
|
|
51
|
-
return f"<{self.__class__.__name__}: code={self.code}, message={self.message!r}>"
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
def on_retry_error(retry_state):
|
|
55
|
-
# 最后抛错后调用
|
|
56
|
-
original_exc = retry_state.outcome.exception()
|
|
57
|
-
print(f"[HTTP 请求重试所有重试失败.] 错误消息{original_exc}")
|
|
58
|
-
|
|
59
|
-
raise HttpError(code=getattr(original_exc, 'code', 455),
|
|
60
|
-
message=f"错误消息:{str(original_exc)}") from original_exc
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def on_retry(retry_state):
|
|
64
|
-
# 每次抛错进入该函数打印消息
|
|
65
|
-
print(
|
|
66
|
-
f"[HTTP 请求重试]"
|
|
67
|
-
f"当前重试 : 第 {retry_state.attempt_number} 次"
|
|
68
|
-
f"睡眠时间 : {retry_state.next_action.sleep:.2f} 秒"
|
|
69
|
-
f"\n异常原因 : {retry_state.outcome.exception()}"
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class ApiNetUtils:
|
|
74
|
-
"""
|
|
75
|
-
HTTP请求工具类(异步版),提供GET/POST/PATCH请求方法
|
|
76
|
-
特性:
|
|
77
|
-
1. 自动复用TCP连接池
|
|
78
|
-
2. 自动重试机制(通过async_retry装饰器)
|
|
79
|
-
3. 进程退出时自动清理资源
|
|
80
|
-
4. 线程安全的延迟初始化
|
|
81
|
-
"""
|
|
82
|
-
|
|
83
|
-
# 类属性使用Optional类型注解,初始化为None实现延迟初始化
|
|
84
|
-
_conn: Optional[aiohttp.TCPConnector] = None
|
|
85
|
-
_session: Optional[aiohttp.ClientSession] = None
|
|
86
|
-
_close_registered: bool = False # 确保清理函数只注册一次
|
|
87
|
-
|
|
88
|
-
@classmethod
|
|
89
|
-
async def _get_connector(cls) -> aiohttp.TCPConnector:
|
|
90
|
-
"""
|
|
91
|
-
获取TCP连接器(延迟初始化)
|
|
92
|
-
解决模块加载时没有事件循环的问题
|
|
93
|
-
"""
|
|
94
|
-
if cls._conn is None or cls._conn.closed or cls.is_loop_closed(cls._session):
|
|
95
|
-
# 只有在首次使用时才创建连接器
|
|
96
|
-
cls._conn = aiohttp.TCPConnector(
|
|
97
|
-
limit=50, # 最大连接数
|
|
98
|
-
ssl=False, # 禁用SSL验证(按需开启)
|
|
99
|
-
force_close=True, # 保持连接活跃
|
|
100
|
-
# enable_cleanup_closed=True, # 自动清理关闭的连接 #
|
|
101
|
-
# keepalive_timeout=4.99 # 比服务器的5s 小一点
|
|
102
|
-
)
|
|
103
|
-
return cls._conn
|
|
104
|
-
|
|
105
|
-
@classmethod
|
|
106
|
-
async def _get_session(cls) -> aiohttp.ClientSession:
|
|
107
|
-
"""
|
|
108
|
-
获取共享会话(线程安全的延迟初始化)
|
|
109
|
-
包含自动注册清理机制
|
|
110
|
-
"""
|
|
111
|
-
if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
|
|
112
|
-
if cls._session:
|
|
113
|
-
await cls.close()
|
|
114
|
-
# 获取连接器(会自动初始化)
|
|
115
|
-
connector = await cls._get_connector()
|
|
116
|
-
|
|
117
|
-
# 强制获取新的事件循环
|
|
118
|
-
loop = asyncio.get_event_loop()
|
|
119
|
-
|
|
120
|
-
# 创建新会话
|
|
121
|
-
cls._session = aiohttp.ClientSession(
|
|
122
|
-
connector=connector,
|
|
123
|
-
timeout=aiohttp.ClientTimeout(total=30), # 默认30秒超时
|
|
124
|
-
loop=loop) # 显式指定事件循环
|
|
125
|
-
|
|
126
|
-
# # 注册退出时的清理钩子
|
|
127
|
-
cls._register_cleanup()
|
|
128
|
-
|
|
129
|
-
return cls._session
|
|
130
|
-
|
|
131
|
-
@staticmethod
|
|
132
|
-
def is_loop_closed(session: aiohttp.ClientSession) -> bool:
|
|
133
|
-
"""
|
|
134
|
-
检查会话绑定的事件循环是否已关闭
|
|
135
|
-
"""
|
|
136
|
-
loop = session._loop # 获取会话绑定的事件循环
|
|
137
|
-
if loop.is_closed():
|
|
138
|
-
# print("Event loop is closed")
|
|
139
|
-
return True
|
|
140
|
-
# print("Event loop not is closed")
|
|
141
|
-
return False
|
|
142
|
-
|
|
143
|
-
@classmethod
|
|
144
|
-
def _register_cleanup(cls):
|
|
145
|
-
"""
|
|
146
|
-
注册进程退出时的资源清理函数
|
|
147
|
-
包含正常退出和异常退出两种情况
|
|
148
|
-
"""
|
|
149
|
-
if not cls._close_registered:
|
|
150
|
-
# 1. 正常退出处理
|
|
151
|
-
atexit.register(lambda: asyncio.run(cls.close()))
|
|
152
|
-
|
|
153
|
-
# 2. 异常退出处理
|
|
154
|
-
original_excepthook = sys.excepthook
|
|
155
|
-
|
|
156
|
-
def custom_excepthook(exctype, value, traceback):
|
|
157
|
-
"""自定义异常钩子,确保资源被清理"""
|
|
158
|
-
# 先执行原始异常处理(打印堆栈等)
|
|
159
|
-
original_excepthook(exctype, value, traceback)
|
|
160
|
-
# 然后执行资源清理
|
|
161
|
-
try:
|
|
162
|
-
asyncio.run(cls.close())
|
|
163
|
-
except RuntimeError:
|
|
164
|
-
# 如果已经没有事件循环,则同步执行
|
|
165
|
-
loop = asyncio.new_event_loop()
|
|
166
|
-
loop.run_until_complete(cls.close())
|
|
167
|
-
loop.close()
|
|
168
|
-
|
|
169
|
-
sys.excepthook = custom_excepthook
|
|
170
|
-
cls._close_registered = True
|
|
171
|
-
|
|
172
|
-
@classmethod
|
|
173
|
-
async def close(cls):
|
|
174
|
-
"""
|
|
175
|
-
安全关闭所有网络资源
|
|
176
|
-
会自动在程序退出时调用,也可手动调用
|
|
177
|
-
"""
|
|
178
|
-
if cls._session and not cls._session.closed:
|
|
179
|
-
await cls._session.close()
|
|
180
|
-
cls._session = None
|
|
181
|
-
|
|
182
|
-
if cls._conn and not cls._conn.closed:
|
|
183
|
-
await cls._conn.close()
|
|
184
|
-
cls._conn = None
|
|
185
|
-
|
|
186
|
-
# print("[ApiNetUtils] 网络资源已安全释放")
|
|
187
|
-
|
|
188
|
-
# -------------------- 公共API方法 -------------------- #
|
|
189
|
-
|
|
190
|
-
@classmethod
|
|
191
|
-
@retry(stop=stop_after_attempt(4), # 本质上执行4次 但重试3次
|
|
192
|
-
wait=wait_random(min=5, max=15),
|
|
193
|
-
before_sleep=on_retry, # 每次抛错后使用
|
|
194
|
-
retry_error_callback=on_retry_error,
|
|
195
|
-
reraise=True)
|
|
196
|
-
async def fetch_get(cls, url: str, headers=None, params=None):
|
|
197
|
-
"""
|
|
198
|
-
GET请求封装
|
|
199
|
-
:param url: 请求URL
|
|
200
|
-
:param headers: 可选请求头(默认使用全局g_headers)
|
|
201
|
-
:param params: 查询参数(字典)
|
|
202
|
-
:return: 解析后的JSON数据
|
|
203
|
-
:raises HttpError: 当状态码非200时抛出
|
|
204
|
-
"""
|
|
205
|
-
headers = headers or g_headers
|
|
206
|
-
session = await cls._get_session()
|
|
207
|
-
|
|
208
|
-
async with session.get(url, headers=headers, params=params) as response:
|
|
209
|
-
if response.status != 200:
|
|
210
|
-
error_text = await response.text()
|
|
211
|
-
raise HttpError(
|
|
212
|
-
code=response.status,
|
|
213
|
-
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
214
|
-
)
|
|
215
|
-
return await response.json()
|
|
216
|
-
|
|
217
|
-
@classmethod
|
|
218
|
-
@retry(stop=stop_after_attempt(4),
|
|
219
|
-
wait=wait_random(min=5, max=15),
|
|
220
|
-
before_sleep=on_retry, # 每次抛错后使用
|
|
221
|
-
retry_error_callback=on_retry_error,
|
|
222
|
-
reraise=True)
|
|
223
|
-
async def fetch_post(cls, url: str, payload: dict, headers=None):
|
|
224
|
-
"""
|
|
225
|
-
POST请求封装(JSON格式)
|
|
226
|
-
"""
|
|
227
|
-
headers = headers or g_headers
|
|
228
|
-
session = await cls._get_session()
|
|
229
|
-
|
|
230
|
-
async with session.post(url, json=payload, headers=headers) as response:
|
|
231
|
-
if response.status != 200:
|
|
232
|
-
error_text = await response.text()
|
|
233
|
-
raise HttpError(
|
|
234
|
-
code=response.status,
|
|
235
|
-
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
236
|
-
)
|
|
237
|
-
return await response.json()
|
|
238
|
-
|
|
239
|
-
@classmethod
|
|
240
|
-
@retry(stop=stop_after_attempt(4),
|
|
241
|
-
wait=wait_random(min=5, max=15),
|
|
242
|
-
before_sleep=on_retry, # 每次抛错后使用
|
|
243
|
-
retry_error_callback=on_retry_error,
|
|
244
|
-
reraise=True)
|
|
245
|
-
async def fetch_patch(cls, url: str, payload: dict, headers=None):
|
|
246
|
-
"""
|
|
247
|
-
PATCH请求封装(JSON格式)
|
|
248
|
-
"""
|
|
249
|
-
headers = headers or g_headers
|
|
250
|
-
session = await cls._get_session()
|
|
251
|
-
|
|
252
|
-
async with session.patch(url, json=payload, headers=headers) as response:
|
|
253
|
-
if response.status != 200:
|
|
254
|
-
error_text = await response.text()
|
|
255
|
-
raise HttpError(
|
|
256
|
-
code=response.status,
|
|
257
|
-
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
258
|
-
)
|
|
259
|
-
return await response.json()
|
|
260
|
-
|
|
261
|
-
@classmethod
|
|
262
|
-
async def __aenter__(cls):
|
|
263
|
-
"""支持async with语法"""
|
|
264
|
-
await cls._get_session()
|
|
265
|
-
return cls
|
|
266
|
-
|
|
267
|
-
@classmethod
|
|
268
|
-
async def __aexit__(cls, exc_type, exc, tb):
|
|
269
|
-
"""async with退出时自动关闭"""
|
|
270
|
-
await cls.close()
|
|
1
|
+
import atexit
|
|
2
|
+
import sys
|
|
3
|
+
import asyncio
|
|
4
|
+
import aiohttp
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from tenacity import retry, stop_after_attempt, wait_random
|
|
8
|
+
|
|
9
|
+
g_headers = {
|
|
10
|
+
'accept': 'application/json',
|
|
11
|
+
'Content-Type': 'application/json',
|
|
12
|
+
'Authorization': 'Bearer eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJ1c2VyX2lkIjotMSwidXNlcl9uYW1lIjoiXHU1ZTk0XHU3NTI4XHU0ZTJkXHU1ZmMzQ2xpZW50In0.'
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
cls._conn = aiohttp.TCPConnector(
|
|
17
|
+
limit=50, # 最大连接数
|
|
18
|
+
ssl=False, # 禁用SSL验证(按需开启)
|
|
19
|
+
force_close=True, # 保持连接活跃
|
|
20
|
+
enable_cleanup_closed=True # 自动清理关闭的连接
|
|
21
|
+
)
|
|
22
|
+
# 由于网络上有重名,没有连接。如果加入域,请转到“控制面板”中的“系统”更改计算机名,然后重试。如果加入工作组,请选择其他工作组名。
|
|
23
|
+
有可能是
|
|
24
|
+
force_close=True, # 保持连接活跃
|
|
25
|
+
enable_cleanup_closed=True # 自动清理关闭的连接
|
|
26
|
+
照成的
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class HttpError(Exception):
|
|
31
|
+
code = 0
|
|
32
|
+
message = ""
|
|
33
|
+
headers = None
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
*,
|
|
38
|
+
code: Optional[int] = None,
|
|
39
|
+
message: str = "",
|
|
40
|
+
headers: Optional[dict] = None,
|
|
41
|
+
) -> None:
|
|
42
|
+
if code is not None:
|
|
43
|
+
self.code = code
|
|
44
|
+
self.headers = headers
|
|
45
|
+
self.message = message
|
|
46
|
+
|
|
47
|
+
def __str__(self) -> str:
|
|
48
|
+
return f"code: {self.code}, message:{self.message}"
|
|
49
|
+
|
|
50
|
+
def __repr__(self) -> str:
|
|
51
|
+
return f"<{self.__class__.__name__}: code={self.code}, message={self.message!r}>"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def on_retry_error(retry_state):
|
|
55
|
+
# 最后抛错后调用
|
|
56
|
+
original_exc = retry_state.outcome.exception()
|
|
57
|
+
print(f"[HTTP 请求重试所有重试失败.] 错误消息{original_exc}")
|
|
58
|
+
|
|
59
|
+
raise HttpError(code=getattr(original_exc, 'code', 455),
|
|
60
|
+
message=f"错误消息:{str(original_exc)}") from original_exc
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def on_retry(retry_state):
|
|
64
|
+
# 每次抛错进入该函数打印消息
|
|
65
|
+
print(
|
|
66
|
+
f"[HTTP 请求重试]"
|
|
67
|
+
f"当前重试 : 第 {retry_state.attempt_number} 次"
|
|
68
|
+
f"睡眠时间 : {retry_state.next_action.sleep:.2f} 秒"
|
|
69
|
+
f"\n异常原因 : {retry_state.outcome.exception()}"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ApiNetUtils:
|
|
74
|
+
"""
|
|
75
|
+
HTTP请求工具类(异步版),提供GET/POST/PATCH请求方法
|
|
76
|
+
特性:
|
|
77
|
+
1. 自动复用TCP连接池
|
|
78
|
+
2. 自动重试机制(通过async_retry装饰器)
|
|
79
|
+
3. 进程退出时自动清理资源
|
|
80
|
+
4. 线程安全的延迟初始化
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
# 类属性使用Optional类型注解,初始化为None实现延迟初始化
|
|
84
|
+
_conn: Optional[aiohttp.TCPConnector] = None
|
|
85
|
+
_session: Optional[aiohttp.ClientSession] = None
|
|
86
|
+
_close_registered: bool = False # 确保清理函数只注册一次
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
async def _get_connector(cls) -> aiohttp.TCPConnector:
|
|
90
|
+
"""
|
|
91
|
+
获取TCP连接器(延迟初始化)
|
|
92
|
+
解决模块加载时没有事件循环的问题
|
|
93
|
+
"""
|
|
94
|
+
if cls._conn is None or cls._conn.closed or cls.is_loop_closed(cls._session):
|
|
95
|
+
# 只有在首次使用时才创建连接器
|
|
96
|
+
cls._conn = aiohttp.TCPConnector(
|
|
97
|
+
limit=50, # 最大连接数
|
|
98
|
+
ssl=False, # 禁用SSL验证(按需开启)
|
|
99
|
+
force_close=True, # 保持连接活跃
|
|
100
|
+
# enable_cleanup_closed=True, # 自动清理关闭的连接 #
|
|
101
|
+
# keepalive_timeout=4.99 # 比服务器的5s 小一点
|
|
102
|
+
)
|
|
103
|
+
return cls._conn
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
async def _get_session(cls) -> aiohttp.ClientSession:
|
|
107
|
+
"""
|
|
108
|
+
获取共享会话(线程安全的延迟初始化)
|
|
109
|
+
包含自动注册清理机制
|
|
110
|
+
"""
|
|
111
|
+
if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
|
|
112
|
+
if cls._session:
|
|
113
|
+
await cls.close()
|
|
114
|
+
# 获取连接器(会自动初始化)
|
|
115
|
+
connector = await cls._get_connector()
|
|
116
|
+
|
|
117
|
+
# 强制获取新的事件循环
|
|
118
|
+
loop = asyncio.get_event_loop()
|
|
119
|
+
|
|
120
|
+
# 创建新会话
|
|
121
|
+
cls._session = aiohttp.ClientSession(
|
|
122
|
+
connector=connector,
|
|
123
|
+
timeout=aiohttp.ClientTimeout(total=30), # 默认30秒超时
|
|
124
|
+
loop=loop) # 显式指定事件循环
|
|
125
|
+
|
|
126
|
+
# # 注册退出时的清理钩子
|
|
127
|
+
cls._register_cleanup()
|
|
128
|
+
|
|
129
|
+
return cls._session
|
|
130
|
+
|
|
131
|
+
@staticmethod
|
|
132
|
+
def is_loop_closed(session: aiohttp.ClientSession) -> bool:
|
|
133
|
+
"""
|
|
134
|
+
检查会话绑定的事件循环是否已关闭
|
|
135
|
+
"""
|
|
136
|
+
loop = session._loop # 获取会话绑定的事件循环
|
|
137
|
+
if loop.is_closed():
|
|
138
|
+
# print("Event loop is closed")
|
|
139
|
+
return True
|
|
140
|
+
# print("Event loop not is closed")
|
|
141
|
+
return False
|
|
142
|
+
|
|
143
|
+
@classmethod
|
|
144
|
+
def _register_cleanup(cls):
|
|
145
|
+
"""
|
|
146
|
+
注册进程退出时的资源清理函数
|
|
147
|
+
包含正常退出和异常退出两种情况
|
|
148
|
+
"""
|
|
149
|
+
if not cls._close_registered:
|
|
150
|
+
# 1. 正常退出处理
|
|
151
|
+
atexit.register(lambda: asyncio.run(cls.close()))
|
|
152
|
+
|
|
153
|
+
# 2. 异常退出处理
|
|
154
|
+
original_excepthook = sys.excepthook
|
|
155
|
+
|
|
156
|
+
def custom_excepthook(exctype, value, traceback):
|
|
157
|
+
"""自定义异常钩子,确保资源被清理"""
|
|
158
|
+
# 先执行原始异常处理(打印堆栈等)
|
|
159
|
+
original_excepthook(exctype, value, traceback)
|
|
160
|
+
# 然后执行资源清理
|
|
161
|
+
try:
|
|
162
|
+
asyncio.run(cls.close())
|
|
163
|
+
except RuntimeError:
|
|
164
|
+
# 如果已经没有事件循环,则同步执行
|
|
165
|
+
loop = asyncio.new_event_loop()
|
|
166
|
+
loop.run_until_complete(cls.close())
|
|
167
|
+
loop.close()
|
|
168
|
+
|
|
169
|
+
sys.excepthook = custom_excepthook
|
|
170
|
+
cls._close_registered = True
|
|
171
|
+
|
|
172
|
+
@classmethod
|
|
173
|
+
async def close(cls):
|
|
174
|
+
"""
|
|
175
|
+
安全关闭所有网络资源
|
|
176
|
+
会自动在程序退出时调用,也可手动调用
|
|
177
|
+
"""
|
|
178
|
+
if cls._session and not cls._session.closed:
|
|
179
|
+
await cls._session.close()
|
|
180
|
+
cls._session = None
|
|
181
|
+
|
|
182
|
+
if cls._conn and not cls._conn.closed:
|
|
183
|
+
await cls._conn.close()
|
|
184
|
+
cls._conn = None
|
|
185
|
+
|
|
186
|
+
# print("[ApiNetUtils] 网络资源已安全释放")
|
|
187
|
+
|
|
188
|
+
# -------------------- 公共API方法 -------------------- #
|
|
189
|
+
|
|
190
|
+
@classmethod
|
|
191
|
+
@retry(stop=stop_after_attempt(4), # 本质上执行4次 但重试3次
|
|
192
|
+
wait=wait_random(min=5, max=15),
|
|
193
|
+
before_sleep=on_retry, # 每次抛错后使用
|
|
194
|
+
retry_error_callback=on_retry_error,
|
|
195
|
+
reraise=True)
|
|
196
|
+
async def fetch_get(cls, url: str, headers=None, params=None):
|
|
197
|
+
"""
|
|
198
|
+
GET请求封装
|
|
199
|
+
:param url: 请求URL
|
|
200
|
+
:param headers: 可选请求头(默认使用全局g_headers)
|
|
201
|
+
:param params: 查询参数(字典)
|
|
202
|
+
:return: 解析后的JSON数据
|
|
203
|
+
:raises HttpError: 当状态码非200时抛出
|
|
204
|
+
"""
|
|
205
|
+
headers = headers or g_headers
|
|
206
|
+
session = await cls._get_session()
|
|
207
|
+
|
|
208
|
+
async with session.get(url, headers=headers, params=params) as response:
|
|
209
|
+
if response.status != 200:
|
|
210
|
+
error_text = await response.text()
|
|
211
|
+
raise HttpError(
|
|
212
|
+
code=response.status,
|
|
213
|
+
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
214
|
+
)
|
|
215
|
+
return await response.json()
|
|
216
|
+
|
|
217
|
+
@classmethod
|
|
218
|
+
@retry(stop=stop_after_attempt(4),
|
|
219
|
+
wait=wait_random(min=5, max=15),
|
|
220
|
+
before_sleep=on_retry, # 每次抛错后使用
|
|
221
|
+
retry_error_callback=on_retry_error,
|
|
222
|
+
reraise=True)
|
|
223
|
+
async def fetch_post(cls, url: str, payload: dict, headers=None):
|
|
224
|
+
"""
|
|
225
|
+
POST请求封装(JSON格式)
|
|
226
|
+
"""
|
|
227
|
+
headers = headers or g_headers
|
|
228
|
+
session = await cls._get_session()
|
|
229
|
+
|
|
230
|
+
async with session.post(url, json=payload, headers=headers) as response:
|
|
231
|
+
if response.status != 200:
|
|
232
|
+
error_text = await response.text()
|
|
233
|
+
raise HttpError(
|
|
234
|
+
code=response.status,
|
|
235
|
+
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
236
|
+
)
|
|
237
|
+
return await response.json()
|
|
238
|
+
|
|
239
|
+
@classmethod
|
|
240
|
+
@retry(stop=stop_after_attempt(4),
|
|
241
|
+
wait=wait_random(min=5, max=15),
|
|
242
|
+
before_sleep=on_retry, # 每次抛错后使用
|
|
243
|
+
retry_error_callback=on_retry_error,
|
|
244
|
+
reraise=True)
|
|
245
|
+
async def fetch_patch(cls, url: str, payload: dict, headers=None):
|
|
246
|
+
"""
|
|
247
|
+
PATCH请求封装(JSON格式)
|
|
248
|
+
"""
|
|
249
|
+
headers = headers or g_headers
|
|
250
|
+
session = await cls._get_session()
|
|
251
|
+
|
|
252
|
+
async with session.patch(url, json=payload, headers=headers) as response:
|
|
253
|
+
if response.status != 200:
|
|
254
|
+
error_text = await response.text()
|
|
255
|
+
raise HttpError(
|
|
256
|
+
code=response.status,
|
|
257
|
+
message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
|
|
258
|
+
)
|
|
259
|
+
return await response.json()
|
|
260
|
+
|
|
261
|
+
@classmethod
|
|
262
|
+
async def __aenter__(cls):
|
|
263
|
+
"""支持async with语法"""
|
|
264
|
+
await cls._get_session()
|
|
265
|
+
return cls
|
|
266
|
+
|
|
267
|
+
@classmethod
|
|
268
|
+
async def __aexit__(cls, exc_type, exc, tb):
|
|
269
|
+
"""async with退出时自动关闭"""
|
|
270
|
+
await cls.close()
|