re-common 10.0.39__py3-none-any.whl → 10.0.40__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. re_common/baselibrary/__init__.py +4 -4
  2. re_common/baselibrary/baseabs/__init__.py +6 -6
  3. re_common/baselibrary/baseabs/baseabs.py +26 -26
  4. re_common/baselibrary/database/mbuilder.py +132 -132
  5. re_common/baselibrary/database/moudle.py +93 -93
  6. re_common/baselibrary/database/msqlite3.py +194 -194
  7. re_common/baselibrary/database/mysql.py +169 -169
  8. re_common/baselibrary/database/sql_factory.py +26 -26
  9. re_common/baselibrary/mthread/MThreadingRun.py +486 -486
  10. re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
  11. re_common/baselibrary/mthread/__init__.py +2 -2
  12. re_common/baselibrary/mthread/mythreading.py +695 -695
  13. re_common/baselibrary/pakge_other/socks.py +404 -404
  14. re_common/baselibrary/readconfig/config_factory.py +18 -18
  15. re_common/baselibrary/readconfig/ini_config.py +317 -317
  16. re_common/baselibrary/readconfig/toml_config.py +49 -49
  17. re_common/baselibrary/temporary/envdata.py +36 -36
  18. re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
  19. re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
  20. re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
  21. re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
  22. re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
  23. re_common/baselibrary/tools/contrast_db3.py +123 -123
  24. re_common/baselibrary/tools/copy_file.py +39 -39
  25. re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
  26. re_common/baselibrary/tools/foreachgz.py +39 -39
  27. re_common/baselibrary/tools/get_attr.py +10 -10
  28. re_common/baselibrary/tools/image_to_pdf.py +61 -61
  29. re_common/baselibrary/tools/java_code_deal.py +139 -139
  30. re_common/baselibrary/tools/javacode.py +79 -79
  31. re_common/baselibrary/tools/mdb_db3.py +48 -48
  32. re_common/baselibrary/tools/merge_file.py +171 -171
  33. re_common/baselibrary/tools/merge_gz_file.py +165 -165
  34. re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
  35. re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
  36. re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
  37. re_common/baselibrary/tools/mongo_tools.py +50 -50
  38. re_common/baselibrary/tools/move_file.py +170 -170
  39. re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
  40. re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
  41. re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
  42. re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
  43. re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
  44. re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
  45. re_common/baselibrary/tools/myparsel.py +104 -104
  46. re_common/baselibrary/tools/rename_dir_file.py +37 -37
  47. re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
  48. re_common/baselibrary/tools/split_line_to_many.py +25 -25
  49. re_common/baselibrary/tools/stringtodicts.py +33 -33
  50. re_common/baselibrary/tools/workwechant_bot.py +84 -84
  51. re_common/baselibrary/utils/baseaiohttp.py +296 -296
  52. re_common/baselibrary/utils/baseaiomysql.py +87 -87
  53. re_common/baselibrary/utils/baseallstep.py +191 -191
  54. re_common/baselibrary/utils/baseavro.py +19 -19
  55. re_common/baselibrary/utils/baseboto3.py +291 -291
  56. re_common/baselibrary/utils/basecsv.py +32 -32
  57. re_common/baselibrary/utils/basedict.py +133 -133
  58. re_common/baselibrary/utils/basedir.py +241 -241
  59. re_common/baselibrary/utils/baseencode.py +351 -351
  60. re_common/baselibrary/utils/baseencoding.py +28 -28
  61. re_common/baselibrary/utils/baseesdsl.py +86 -86
  62. re_common/baselibrary/utils/baseexcel.py +264 -264
  63. re_common/baselibrary/utils/baseexcept.py +109 -109
  64. re_common/baselibrary/utils/basefile.py +654 -654
  65. re_common/baselibrary/utils/baseftp.py +214 -214
  66. re_common/baselibrary/utils/basegzip.py +60 -60
  67. re_common/baselibrary/utils/basehdfs.py +135 -135
  68. re_common/baselibrary/utils/basehttpx.py +268 -268
  69. re_common/baselibrary/utils/baseip.py +87 -87
  70. re_common/baselibrary/utils/basejson.py +2 -2
  71. re_common/baselibrary/utils/baselist.py +32 -32
  72. re_common/baselibrary/utils/basemotor.py +190 -190
  73. re_common/baselibrary/utils/basemssql.py +98 -98
  74. re_common/baselibrary/utils/baseodbc.py +113 -113
  75. re_common/baselibrary/utils/basepandas.py +302 -302
  76. re_common/baselibrary/utils/basepeewee.py +11 -11
  77. re_common/baselibrary/utils/basepika.py +180 -180
  78. re_common/baselibrary/utils/basepydash.py +143 -143
  79. re_common/baselibrary/utils/basepymongo.py +230 -230
  80. re_common/baselibrary/utils/basequeue.py +22 -22
  81. re_common/baselibrary/utils/baserar.py +57 -57
  82. re_common/baselibrary/utils/baserequest.py +279 -279
  83. re_common/baselibrary/utils/baseset.py +8 -8
  84. re_common/baselibrary/utils/basesmb.py +403 -403
  85. re_common/baselibrary/utils/basestring.py +382 -382
  86. re_common/baselibrary/utils/basetime.py +320 -320
  87. re_common/baselibrary/utils/baseurl.py +121 -121
  88. re_common/baselibrary/utils/basezip.py +57 -57
  89. re_common/baselibrary/utils/core/__init__.py +7 -7
  90. re_common/baselibrary/utils/core/bottomutils.py +18 -18
  91. re_common/baselibrary/utils/core/mdeprecated.py +327 -327
  92. re_common/baselibrary/utils/core/mlamada.py +16 -16
  93. re_common/baselibrary/utils/core/msginfo.py +25 -25
  94. re_common/baselibrary/utils/core/requests_core.py +103 -103
  95. re_common/baselibrary/utils/fateadm.py +429 -429
  96. re_common/baselibrary/utils/importfun.py +123 -123
  97. re_common/baselibrary/utils/mfaker.py +57 -57
  98. re_common/baselibrary/utils/my_abc/__init__.py +3 -3
  99. re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
  100. re_common/baselibrary/utils/mylogger.py +414 -414
  101. re_common/baselibrary/utils/myredisclient.py +861 -861
  102. re_common/baselibrary/utils/pipupgrade.py +21 -21
  103. re_common/baselibrary/utils/ringlist.py +85 -85
  104. re_common/baselibrary/utils/version_compare.py +36 -36
  105. re_common/baselibrary/utils/ydmhttp.py +126 -126
  106. re_common/facade/lazy_import.py +11 -11
  107. re_common/facade/loggerfacade.py +25 -25
  108. re_common/facade/mysqlfacade.py +467 -467
  109. re_common/facade/now.py +31 -31
  110. re_common/facade/sqlite3facade.py +257 -257
  111. re_common/facade/use/mq_use_facade.py +83 -83
  112. re_common/facade/use/proxy_use_facade.py +19 -19
  113. re_common/libtest/base_dict_test.py +19 -19
  114. re_common/libtest/baseavro_test.py +13 -13
  115. re_common/libtest/basefile_test.py +14 -14
  116. re_common/libtest/basemssql_test.py +77 -77
  117. re_common/libtest/baseodbc_test.py +7 -7
  118. re_common/libtest/basepandas_test.py +38 -38
  119. re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
  120. re_common/libtest/get_attr_test/settings.py +54 -54
  121. re_common/libtest/idencode_test.py +53 -53
  122. re_common/libtest/iniconfig_test.py +35 -35
  123. re_common/libtest/ip_test.py +34 -34
  124. re_common/libtest/merge_file_test.py +20 -20
  125. re_common/libtest/mfaker_test.py +8 -8
  126. re_common/libtest/mm3_test.py +31 -31
  127. re_common/libtest/mylogger_test.py +88 -88
  128. re_common/libtest/myparsel_test.py +27 -27
  129. re_common/libtest/mysql_test.py +151 -151
  130. re_common/libtest/pymongo_test.py +21 -21
  131. re_common/libtest/split_test.py +11 -11
  132. re_common/libtest/sqlite3_merge_test.py +5 -5
  133. re_common/libtest/sqlite3_test.py +34 -34
  134. re_common/libtest/tomlconfig_test.py +30 -30
  135. re_common/libtest/use_tools_test/__init__.py +2 -2
  136. re_common/libtest/user/__init__.py +4 -4
  137. re_common/studio/__init__.py +4 -4
  138. re_common/studio/assignment_expressions.py +36 -36
  139. re_common/studio/mydash/test1.py +18 -18
  140. re_common/studio/pydashstudio/first.py +9 -9
  141. re_common/studio/streamlitstudio/first_app.py +65 -65
  142. re_common/studio/streamlitstudio/uber_pickups.py +23 -23
  143. re_common/studio/test.py +18 -18
  144. re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
  145. re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
  146. re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
  147. re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
  148. re_common/v2/baselibrary/decorators/utils.py +59 -59
  149. re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
  150. re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
  151. re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
  152. re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
  153. re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
  154. re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
  155. re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
  156. re_common/v2/baselibrary/tools/concurrency.py +35 -35
  157. re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
  158. re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
  159. re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
  160. re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
  161. re_common/v2/baselibrary/tools/dict_tools.py +44 -44
  162. re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
  163. re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
  164. re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
  165. re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
  166. re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
  167. re_common/v2/baselibrary/tools/list_tools.py +69 -69
  168. re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
  169. re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
  170. re_common/v2/baselibrary/tools/text_matcher.py +326 -326
  171. re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
  172. re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
  173. re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
  174. re_common/v2/baselibrary/utils/author_smi.py +361 -361
  175. re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
  176. re_common/v2/baselibrary/utils/basedict.py +37 -37
  177. re_common/v2/baselibrary/utils/basehdfs.py +163 -163
  178. re_common/v2/baselibrary/utils/basepika.py +180 -180
  179. re_common/v2/baselibrary/utils/basetime.py +77 -77
  180. re_common/v2/baselibrary/utils/db.py +156 -156
  181. re_common/v2/baselibrary/utils/elasticsearch.py +46 -0
  182. re_common/v2/baselibrary/utils/json_cls.py +16 -16
  183. re_common/v2/baselibrary/utils/mq.py +83 -83
  184. re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
  185. re_common/v2/baselibrary/utils/string_bool.py +186 -186
  186. re_common/v2/baselibrary/utils/string_clear.py +246 -246
  187. re_common/v2/baselibrary/utils/string_smi.py +18 -18
  188. re_common/v2/baselibrary/utils/stringutils.py +271 -271
  189. re_common/vip/base_step_process.py +11 -11
  190. re_common/vip/baseencodeid.py +90 -90
  191. re_common/vip/changetaskname.py +28 -28
  192. re_common/vip/core_var.py +24 -24
  193. re_common/vip/mmh3Hash.py +89 -89
  194. re_common/vip/proxy/allproxys.py +127 -127
  195. re_common/vip/proxy/allproxys_thread.py +159 -159
  196. re_common/vip/proxy/cnki_proxy.py +153 -153
  197. re_common/vip/proxy/kuaidaili.py +87 -87
  198. re_common/vip/proxy/proxy_all.py +113 -113
  199. re_common/vip/proxy/update_kuaidaili_0.py +42 -42
  200. re_common/vip/proxy/wanfang_proxy.py +152 -152
  201. re_common/vip/proxy/wp_proxy_all.py +181 -181
  202. re_common/vip/read_rawid_to_txt.py +91 -91
  203. re_common/vip/title/__init__.py +5 -5
  204. re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
  205. re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
  206. re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
  207. re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
  208. re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
  209. re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
  210. re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
  211. re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
  212. re_common/vip/title/transform/__init__.py +10 -10
  213. {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/LICENSE +201 -201
  214. {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/METADATA +24 -16
  215. re_common-10.0.40.dist-info/RECORD +249 -0
  216. {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/WHEEL +1 -1
  217. re_common-10.0.39.dist-info/RECORD +0 -248
  218. {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/top_level.txt +0 -0
@@ -1,270 +1,270 @@
1
- import atexit
2
- import sys
3
- import asyncio
4
- import aiohttp
5
- from typing import Optional
6
-
7
- from tenacity import retry, stop_after_attempt, wait_random
8
-
9
- g_headers = {
10
- 'accept': 'application/json',
11
- 'Content-Type': 'application/json',
12
- 'Authorization': 'Bearer eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJ1c2VyX2lkIjotMSwidXNlcl9uYW1lIjoiXHU1ZTk0XHU3NTI4XHU0ZTJkXHU1ZmMzQ2xpZW50In0.'
13
- }
14
-
15
- """
16
- cls._conn = aiohttp.TCPConnector(
17
- limit=50, # 最大连接数
18
- ssl=False, # 禁用SSL验证(按需开启)
19
- force_close=True, # 保持连接活跃
20
- enable_cleanup_closed=True # 自动清理关闭的连接
21
- )
22
- # 由于网络上有重名,没有连接。如果加入域,请转到“控制面板”中的“系统”更改计算机名,然后重试。如果加入工作组,请选择其他工作组名。
23
- 有可能是
24
- force_close=True, # 保持连接活跃
25
- enable_cleanup_closed=True # 自动清理关闭的连接
26
- 照成的
27
- """
28
-
29
-
30
- class HttpError(Exception):
31
- code = 0
32
- message = ""
33
- headers = None
34
-
35
- def __init__(
36
- self,
37
- *,
38
- code: Optional[int] = None,
39
- message: str = "",
40
- headers: Optional[dict] = None,
41
- ) -> None:
42
- if code is not None:
43
- self.code = code
44
- self.headers = headers
45
- self.message = message
46
-
47
- def __str__(self) -> str:
48
- return f"code: {self.code}, message:{self.message}"
49
-
50
- def __repr__(self) -> str:
51
- return f"<{self.__class__.__name__}: code={self.code}, message={self.message!r}>"
52
-
53
-
54
- def on_retry_error(retry_state):
55
- # 最后抛错后调用
56
- original_exc = retry_state.outcome.exception()
57
- print(f"[HTTP 请求重试所有重试失败.] 错误消息{original_exc}")
58
-
59
- raise HttpError(code=getattr(original_exc, 'code', 455),
60
- message=f"错误消息:{str(original_exc)}") from original_exc
61
-
62
-
63
- def on_retry(retry_state):
64
- # 每次抛错进入该函数打印消息
65
- print(
66
- f"[HTTP 请求重试]"
67
- f"当前重试 : 第 {retry_state.attempt_number} 次"
68
- f"睡眠时间 : {retry_state.next_action.sleep:.2f} 秒"
69
- f"\n异常原因 : {retry_state.outcome.exception()}"
70
- )
71
-
72
-
73
- class ApiNetUtils:
74
- """
75
- HTTP请求工具类(异步版),提供GET/POST/PATCH请求方法
76
- 特性:
77
- 1. 自动复用TCP连接池
78
- 2. 自动重试机制(通过async_retry装饰器)
79
- 3. 进程退出时自动清理资源
80
- 4. 线程安全的延迟初始化
81
- """
82
-
83
- # 类属性使用Optional类型注解,初始化为None实现延迟初始化
84
- _conn: Optional[aiohttp.TCPConnector] = None
85
- _session: Optional[aiohttp.ClientSession] = None
86
- _close_registered: bool = False # 确保清理函数只注册一次
87
-
88
- @classmethod
89
- async def _get_connector(cls) -> aiohttp.TCPConnector:
90
- """
91
- 获取TCP连接器(延迟初始化)
92
- 解决模块加载时没有事件循环的问题
93
- """
94
- if cls._conn is None or cls._conn.closed or cls.is_loop_closed(cls._session):
95
- # 只有在首次使用时才创建连接器
96
- cls._conn = aiohttp.TCPConnector(
97
- limit=50, # 最大连接数
98
- ssl=False, # 禁用SSL验证(按需开启)
99
- force_close=True, # 保持连接活跃
100
- # enable_cleanup_closed=True, # 自动清理关闭的连接 #
101
- # keepalive_timeout=4.99 # 比服务器的5s 小一点
102
- )
103
- return cls._conn
104
-
105
- @classmethod
106
- async def _get_session(cls) -> aiohttp.ClientSession:
107
- """
108
- 获取共享会话(线程安全的延迟初始化)
109
- 包含自动注册清理机制
110
- """
111
- if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
112
- if cls._session:
113
- await cls.close()
114
- # 获取连接器(会自动初始化)
115
- connector = await cls._get_connector()
116
-
117
- # 强制获取新的事件循环
118
- loop = asyncio.get_event_loop()
119
-
120
- # 创建新会话
121
- cls._session = aiohttp.ClientSession(
122
- connector=connector,
123
- timeout=aiohttp.ClientTimeout(total=30), # 默认30秒超时
124
- loop=loop) # 显式指定事件循环
125
-
126
- # # 注册退出时的清理钩子
127
- cls._register_cleanup()
128
-
129
- return cls._session
130
-
131
- @staticmethod
132
- def is_loop_closed(session: aiohttp.ClientSession) -> bool:
133
- """
134
- 检查会话绑定的事件循环是否已关闭
135
- """
136
- loop = session._loop # 获取会话绑定的事件循环
137
- if loop.is_closed():
138
- # print("Event loop is closed")
139
- return True
140
- # print("Event loop not is closed")
141
- return False
142
-
143
- @classmethod
144
- def _register_cleanup(cls):
145
- """
146
- 注册进程退出时的资源清理函数
147
- 包含正常退出和异常退出两种情况
148
- """
149
- if not cls._close_registered:
150
- # 1. 正常退出处理
151
- atexit.register(lambda: asyncio.run(cls.close()))
152
-
153
- # 2. 异常退出处理
154
- original_excepthook = sys.excepthook
155
-
156
- def custom_excepthook(exctype, value, traceback):
157
- """自定义异常钩子,确保资源被清理"""
158
- # 先执行原始异常处理(打印堆栈等)
159
- original_excepthook(exctype, value, traceback)
160
- # 然后执行资源清理
161
- try:
162
- asyncio.run(cls.close())
163
- except RuntimeError:
164
- # 如果已经没有事件循环,则同步执行
165
- loop = asyncio.new_event_loop()
166
- loop.run_until_complete(cls.close())
167
- loop.close()
168
-
169
- sys.excepthook = custom_excepthook
170
- cls._close_registered = True
171
-
172
- @classmethod
173
- async def close(cls):
174
- """
175
- 安全关闭所有网络资源
176
- 会自动在程序退出时调用,也可手动调用
177
- """
178
- if cls._session and not cls._session.closed:
179
- await cls._session.close()
180
- cls._session = None
181
-
182
- if cls._conn and not cls._conn.closed:
183
- await cls._conn.close()
184
- cls._conn = None
185
-
186
- # print("[ApiNetUtils] 网络资源已安全释放")
187
-
188
- # -------------------- 公共API方法 -------------------- #
189
-
190
- @classmethod
191
- @retry(stop=stop_after_attempt(4), # 本质上执行4次 但重试3次
192
- wait=wait_random(min=5, max=15),
193
- before_sleep=on_retry, # 每次抛错后使用
194
- retry_error_callback=on_retry_error,
195
- reraise=True)
196
- async def fetch_get(cls, url: str, headers=None, params=None):
197
- """
198
- GET请求封装
199
- :param url: 请求URL
200
- :param headers: 可选请求头(默认使用全局g_headers)
201
- :param params: 查询参数(字典)
202
- :return: 解析后的JSON数据
203
- :raises HttpError: 当状态码非200时抛出
204
- """
205
- headers = headers or g_headers
206
- session = await cls._get_session()
207
-
208
- async with session.get(url, headers=headers, params=params) as response:
209
- if response.status != 200:
210
- error_text = await response.text()
211
- raise HttpError(
212
- code=response.status,
213
- message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
214
- )
215
- return await response.json()
216
-
217
- @classmethod
218
- @retry(stop=stop_after_attempt(4),
219
- wait=wait_random(min=5, max=15),
220
- before_sleep=on_retry, # 每次抛错后使用
221
- retry_error_callback=on_retry_error,
222
- reraise=True)
223
- async def fetch_post(cls, url: str, payload: dict, headers=None):
224
- """
225
- POST请求封装(JSON格式)
226
- """
227
- headers = headers or g_headers
228
- session = await cls._get_session()
229
-
230
- async with session.post(url, json=payload, headers=headers) as response:
231
- if response.status != 200:
232
- error_text = await response.text()
233
- raise HttpError(
234
- code=response.status,
235
- message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
236
- )
237
- return await response.json()
238
-
239
- @classmethod
240
- @retry(stop=stop_after_attempt(4),
241
- wait=wait_random(min=5, max=15),
242
- before_sleep=on_retry, # 每次抛错后使用
243
- retry_error_callback=on_retry_error,
244
- reraise=True)
245
- async def fetch_patch(cls, url: str, payload: dict, headers=None):
246
- """
247
- PATCH请求封装(JSON格式)
248
- """
249
- headers = headers or g_headers
250
- session = await cls._get_session()
251
-
252
- async with session.patch(url, json=payload, headers=headers) as response:
253
- if response.status != 200:
254
- error_text = await response.text()
255
- raise HttpError(
256
- code=response.status,
257
- message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
258
- )
259
- return await response.json()
260
-
261
- @classmethod
262
- async def __aenter__(cls):
263
- """支持async with语法"""
264
- await cls._get_session()
265
- return cls
266
-
267
- @classmethod
268
- async def __aexit__(cls, exc_type, exc, tb):
269
- """async with退出时自动关闭"""
270
- await cls.close()
1
+ import atexit
2
+ import sys
3
+ import asyncio
4
+ import aiohttp
5
+ from typing import Optional
6
+
7
+ from tenacity import retry, stop_after_attempt, wait_random
8
+
9
+ g_headers = {
10
+ 'accept': 'application/json',
11
+ 'Content-Type': 'application/json',
12
+ 'Authorization': 'Bearer eyJhbGciOiJub25lIiwidHlwIjoiSldUIn0.eyJ1c2VyX2lkIjotMSwidXNlcl9uYW1lIjoiXHU1ZTk0XHU3NTI4XHU0ZTJkXHU1ZmMzQ2xpZW50In0.'
13
+ }
14
+
15
+ """
16
+ cls._conn = aiohttp.TCPConnector(
17
+ limit=50, # 最大连接数
18
+ ssl=False, # 禁用SSL验证(按需开启)
19
+ force_close=True, # 保持连接活跃
20
+ enable_cleanup_closed=True # 自动清理关闭的连接
21
+ )
22
+ # 由于网络上有重名,没有连接。如果加入域,请转到“控制面板”中的“系统”更改计算机名,然后重试。如果加入工作组,请选择其他工作组名。
23
+ 有可能是
24
+ force_close=True, # 保持连接活跃
25
+ enable_cleanup_closed=True # 自动清理关闭的连接
26
+ 照成的
27
+ """
28
+
29
+
30
+ class HttpError(Exception):
31
+ code = 0
32
+ message = ""
33
+ headers = None
34
+
35
+ def __init__(
36
+ self,
37
+ *,
38
+ code: Optional[int] = None,
39
+ message: str = "",
40
+ headers: Optional[dict] = None,
41
+ ) -> None:
42
+ if code is not None:
43
+ self.code = code
44
+ self.headers = headers
45
+ self.message = message
46
+
47
+ def __str__(self) -> str:
48
+ return f"code: {self.code}, message:{self.message}"
49
+
50
+ def __repr__(self) -> str:
51
+ return f"<{self.__class__.__name__}: code={self.code}, message={self.message!r}>"
52
+
53
+
54
+ def on_retry_error(retry_state):
55
+ # 最后抛错后调用
56
+ original_exc = retry_state.outcome.exception()
57
+ print(f"[HTTP 请求重试所有重试失败.] 错误消息{original_exc}")
58
+
59
+ raise HttpError(code=getattr(original_exc, 'code', 455),
60
+ message=f"错误消息:{str(original_exc)}") from original_exc
61
+
62
+
63
+ def on_retry(retry_state):
64
+ # 每次抛错进入该函数打印消息
65
+ print(
66
+ f"[HTTP 请求重试]"
67
+ f"当前重试 : 第 {retry_state.attempt_number} 次"
68
+ f"睡眠时间 : {retry_state.next_action.sleep:.2f} 秒"
69
+ f"\n异常原因 : {retry_state.outcome.exception()}"
70
+ )
71
+
72
+
73
+ class ApiNetUtils:
74
+ """
75
+ HTTP请求工具类(异步版),提供GET/POST/PATCH请求方法
76
+ 特性:
77
+ 1. 自动复用TCP连接池
78
+ 2. 自动重试机制(通过async_retry装饰器)
79
+ 3. 进程退出时自动清理资源
80
+ 4. 线程安全的延迟初始化
81
+ """
82
+
83
+ # 类属性使用Optional类型注解,初始化为None实现延迟初始化
84
+ _conn: Optional[aiohttp.TCPConnector] = None
85
+ _session: Optional[aiohttp.ClientSession] = None
86
+ _close_registered: bool = False # 确保清理函数只注册一次
87
+
88
+ @classmethod
89
+ async def _get_connector(cls) -> aiohttp.TCPConnector:
90
+ """
91
+ 获取TCP连接器(延迟初始化)
92
+ 解决模块加载时没有事件循环的问题
93
+ """
94
+ if cls._conn is None or cls._conn.closed or cls.is_loop_closed(cls._session):
95
+ # 只有在首次使用时才创建连接器
96
+ cls._conn = aiohttp.TCPConnector(
97
+ limit=50, # 最大连接数
98
+ ssl=False, # 禁用SSL验证(按需开启)
99
+ force_close=True, # 保持连接活跃
100
+ # enable_cleanup_closed=True, # 自动清理关闭的连接 #
101
+ # keepalive_timeout=4.99 # 比服务器的5s 小一点
102
+ )
103
+ return cls._conn
104
+
105
+ @classmethod
106
+ async def _get_session(cls) -> aiohttp.ClientSession:
107
+ """
108
+ 获取共享会话(线程安全的延迟初始化)
109
+ 包含自动注册清理机制
110
+ """
111
+ if cls._session is None or cls._session.closed or cls.is_loop_closed(cls._session):
112
+ if cls._session:
113
+ await cls.close()
114
+ # 获取连接器(会自动初始化)
115
+ connector = await cls._get_connector()
116
+
117
+ # 强制获取新的事件循环
118
+ loop = asyncio.get_event_loop()
119
+
120
+ # 创建新会话
121
+ cls._session = aiohttp.ClientSession(
122
+ connector=connector,
123
+ timeout=aiohttp.ClientTimeout(total=30), # 默认30秒超时
124
+ loop=loop) # 显式指定事件循环
125
+
126
+ # # 注册退出时的清理钩子
127
+ cls._register_cleanup()
128
+
129
+ return cls._session
130
+
131
+ @staticmethod
132
+ def is_loop_closed(session: aiohttp.ClientSession) -> bool:
133
+ """
134
+ 检查会话绑定的事件循环是否已关闭
135
+ """
136
+ loop = session._loop # 获取会话绑定的事件循环
137
+ if loop.is_closed():
138
+ # print("Event loop is closed")
139
+ return True
140
+ # print("Event loop not is closed")
141
+ return False
142
+
143
+ @classmethod
144
+ def _register_cleanup(cls):
145
+ """
146
+ 注册进程退出时的资源清理函数
147
+ 包含正常退出和异常退出两种情况
148
+ """
149
+ if not cls._close_registered:
150
+ # 1. 正常退出处理
151
+ atexit.register(lambda: asyncio.run(cls.close()))
152
+
153
+ # 2. 异常退出处理
154
+ original_excepthook = sys.excepthook
155
+
156
+ def custom_excepthook(exctype, value, traceback):
157
+ """自定义异常钩子,确保资源被清理"""
158
+ # 先执行原始异常处理(打印堆栈等)
159
+ original_excepthook(exctype, value, traceback)
160
+ # 然后执行资源清理
161
+ try:
162
+ asyncio.run(cls.close())
163
+ except RuntimeError:
164
+ # 如果已经没有事件循环,则同步执行
165
+ loop = asyncio.new_event_loop()
166
+ loop.run_until_complete(cls.close())
167
+ loop.close()
168
+
169
+ sys.excepthook = custom_excepthook
170
+ cls._close_registered = True
171
+
172
+ @classmethod
173
+ async def close(cls):
174
+ """
175
+ 安全关闭所有网络资源
176
+ 会自动在程序退出时调用,也可手动调用
177
+ """
178
+ if cls._session and not cls._session.closed:
179
+ await cls._session.close()
180
+ cls._session = None
181
+
182
+ if cls._conn and not cls._conn.closed:
183
+ await cls._conn.close()
184
+ cls._conn = None
185
+
186
+ # print("[ApiNetUtils] 网络资源已安全释放")
187
+
188
+ # -------------------- 公共API方法 -------------------- #
189
+
190
+ @classmethod
191
+ @retry(stop=stop_after_attempt(4), # 本质上执行4次 但重试3次
192
+ wait=wait_random(min=5, max=15),
193
+ before_sleep=on_retry, # 每次抛错后使用
194
+ retry_error_callback=on_retry_error,
195
+ reraise=True)
196
+ async def fetch_get(cls, url: str, headers=None, params=None):
197
+ """
198
+ GET请求封装
199
+ :param url: 请求URL
200
+ :param headers: 可选请求头(默认使用全局g_headers)
201
+ :param params: 查询参数(字典)
202
+ :return: 解析后的JSON数据
203
+ :raises HttpError: 当状态码非200时抛出
204
+ """
205
+ headers = headers or g_headers
206
+ session = await cls._get_session()
207
+
208
+ async with session.get(url, headers=headers, params=params) as response:
209
+ if response.status != 200:
210
+ error_text = await response.text()
211
+ raise HttpError(
212
+ code=response.status,
213
+ message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
214
+ )
215
+ return await response.json()
216
+
217
+ @classmethod
218
+ @retry(stop=stop_after_attempt(4),
219
+ wait=wait_random(min=5, max=15),
220
+ before_sleep=on_retry, # 每次抛错后使用
221
+ retry_error_callback=on_retry_error,
222
+ reraise=True)
223
+ async def fetch_post(cls, url: str, payload: dict, headers=None):
224
+ """
225
+ POST请求封装(JSON格式)
226
+ """
227
+ headers = headers or g_headers
228
+ session = await cls._get_session()
229
+
230
+ async with session.post(url, json=payload, headers=headers) as response:
231
+ if response.status != 200:
232
+ error_text = await response.text()
233
+ raise HttpError(
234
+ code=response.status,
235
+ message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
236
+ )
237
+ return await response.json()
238
+
239
+ @classmethod
240
+ @retry(stop=stop_after_attempt(4),
241
+ wait=wait_random(min=5, max=15),
242
+ before_sleep=on_retry, # 每次抛错后使用
243
+ retry_error_callback=on_retry_error,
244
+ reraise=True)
245
+ async def fetch_patch(cls, url: str, payload: dict, headers=None):
246
+ """
247
+ PATCH请求封装(JSON格式)
248
+ """
249
+ headers = headers or g_headers
250
+ session = await cls._get_session()
251
+
252
+ async with session.patch(url, json=payload, headers=headers) as response:
253
+ if response.status != 200:
254
+ error_text = await response.text()
255
+ raise HttpError(
256
+ code=response.status,
257
+ message=f"请求失败: url={url}, status={response.status}, 错误详情={error_text}"
258
+ )
259
+ return await response.json()
260
+
261
+ @classmethod
262
+ async def __aenter__(cls):
263
+ """支持async with语法"""
264
+ await cls._get_session()
265
+ return cls
266
+
267
+ @classmethod
268
+ async def __aexit__(cls, exc_type, exc, tb):
269
+ """async with退出时自动关闭"""
270
+ await cls.close()