sycommon-python-lib 0.1.48__py3-none-any.whl → 0.1.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/rabbitmq/rabbitmq_pool.py +148 -317
- {sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/RECORD +6 -6
- {sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/top_level.txt +0 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
|
|
2
|
+
import random
|
|
3
|
+
from typing import Optional, List, Set, Tuple, Dict, Callable
|
|
3
4
|
from aio_pika import connect_robust, Channel, Message
|
|
4
5
|
from aio_pika.abc import (
|
|
5
6
|
AbstractRobustConnection, AbstractQueue, AbstractExchange, AbstractMessage
|
|
@@ -13,7 +14,7 @@ logger = SYLogger
|
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class RabbitMQConnectionPool:
|
|
16
|
-
"""单连接RabbitMQ
|
|
17
|
+
"""单连接RabbitMQ通道池(核心特性:依赖connect_robust原生自动重连/恢复 + 仅关闭时释放资源 + 全场景加锁)"""
|
|
17
18
|
|
|
18
19
|
def __init__(
|
|
19
20
|
self,
|
|
@@ -45,273 +46,133 @@ class RabbitMQConnectionPool:
|
|
|
45
46
|
self.prefetch_count = prefetch_count
|
|
46
47
|
self.channel_pool_size = max(1, channel_pool_size) # 确保池大小不小于1
|
|
47
48
|
|
|
48
|
-
#
|
|
49
|
-
self.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
56
|
-
self.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
49
|
+
# 初始化时随机选择一个主机地址(固定使用,依赖原生重连)
|
|
50
|
+
self._current_host: str = random.choice(self.hosts)
|
|
51
|
+
logger.info(
|
|
52
|
+
f"随机选择RabbitMQ主机: {self._current_host}(依赖connect_robust原生自动重连/恢复)")
|
|
53
|
+
|
|
54
|
+
# 核心资源(单连接+通道池,基于原生自动重连)
|
|
55
|
+
self._connection: Optional[AbstractRobustConnection] = None # 原生自动重连连接
|
|
56
|
+
self._free_channels: List[Channel] = [] # 空闲通道(原生自动恢复)
|
|
57
|
+
self._used_channels: Set[Channel] = set() # 使用中通道(原生自动恢复)
|
|
58
|
+
# 消费者通道跟踪
|
|
59
|
+
self._consumer_channels: Dict[str,
|
|
60
|
+
Tuple[Channel, Callable, bool, dict]] = {}
|
|
61
|
+
|
|
62
|
+
# 状态控制(并发安全+生命周期管理)
|
|
63
|
+
self._lock = asyncio.Lock()
|
|
61
64
|
self._initialized = False
|
|
62
65
|
self._is_shutdown = False
|
|
63
|
-
self._reconnecting = False # 避免并发重连
|
|
64
|
-
self._connection_version = 0 # 连接版本号(区分新旧连接/通道)
|
|
65
|
-
|
|
66
|
-
def _create_host_iterator(self) -> Iterator[str]:
|
|
67
|
-
"""创建无限循环的节点轮询迭代器"""
|
|
68
|
-
while True:
|
|
69
|
-
for host in self.hosts:
|
|
70
|
-
yield host
|
|
71
66
|
|
|
72
67
|
async def _is_connection_valid(self) -> bool:
|
|
73
|
-
"""
|
|
68
|
+
"""原子化检查连接有效性(仅判断是否初始化完成且未关闭)"""
|
|
74
69
|
async with self._lock:
|
|
75
70
|
return (
|
|
76
|
-
self.
|
|
71
|
+
self._initialized
|
|
72
|
+
and self._connection is not None
|
|
77
73
|
and not self._connection.is_closed
|
|
78
|
-
and not self.
|
|
74
|
+
and not self._is_shutdown
|
|
79
75
|
)
|
|
80
76
|
|
|
81
77
|
@property
|
|
82
78
|
async def is_alive(self) -> bool:
|
|
83
|
-
"""
|
|
79
|
+
"""对外暴露的连接存活状态"""
|
|
84
80
|
if self._is_shutdown:
|
|
85
81
|
return False
|
|
86
82
|
return await self._is_connection_valid()
|
|
87
83
|
|
|
88
|
-
async def
|
|
89
|
-
"""
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
# 1. 关闭所有消费者通道(独立管理,终止旧自动恢复)
|
|
94
|
-
for queue_name, (channel, _, _, _, _) in self._consumer_channels.items():
|
|
95
|
-
try:
|
|
96
|
-
if not channel.is_closed:
|
|
97
|
-
await channel.close()
|
|
98
|
-
logger.info(f"已关闭队列 {queue_name} 的旧消费者通道(自动恢复终止)")
|
|
99
|
-
except Exception as e:
|
|
100
|
-
logger.warning(f"关闭消费者通道 {queue_name} 失败: {str(e)}")
|
|
101
|
-
self._consumer_channels.clear()
|
|
102
|
-
|
|
103
|
-
# 2. 关闭所有普通通道(空闲+使用中,终止旧自动恢复)
|
|
104
|
-
all_channels = self._free_channels + list(self._used_channels)
|
|
105
|
-
for channel in all_channels:
|
|
106
|
-
try:
|
|
107
|
-
if not channel.is_closed:
|
|
108
|
-
await channel.close()
|
|
109
|
-
except Exception as e:
|
|
110
|
-
logger.warning(f"关闭旧通道失败: {str(e)}")
|
|
111
|
-
self._free_channels.clear()
|
|
112
|
-
self._used_channels.clear()
|
|
113
|
-
|
|
114
|
-
# 3. 强制关闭旧连接(彻底终止旧连接的所有自动恢复)
|
|
115
|
-
if self._connection:
|
|
116
|
-
try:
|
|
117
|
-
if not self._connection.is_closed:
|
|
118
|
-
await self._connection.close()
|
|
119
|
-
logger.info(
|
|
120
|
-
f"已关闭旧连接: {self._current_host}:{self.port}(版本: {self._connection_version})")
|
|
121
|
-
except Exception as e:
|
|
122
|
-
logger.warning(f"关闭旧连接失败: {str(e)}")
|
|
123
|
-
self._connection = None # 置空,确保单连接
|
|
124
|
-
|
|
125
|
-
logger.info("旧资源释放完成(所有旧自动恢复逻辑已终止)")
|
|
126
|
-
|
|
127
|
-
async def _create_single_connection(self) -> AbstractRobustConnection:
|
|
128
|
-
"""创建唯一活跃连接(重连前已释放旧资源,确保单连接)"""
|
|
129
|
-
max_attempts = len(self.hosts) * 2 # 每个节点尝试2次
|
|
130
|
-
attempts = 0
|
|
131
|
-
last_error: Optional[Exception] = None
|
|
84
|
+
async def _create_connection(self) -> AbstractRobustConnection:
|
|
85
|
+
"""创建原生自动重连连接(仅创建一次,内部自动重试)"""
|
|
86
|
+
if self._is_shutdown:
|
|
87
|
+
raise RuntimeError("通道池已关闭,无法创建连接")
|
|
132
88
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
conn_url = f"amqp://{self.username}:{self.password}@{self._current_host}:{self.port}/{self.virtualhost}"
|
|
89
|
+
conn_url = f"amqp://{self.username}:{self.password}@{self._current_host}:{self.port}/{self.virtualhost}"
|
|
90
|
+
logger.info(f"尝试创建原生自动重连连接: {self._current_host}:{self.port}")
|
|
136
91
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
f"
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
)
|
|
155
|
-
logger.info(
|
|
156
|
-
f"连接创建成功: {self._current_host}:{self.port}(版本: {target_version})")
|
|
157
|
-
return conn
|
|
158
|
-
except Exception as e:
|
|
159
|
-
attempts += 1
|
|
160
|
-
last_error = e
|
|
161
|
-
logger.error(
|
|
162
|
-
f"连接节点 {self._current_host}:{self.port} 失败({attempts}/{max_attempts}): {str(e)}",
|
|
163
|
-
exc_info=True
|
|
164
|
-
)
|
|
165
|
-
await asyncio.sleep(min(5 * attempts, self.reconnect_interval))
|
|
166
|
-
|
|
167
|
-
raise ConnectionError(
|
|
168
|
-
f"所有节点创建连接失败(节点列表: {self.hosts})"
|
|
169
|
-
) from last_error
|
|
92
|
+
try:
|
|
93
|
+
conn = await connect_robust(
|
|
94
|
+
conn_url,
|
|
95
|
+
properties={
|
|
96
|
+
"connection_name": f"{self.app_name}_conn",
|
|
97
|
+
"product": self.app_name
|
|
98
|
+
},
|
|
99
|
+
heartbeat=self.heartbeat,
|
|
100
|
+
timeout=self.connection_timeout,
|
|
101
|
+
reconnect_interval=self.reconnect_interval, # 原生重连间隔
|
|
102
|
+
max_reconnect_attempts=None, # 无限重试(按需调整)
|
|
103
|
+
)
|
|
104
|
+
logger.info(f"连接创建成功: {self._current_host}:{self.port}(原生自动重连已启用)")
|
|
105
|
+
return conn
|
|
106
|
+
except Exception as e:
|
|
107
|
+
logger.error(f"连接创建失败: {str(e)}", exc_info=True)
|
|
108
|
+
raise ConnectionError(
|
|
109
|
+
f"无法连接RabbitMQ主机 {self._current_host}:{self.port}") from e
|
|
170
110
|
|
|
171
111
|
async def _init_channel_pool(self):
|
|
172
|
-
"""
|
|
112
|
+
"""初始化通道池(通道自带原生自动恢复)"""
|
|
173
113
|
async with self._lock:
|
|
174
114
|
if self._is_shutdown:
|
|
175
115
|
raise RuntimeError("通道池已关闭,无法初始化")
|
|
176
|
-
|
|
177
|
-
# 校验当前连接有效性
|
|
178
116
|
if not self._connection or self._connection.is_closed:
|
|
179
117
|
raise RuntimeError("无有效连接,无法初始化通道池")
|
|
180
118
|
|
|
119
|
+
# 清理无效通道(初始化前确保池为空)
|
|
181
120
|
self._free_channels.clear()
|
|
182
121
|
self._used_channels.clear()
|
|
183
122
|
|
|
184
|
-
#
|
|
123
|
+
# 创建指定数量的通道
|
|
185
124
|
for i in range(self.channel_pool_size):
|
|
186
125
|
try:
|
|
187
|
-
channel = await self._connection.channel() #
|
|
126
|
+
channel = await self._connection.channel() # 通道原生自动恢复
|
|
188
127
|
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
189
128
|
self._free_channels.append(channel)
|
|
190
129
|
except Exception as e:
|
|
191
|
-
logger.error(
|
|
192
|
-
f"创建通道失败(第{i+1}个,连接版本: {self._connection_version}): {str(e)}",
|
|
193
|
-
exc_info=True
|
|
194
|
-
)
|
|
130
|
+
logger.error(f"创建通道失败(第{i+1}个): {str(e)}", exc_info=True)
|
|
195
131
|
continue
|
|
196
132
|
|
|
197
133
|
logger.info(
|
|
198
|
-
f"通道池初始化完成 -
|
|
199
|
-
f"
|
|
134
|
+
f"通道池初始化完成 - 可用通道数: {len(self._free_channels)}/{self.channel_pool_size} "
|
|
135
|
+
f"(均带原生自动恢复)"
|
|
200
136
|
)
|
|
201
137
|
|
|
202
|
-
async def _reconnect_if_needed(self) -> bool:
|
|
203
|
-
"""连接失效时重连(加锁保护,严格单连接+释放旧资源)"""
|
|
204
|
-
# 快速判断,避免无效加锁
|
|
205
|
-
if self._is_shutdown or self._reconnecting:
|
|
206
|
-
return False
|
|
207
|
-
|
|
208
|
-
self._reconnecting = True
|
|
209
|
-
try:
|
|
210
|
-
logger.warning(f"连接失效(当前版本: {self._connection_version}),开始重连...")
|
|
211
|
-
|
|
212
|
-
# 1. 强制释放所有旧资源(加锁保证原子性)
|
|
213
|
-
await self._safe_close_old_resources()
|
|
214
|
-
|
|
215
|
-
# 2. 递增连接版本号(加锁保证原子性,区分新旧连接)
|
|
216
|
-
async with self._lock:
|
|
217
|
-
self._connection_version += 1
|
|
218
|
-
target_version = self._connection_version
|
|
219
|
-
|
|
220
|
-
# 3. 创建新连接(保留原生自动恢复)
|
|
221
|
-
new_conn = await self._create_single_connection()
|
|
222
|
-
|
|
223
|
-
# 4. 绑定新连接(加锁保证原子性)
|
|
224
|
-
async with self._lock:
|
|
225
|
-
self._connection = new_conn
|
|
226
|
-
|
|
227
|
-
# 5. 重新初始化通道池(新通道带自动恢复)
|
|
228
|
-
await self._init_channel_pool()
|
|
229
|
-
|
|
230
|
-
# 6. 恢复消费者通道(新通道带自动恢复)
|
|
231
|
-
await self._restore_consumer_channels()
|
|
232
|
-
|
|
233
|
-
logger.info(f"重连成功(新连接版本: {target_version}),所有通道均带自动恢复")
|
|
234
|
-
async with self._lock:
|
|
235
|
-
self._initialized = True # 重连成功后标记为已初始化
|
|
236
|
-
return True
|
|
237
|
-
except Exception as e:
|
|
238
|
-
logger.error(f"重连失败: {str(e)}", exc_info=True)
|
|
239
|
-
async with self._lock:
|
|
240
|
-
self._initialized = False
|
|
241
|
-
return False
|
|
242
|
-
finally:
|
|
243
|
-
self._reconnecting = False
|
|
244
|
-
|
|
245
|
-
async def _restore_consumer_channels(self):
|
|
246
|
-
"""重连后恢复消费者通道(加锁保证原子性,新通道带自动恢复)"""
|
|
247
|
-
async with self._lock:
|
|
248
|
-
if not self._consumer_channels or not self._connection or self._connection.is_closed:
|
|
249
|
-
return
|
|
250
|
-
logger.info(
|
|
251
|
-
f"开始恢复 {len(self._consumer_channels)} 个消费者通道(连接版本: {self._connection_version})")
|
|
252
|
-
|
|
253
|
-
# 临时保存消费者配置(队列名、回调、auto_ack、参数)
|
|
254
|
-
consumer_configs = list(self._consumer_channels.items())
|
|
255
|
-
self._consumer_channels.clear()
|
|
256
|
-
|
|
257
|
-
# 重新创建消费者(不加锁,避免阻塞其他操作)
|
|
258
|
-
for queue_name, (_, _, callback, auto_ack, kwargs) in consumer_configs:
|
|
259
|
-
try:
|
|
260
|
-
await self.consume_queue(queue_name, callback, auto_ack, **kwargs)
|
|
261
|
-
except Exception as e:
|
|
262
|
-
logger.error(
|
|
263
|
-
f"恢复消费者队列 {queue_name} 失败: {str(e)}", exc_info=True)
|
|
264
|
-
|
|
265
138
|
async def _clean_invalid_channels(self):
|
|
266
|
-
"""
|
|
139
|
+
"""清理失效通道并补充(仅处理通道级失效,依赖原生恢复)"""
|
|
267
140
|
async with self._lock:
|
|
268
|
-
if self._is_shutdown or self.
|
|
269
|
-
return
|
|
270
|
-
|
|
271
|
-
# 1. 校验当前连接有效性
|
|
272
|
-
current_valid = (
|
|
273
|
-
self._connection is not None
|
|
274
|
-
and not self._connection.is_closed
|
|
275
|
-
and not self._reconnecting
|
|
276
|
-
)
|
|
277
|
-
if not current_valid:
|
|
278
|
-
# 连接失效,触发重连(不加锁,避免死锁)
|
|
279
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
141
|
+
if self._is_shutdown or not self._connection:
|
|
280
142
|
return
|
|
281
143
|
|
|
282
|
-
#
|
|
144
|
+
# 1. 清理空闲通道(保留有效通道)
|
|
283
145
|
valid_free = []
|
|
284
146
|
for chan in self._free_channels:
|
|
285
147
|
try:
|
|
286
|
-
if not chan.is_closed
|
|
148
|
+
if not chan.is_closed:
|
|
287
149
|
valid_free.append(chan)
|
|
288
150
|
else:
|
|
289
|
-
logger.warning(f"
|
|
151
|
+
logger.warning(f"清理失效空闲通道(将自动补充)")
|
|
290
152
|
except Exception:
|
|
291
153
|
logger.warning(f"清理异常空闲通道")
|
|
292
154
|
self._free_channels = valid_free
|
|
293
155
|
|
|
294
|
-
#
|
|
156
|
+
# 2. 清理使用中通道(保留有效通道)
|
|
295
157
|
valid_used = set()
|
|
296
158
|
for chan in self._used_channels:
|
|
297
159
|
try:
|
|
298
|
-
if not chan.is_closed
|
|
160
|
+
if not chan.is_closed:
|
|
299
161
|
valid_used.add(chan)
|
|
300
162
|
else:
|
|
301
|
-
logger.warning(f"
|
|
163
|
+
logger.warning(f"清理失效使用中通道")
|
|
302
164
|
except Exception:
|
|
303
165
|
logger.warning(f"清理异常使用中通道")
|
|
304
166
|
self._used_channels = valid_used
|
|
305
167
|
|
|
306
|
-
#
|
|
168
|
+
# 3. 补充缺失的通道(新通道带原生自动恢复)
|
|
307
169
|
total_valid = len(self._free_channels) + len(self._used_channels)
|
|
308
170
|
missing = self.channel_pool_size - total_valid
|
|
309
171
|
if missing > 0:
|
|
310
|
-
logger.info(
|
|
311
|
-
f"通道池缺少{missing}个通道,补充中(连接版本: {self._connection_version})...")
|
|
172
|
+
logger.info(f"通道池缺少{missing}个通道,补充中...")
|
|
312
173
|
for _ in range(missing):
|
|
313
174
|
try:
|
|
314
|
-
channel = await self._connection.channel()
|
|
175
|
+
channel = await self._connection.channel()
|
|
315
176
|
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
316
177
|
self._free_channels.append(channel)
|
|
317
178
|
except Exception as e:
|
|
@@ -319,7 +180,7 @@ class RabbitMQConnectionPool:
|
|
|
319
180
|
break
|
|
320
181
|
|
|
321
182
|
async def init_pools(self):
|
|
322
|
-
"""
|
|
183
|
+
"""初始化通道池(仅执行一次)"""
|
|
323
184
|
async with self._lock:
|
|
324
185
|
if self._initialized:
|
|
325
186
|
logger.warning("通道池已初始化,无需重复调用")
|
|
@@ -328,88 +189,62 @@ class RabbitMQConnectionPool:
|
|
|
328
189
|
raise RuntimeError("通道池已关闭,无法初始化")
|
|
329
190
|
|
|
330
191
|
try:
|
|
331
|
-
# 1.
|
|
332
|
-
|
|
192
|
+
# 1. 创建原生自动重连连接
|
|
193
|
+
self._connection = await self._create_connection()
|
|
333
194
|
|
|
334
|
-
# 2.
|
|
335
|
-
async with self._lock:
|
|
336
|
-
self._connection_version += 1
|
|
337
|
-
self._connection = new_conn
|
|
338
|
-
|
|
339
|
-
# 3. 初始化通道池(新通道带自动恢复)
|
|
195
|
+
# 2. 初始化通道池
|
|
340
196
|
await self._init_channel_pool()
|
|
341
197
|
|
|
342
|
-
#
|
|
198
|
+
# 3. 标记为已初始化
|
|
343
199
|
async with self._lock:
|
|
344
200
|
self._initialized = True
|
|
345
201
|
|
|
346
|
-
logger.info("RabbitMQ
|
|
202
|
+
logger.info("RabbitMQ通道池初始化完成(原生自动重连/恢复已启用)")
|
|
347
203
|
except Exception as e:
|
|
348
204
|
logger.error(f"初始化失败: {str(e)}", exc_info=True)
|
|
349
|
-
await self.
|
|
205
|
+
await self.close() # 初始化失败直接关闭
|
|
350
206
|
raise
|
|
351
207
|
|
|
352
208
|
async def acquire_channel(self) -> Tuple[Channel, AbstractRobustConnection]:
|
|
353
|
-
"""
|
|
354
|
-
#
|
|
209
|
+
"""获取通道(返回 (通道, 连接) 元组,保持API兼容)"""
|
|
210
|
+
# 快速校验状态
|
|
355
211
|
async with self._lock:
|
|
356
212
|
if not self._initialized:
|
|
357
213
|
raise RuntimeError("通道池未初始化,请先调用init_pools()")
|
|
358
214
|
if self._is_shutdown:
|
|
359
215
|
raise RuntimeError("通道池已关闭,无法获取通道")
|
|
216
|
+
if not self._connection or self._connection.is_closed:
|
|
217
|
+
raise RuntimeError("连接已关闭(等待原生重连)")
|
|
360
218
|
|
|
361
|
-
#
|
|
219
|
+
# 清理失效通道
|
|
362
220
|
await self._clean_invalid_channels()
|
|
363
221
|
|
|
364
222
|
async with self._lock:
|
|
365
|
-
#
|
|
366
|
-
current_valid = (
|
|
367
|
-
self._connection is not None
|
|
368
|
-
and not self._connection.is_closed
|
|
369
|
-
and not self._reconnecting
|
|
370
|
-
)
|
|
371
|
-
if not current_valid:
|
|
372
|
-
# 连接失效,触发重连(不加锁,避免死锁)
|
|
373
|
-
reconnect_success = await self._reconnect_if_needed()
|
|
374
|
-
if not reconnect_success:
|
|
375
|
-
raise RuntimeError("连接失效且重连失败,无法获取通道")
|
|
376
|
-
|
|
377
|
-
# 优先从空闲池获取(带自动恢复的通道)
|
|
223
|
+
# 优先从空闲池获取
|
|
378
224
|
if self._free_channels:
|
|
379
225
|
channel = self._free_channels.pop()
|
|
380
226
|
self._used_channels.add(channel)
|
|
381
|
-
return channel, self._connection
|
|
227
|
+
return channel, self._connection # 返回 (通道, 连接) 元组
|
|
382
228
|
|
|
383
|
-
#
|
|
229
|
+
# 池满时创建临时通道(用完自动关闭)
|
|
384
230
|
try:
|
|
385
|
-
channel = await self._connection.channel()
|
|
231
|
+
channel = await self._connection.channel()
|
|
386
232
|
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
387
233
|
self._used_channels.add(channel)
|
|
388
234
|
logger.warning(
|
|
389
|
-
f"通道池已达上限({self.channel_pool_size}
|
|
235
|
+
f"通道池已达上限({self.channel_pool_size}),创建临时通道(用完自动关闭)"
|
|
390
236
|
)
|
|
391
|
-
return channel, self._connection
|
|
237
|
+
return channel, self._connection # 返回 (通道, 连接) 元组
|
|
392
238
|
except Exception as e:
|
|
393
239
|
logger.error(f"获取通道失败: {str(e)}", exc_info=True)
|
|
394
240
|
raise
|
|
395
241
|
|
|
396
242
|
async def release_channel(self, channel: Channel, conn: AbstractRobustConnection):
|
|
397
|
-
"""
|
|
398
|
-
# 快速判断,避免无效加锁
|
|
243
|
+
"""释放通道(接收通道和连接参数,保持API兼容)"""
|
|
399
244
|
if not channel or not conn or self._is_shutdown:
|
|
400
245
|
return
|
|
401
246
|
|
|
402
247
|
async with self._lock:
|
|
403
|
-
# 仅处理当前连接的通道(旧连接的通道直接关闭)
|
|
404
|
-
if conn != self._connection:
|
|
405
|
-
try:
|
|
406
|
-
if not channel.is_closed:
|
|
407
|
-
await channel.close()
|
|
408
|
-
logger.warning(f"已关闭非当前连接的通道(版本不匹配,自动恢复终止)")
|
|
409
|
-
except Exception as e:
|
|
410
|
-
logger.warning(f"关闭非当前连接通道失败: {str(e)}")
|
|
411
|
-
return
|
|
412
|
-
|
|
413
248
|
# 通道不在使用中,直接返回
|
|
414
249
|
if channel not in self._used_channels:
|
|
415
250
|
return
|
|
@@ -417,16 +252,13 @@ class RabbitMQConnectionPool:
|
|
|
417
252
|
# 移除使用中标记
|
|
418
253
|
self._used_channels.remove(channel)
|
|
419
254
|
|
|
420
|
-
#
|
|
421
|
-
|
|
422
|
-
self.
|
|
423
|
-
|
|
424
|
-
and not self._reconnecting
|
|
425
|
-
)
|
|
426
|
-
if current_valid and not channel.is_closed and len(self._free_channels) < self.channel_pool_size:
|
|
255
|
+
# 仅归还有效通道(通道未关闭+池未满+连接匹配)
|
|
256
|
+
if (not channel.is_closed
|
|
257
|
+
and len(self._free_channels) < self.channel_pool_size
|
|
258
|
+
and conn == self._connection): # 确保是当前连接的通道
|
|
427
259
|
self._free_channels.append(channel)
|
|
428
260
|
else:
|
|
429
|
-
#
|
|
261
|
+
# 无效通道直接关闭(临时通道、池满或连接不匹配时)
|
|
430
262
|
try:
|
|
431
263
|
if not channel.is_closed:
|
|
432
264
|
await channel.close()
|
|
@@ -434,7 +266,7 @@ class RabbitMQConnectionPool:
|
|
|
434
266
|
logger.warning(f"关闭通道失败: {str(e)}")
|
|
435
267
|
|
|
436
268
|
async def declare_queue(self, queue_name: str, **kwargs) -> AbstractQueue:
|
|
437
|
-
"""
|
|
269
|
+
"""声明队列(使用池内通道)"""
|
|
438
270
|
channel, conn = await self.acquire_channel()
|
|
439
271
|
try:
|
|
440
272
|
return await channel.declare_queue(queue_name, **kwargs)
|
|
@@ -442,7 +274,7 @@ class RabbitMQConnectionPool:
|
|
|
442
274
|
await self.release_channel(channel, conn)
|
|
443
275
|
|
|
444
276
|
async def declare_exchange(self, exchange_name: str, exchange_type: str = "direct", **kwargs) -> AbstractExchange:
|
|
445
|
-
"""
|
|
277
|
+
"""声明交换机(使用池内通道)"""
|
|
446
278
|
channel, conn = await self.acquire_channel()
|
|
447
279
|
try:
|
|
448
280
|
return await channel.declare_exchange(exchange_name, exchange_type, **kwargs)
|
|
@@ -450,27 +282,24 @@ class RabbitMQConnectionPool:
|
|
|
450
282
|
await self.release_channel(channel, conn)
|
|
451
283
|
|
|
452
284
|
async def publish_message(self, routing_key: str, message_body: bytes, exchange_name: str = "", **kwargs):
|
|
453
|
-
"""
|
|
285
|
+
"""发布消息(依赖原生自动重连/恢复)"""
|
|
454
286
|
channel, conn = await self.acquire_channel()
|
|
455
287
|
try:
|
|
456
288
|
exchange = channel.default_exchange if not exchange_name else await channel.get_exchange(exchange_name)
|
|
457
289
|
message = Message(body=message_body, **kwargs)
|
|
458
290
|
await exchange.publish(message, routing_key=routing_key)
|
|
459
291
|
logger.debug(
|
|
460
|
-
f"消息发布成功 -
|
|
461
|
-
f"交换机: {exchange.name}, 路由键: {routing_key}"
|
|
292
|
+
f"消息发布成功 - 交换机: {exchange.name}, 路由键: {routing_key}"
|
|
462
293
|
)
|
|
463
294
|
except Exception as e:
|
|
464
295
|
logger.error(f"发布消息失败: {str(e)}", exc_info=True)
|
|
465
|
-
#
|
|
466
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
467
|
-
raise
|
|
296
|
+
raise # 原生会自动重连,无需手动处理
|
|
468
297
|
finally:
|
|
469
298
|
await self.release_channel(channel, conn)
|
|
470
299
|
|
|
471
300
|
async def consume_queue(self, queue_name: str, callback: Callable[[AbstractMessage], asyncio.Future], auto_ack: bool = False, **kwargs):
|
|
472
|
-
"""
|
|
473
|
-
#
|
|
301
|
+
"""消费队列(独立通道,带原生自动恢复)"""
|
|
302
|
+
# 快速校验状态
|
|
474
303
|
async with self._lock:
|
|
475
304
|
if not self._initialized:
|
|
476
305
|
raise RuntimeError("通道池未初始化,请先调用init_pools()")
|
|
@@ -480,45 +309,25 @@ class RabbitMQConnectionPool:
|
|
|
480
309
|
# 先声明队列(确保队列存在)
|
|
481
310
|
await self.declare_queue(queue_name, **kwargs)
|
|
482
311
|
|
|
483
|
-
#
|
|
312
|
+
# 获取独立通道(不放入普通池)
|
|
484
313
|
channel, conn = await self.acquire_channel()
|
|
485
314
|
|
|
486
|
-
#
|
|
315
|
+
# 注册消费者通道
|
|
487
316
|
async with self._lock:
|
|
488
317
|
self._consumer_channels[queue_name] = (
|
|
489
|
-
channel,
|
|
318
|
+
channel, callback, auto_ack, kwargs)
|
|
490
319
|
|
|
491
320
|
async def consume_callback_wrapper(message: AbstractMessage):
|
|
492
|
-
"""
|
|
321
|
+
"""消费回调包装(处理通道失效,依赖原生恢复)"""
|
|
493
322
|
try:
|
|
494
|
-
#
|
|
323
|
+
# 校验通道和连接状态
|
|
495
324
|
async with self._lock:
|
|
496
|
-
conn_matched = conn == self._connection
|
|
497
325
|
channel_valid = not channel.is_closed
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
)
|
|
503
|
-
|
|
504
|
-
# 通道/连接失效,触发重连恢复
|
|
505
|
-
if not conn_matched or not channel_valid or not current_conn_valid:
|
|
506
|
-
logger.warning(
|
|
507
|
-
f"消费者通道 {queue_name} 失效(连接版本不匹配/通道关闭),触发重连恢复")
|
|
508
|
-
|
|
509
|
-
# 移除旧消费者记录(加锁保证原子性)
|
|
510
|
-
async with self._lock:
|
|
511
|
-
if self._consumer_channels.get(queue_name) == (channel, conn, callback, auto_ack, kwargs):
|
|
512
|
-
del self._consumer_channels[queue_name]
|
|
513
|
-
|
|
514
|
-
# 释放旧通道(加锁保证原子性)
|
|
515
|
-
await self.release_channel(channel, conn)
|
|
516
|
-
|
|
517
|
-
# 重新创建消费者(新通道带自动恢复)
|
|
518
|
-
asyncio.create_task(self.consume_queue(
|
|
519
|
-
queue_name, callback, auto_ack, **kwargs))
|
|
520
|
-
|
|
521
|
-
# Nack消息(避免丢失)
|
|
326
|
+
conn_valid = self._connection and not self._connection.is_closed
|
|
327
|
+
conn_matched = conn == self._connection
|
|
328
|
+
|
|
329
|
+
if not channel_valid or not conn_valid or not conn_matched:
|
|
330
|
+
logger.warning(f"消费者通道 {queue_name} 失效(等待原生自动恢复)")
|
|
522
331
|
if not auto_ack:
|
|
523
332
|
await message.nack(requeue=True)
|
|
524
333
|
return
|
|
@@ -531,32 +340,20 @@ class RabbitMQConnectionPool:
|
|
|
531
340
|
logger.error(f"消费者通道 {queue_name} 关闭: {str(e)}", exc_info=True)
|
|
532
341
|
if not auto_ack:
|
|
533
342
|
await message.nack(requeue=True)
|
|
534
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
535
343
|
except aiormq.exceptions.ChannelInvalidStateError as e:
|
|
536
344
|
logger.error(
|
|
537
345
|
f"消费者通道 {queue_name} 状态异常: {str(e)}", exc_info=True)
|
|
538
346
|
if not auto_ack:
|
|
539
347
|
await message.nack(requeue=True)
|
|
540
|
-
asyncio.create_task(self.consume_queue(
|
|
541
|
-
queue_name, callback, auto_ack, **kwargs))
|
|
542
348
|
except Exception as e:
|
|
543
349
|
logger.error(
|
|
544
350
|
f"消费消息失败(队列: {queue_name}): {str(e)}", exc_info=True)
|
|
545
351
|
if not auto_ack:
|
|
546
352
|
await message.nack(requeue=True)
|
|
547
353
|
|
|
548
|
-
|
|
549
|
-
async with self._lock:
|
|
550
|
-
current_host = self._current_host
|
|
551
|
-
current_version = self._connection_version
|
|
552
|
-
|
|
553
|
-
logger.info(
|
|
554
|
-
f"开始消费队列: {queue_name} - 连接: {current_host}:{self.port}(版本: {current_version}), "
|
|
555
|
-
f"通道带自动恢复"
|
|
556
|
-
)
|
|
354
|
+
logger.info(f"开始消费队列: {queue_name}(通道带原生自动恢复)")
|
|
557
355
|
|
|
558
356
|
try:
|
|
559
|
-
# 启动消费(使用带自动恢复的通道)
|
|
560
357
|
await channel.basic_consume(
|
|
561
358
|
queue_name,
|
|
562
359
|
consumer_callback=consume_callback_wrapper,
|
|
@@ -565,22 +362,56 @@ class RabbitMQConnectionPool:
|
|
|
565
362
|
)
|
|
566
363
|
except Exception as e:
|
|
567
364
|
logger.error(f"启动消费失败(队列: {queue_name}): {str(e)}", exc_info=True)
|
|
568
|
-
#
|
|
365
|
+
# 清理异常资源
|
|
569
366
|
await self.release_channel(channel, conn)
|
|
570
367
|
async with self._lock:
|
|
571
|
-
if self._consumer_channels.get(queue_name) == (channel,
|
|
368
|
+
if self._consumer_channels.get(queue_name) == (channel, callback, auto_ack, kwargs):
|
|
572
369
|
del self._consumer_channels[queue_name]
|
|
573
370
|
raise
|
|
574
371
|
|
|
575
372
|
async def close(self):
|
|
576
|
-
"""
|
|
373
|
+
"""关闭通道池(仅主动关闭时释放所有资源)"""
|
|
577
374
|
async with self._lock:
|
|
578
375
|
if self._is_shutdown:
|
|
579
376
|
logger.warning("通道池已关闭,无需重复操作")
|
|
580
377
|
return
|
|
581
378
|
self._is_shutdown = True
|
|
379
|
+
self._initialized = False
|
|
380
|
+
|
|
381
|
+
logger.info("开始关闭RabbitMQ通道池(释放所有资源)...")
|
|
382
|
+
|
|
383
|
+
# 1. 关闭所有消费者通道
|
|
384
|
+
async with self._lock:
|
|
385
|
+
consumer_channels = list(self._consumer_channels.values())
|
|
386
|
+
self._consumer_channels.clear()
|
|
387
|
+
for channel, _, _, _ in consumer_channels:
|
|
388
|
+
try:
|
|
389
|
+
if not channel.is_closed:
|
|
390
|
+
await channel.close()
|
|
391
|
+
except Exception as e:
|
|
392
|
+
logger.warning(f"关闭消费者通道失败: {str(e)}")
|
|
393
|
+
|
|
394
|
+
# 2. 关闭所有普通通道
|
|
395
|
+
async with self._lock:
|
|
396
|
+
all_channels = self._free_channels + list(self._used_channels)
|
|
397
|
+
self._free_channels.clear()
|
|
398
|
+
self._used_channels.clear()
|
|
399
|
+
for channel in all_channels:
|
|
400
|
+
try:
|
|
401
|
+
if not channel.is_closed:
|
|
402
|
+
await channel.close()
|
|
403
|
+
except Exception as e:
|
|
404
|
+
logger.warning(f"关闭通道失败: {str(e)}")
|
|
405
|
+
|
|
406
|
+
# 3. 关闭连接(终止原生自动重连)
|
|
407
|
+
if self._connection:
|
|
408
|
+
try:
|
|
409
|
+
if not self._connection.is_closed:
|
|
410
|
+
await self._connection.close()
|
|
411
|
+
logger.info(
|
|
412
|
+
f"已关闭连接: {self._current_host}:{self.port}(终止原生自动重连)")
|
|
413
|
+
except Exception as e:
|
|
414
|
+
logger.warning(f"关闭连接失败: {str(e)}")
|
|
415
|
+
self._connection = None
|
|
582
416
|
|
|
583
|
-
logger.info("
|
|
584
|
-
# 强制释放所有资源(包括自动恢复的通道)
|
|
585
|
-
await self._safe_close_old_resources()
|
|
586
|
-
logger.info("RabbitMQ单连接通道池已完全关闭(所有自动恢复逻辑已终止)")
|
|
417
|
+
logger.info("RabbitMQ通道池已完全关闭")
|
|
@@ -36,7 +36,7 @@ sycommon/models/mqmsg_model.py,sha256=cxn0M5b0utQK6crMYmL-1waeGYHvK3AlGaRy23clqT
|
|
|
36
36
|
sycommon/models/mqsend_config.py,sha256=NQX9dc8PpuquMG36GCVhJe8omAW1KVXXqr6lSRU6D7I,268
|
|
37
37
|
sycommon/models/sso_user.py,sha256=i1WAN6k5sPcPApQEdtjpWDy7VrzWLpOrOQewGLGoGIw,2702
|
|
38
38
|
sycommon/rabbitmq/rabbitmq_client.py,sha256=dKpiptV892VbW3fErAoRZSE9UVVBZvbVPM3qPu8rQIk,21573
|
|
39
|
-
sycommon/rabbitmq/rabbitmq_pool.py,sha256=
|
|
39
|
+
sycommon/rabbitmq/rabbitmq_pool.py,sha256=q5w1xrBJLh7Z0b_SPDQJsBESjxb06oJj9SfLZglJ6MQ,18225
|
|
40
40
|
sycommon/rabbitmq/rabbitmq_service.py,sha256=UUvGkmDBQ7xTdCLYqkDSYZw5-FUNU7vnmgmANh4Z_bI,39960
|
|
41
41
|
sycommon/sse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
42
|
sycommon/sse/event.py,sha256=k_rBJy23R7crtzQeetT0Q73D8o5-5p-eESGSs_BPOj0,2797
|
|
@@ -52,8 +52,8 @@ sycommon/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
52
52
|
sycommon/tools/docs.py,sha256=OPj2ETheuWjXLyaXtaZPbwmJKfJaYXV5s4XMVAUNrms,1607
|
|
53
53
|
sycommon/tools/snowflake.py,sha256=DdEj3T5r5OEvikp3puxqmmmz6BrggxomoSlnsRFb5dM,1174
|
|
54
54
|
sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
|
|
55
|
-
sycommon_python_lib-0.1.
|
|
56
|
-
sycommon_python_lib-0.1.
|
|
57
|
-
sycommon_python_lib-0.1.
|
|
58
|
-
sycommon_python_lib-0.1.
|
|
59
|
-
sycommon_python_lib-0.1.
|
|
55
|
+
sycommon_python_lib-0.1.49.dist-info/METADATA,sha256=63TO0OnSgCxSzGby7K-c_UQlVXAMf3f8TOeYH63cXDE,7037
|
|
56
|
+
sycommon_python_lib-0.1.49.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
57
|
+
sycommon_python_lib-0.1.49.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
|
|
58
|
+
sycommon_python_lib-0.1.49.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
|
|
59
|
+
sycommon_python_lib-0.1.49.dist-info/RECORD,,
|
|
File without changes
|
{sycommon_python_lib-0.1.48.dist-info → sycommon_python_lib-0.1.49.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|