sycommon-python-lib 0.1.34__py3-none-any.whl → 0.1.35__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sycommon-python-lib might be problematic. Click here for more details.
- sycommon/rabbitmq/rabbitmq_client.py +530 -271
- sycommon/rabbitmq/rabbitmq_service.py +213 -115
- sycommon/services.py +58 -26
- sycommon/synacos/feign.py +22 -9
- {sycommon_python_lib-0.1.34.dist-info → sycommon_python_lib-0.1.35.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.1.34.dist-info → sycommon_python_lib-0.1.35.dist-info}/RECORD +9 -9
- {sycommon_python_lib-0.1.34.dist-info → sycommon_python_lib-0.1.35.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.34.dist-info → sycommon_python_lib-0.1.35.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.34.dist-info → sycommon_python_lib-0.1.35.dist-info}/top_level.txt +0 -0
|
@@ -25,6 +25,7 @@ class RabbitMQClient:
|
|
|
25
25
|
"""
|
|
26
26
|
RabbitMQ客户端(基于连接池),支持集群多节点配置
|
|
27
27
|
提供自动故障转移、连接恢复和消息可靠性保障
|
|
28
|
+
采用细粒度锁设计,彻底避免死锁隐患
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
31
|
def __init__(
|
|
@@ -70,18 +71,19 @@ class RabbitMQClient:
|
|
|
70
71
|
# 消息处理参数
|
|
71
72
|
self.consumption_stall_threshold = consumption_stall_threshold
|
|
72
73
|
|
|
73
|
-
#
|
|
74
|
+
# 通道和资源对象(由 _connection_lock 保护)
|
|
74
75
|
self.channel: Optional[AbstractChannel] = None
|
|
75
76
|
self.exchange: Optional[AbstractExchange] = None
|
|
76
77
|
self.queue: Optional[AbstractQueue] = None
|
|
77
78
|
|
|
78
|
-
#
|
|
79
|
+
# 状态跟踪(按类型拆分锁保护)
|
|
79
80
|
self.actual_queue_name: Optional[str] = None
|
|
80
|
-
self._exchange_exists = False
|
|
81
|
-
self._queue_exists = False
|
|
82
|
-
self._queue_bound = False
|
|
83
|
-
self._is_consuming = False
|
|
84
|
-
self._closed = False
|
|
81
|
+
self._exchange_exists = False # 由 _connection_lock 保护
|
|
82
|
+
self._queue_exists = False # 由 _connection_lock 保护
|
|
83
|
+
self._queue_bound = False # 由 _connection_lock 保护
|
|
84
|
+
self._is_consuming = False # 由 _consume_state_lock 保护
|
|
85
|
+
self._closed = False # 由 _connection_lock 保护
|
|
86
|
+
# 由 _consume_state_lock 保护
|
|
85
87
|
self._consumer_tag: Optional[ConsumerTag] = None
|
|
86
88
|
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
87
89
|
self._last_message_processed = asyncio.get_event_loop().time()
|
|
@@ -90,42 +92,156 @@ class RabbitMQClient:
|
|
|
90
92
|
self.message_handler: Optional[Callable[
|
|
91
93
|
[Union[Dict[str, Any], str], AbstractIncomingMessage],
|
|
92
94
|
Coroutine[Any, Any, None]
|
|
93
|
-
]] = None
|
|
95
|
+
]] = None # 由 _consume_state_lock 保护
|
|
94
96
|
self._consuming_task: Optional[asyncio.Task] = None
|
|
95
97
|
self._reconnect_task: Optional[asyncio.Task] = None
|
|
96
98
|
self._keepalive_task: Optional[asyncio.Task] = None
|
|
97
99
|
self._monitor_task: Optional[asyncio.Task] = None
|
|
98
100
|
|
|
99
|
-
#
|
|
101
|
+
# 消息处理跟踪(由 _tracking_lock 保护)
|
|
100
102
|
self._tracking_messages: Dict[str, Dict[str, Any]] = {}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
+
|
|
104
|
+
# 细粒度锁(核心设计:按资源类型拆分,避免嵌套)
|
|
105
|
+
# 保护消费状态(_is_consuming、message_handler、_consumer_tag)
|
|
106
|
+
self._consume_state_lock = asyncio.Lock()
|
|
107
|
+
self._tracking_lock = asyncio.Lock() # 保护消息跟踪记录(_tracking_messages)
|
|
108
|
+
# 保护连接/资源状态(channel、exchange、queue、_closed等)
|
|
109
|
+
self._connection_lock = asyncio.Lock()
|
|
103
110
|
|
|
104
111
|
@property
|
|
105
|
-
def is_connected(self) -> bool:
|
|
106
|
-
"""
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
112
|
+
async def is_connected(self) -> bool:
|
|
113
|
+
"""异步属性:检查当前通道是否有效(线程安全)"""
|
|
114
|
+
async with self._connection_lock:
|
|
115
|
+
return (not self._closed and
|
|
116
|
+
self.channel is not None and
|
|
117
|
+
not self.channel.is_closed and
|
|
118
|
+
self.exchange is not None)
|
|
111
119
|
|
|
112
120
|
def _update_activity_timestamp(self) -> None:
|
|
113
|
-
"""
|
|
121
|
+
"""更新最后活动时间戳(非共享状态,无需锁)"""
|
|
114
122
|
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
115
123
|
|
|
116
124
|
def _update_message_processed_timestamp(self) -> None:
|
|
117
|
-
"""
|
|
125
|
+
"""更新最后消息处理时间戳(非共享状态,无需锁)"""
|
|
118
126
|
self._last_message_processed = asyncio.get_event_loop().time()
|
|
119
127
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
128
|
+
# ------------------------------
|
|
129
|
+
# 消费状态操作(_consume_state_lock 专属保护)
|
|
130
|
+
# ------------------------------
|
|
131
|
+
async def _get_consume_state(self) -> tuple[bool, Optional[Callable], Optional[ConsumerTag]]:
|
|
132
|
+
"""安全获取消费相关状态(一次性获取,避免多次加锁)"""
|
|
133
|
+
async with self._consume_state_lock:
|
|
134
|
+
return self._is_consuming, self.message_handler, self._consumer_tag
|
|
135
|
+
|
|
136
|
+
async def _set_consume_state(self, is_consuming: bool, consumer_tag: Optional[ConsumerTag] = None):
|
|
137
|
+
"""安全更新消费状态(原子操作)"""
|
|
138
|
+
async with self._consume_state_lock:
|
|
139
|
+
old_is_consuming = self._is_consuming
|
|
140
|
+
self._is_consuming = is_consuming
|
|
141
|
+
if consumer_tag is not None:
|
|
142
|
+
self._consumer_tag = consumer_tag
|
|
143
|
+
if old_is_consuming != is_consuming:
|
|
144
|
+
logger.info(f"消费状态变更: {old_is_consuming} → {is_consuming}")
|
|
126
145
|
|
|
146
|
+
async def set_message_handler(self, handler):
|
|
147
|
+
"""设置消息处理器(加锁保护,避免并发修改)"""
|
|
148
|
+
async with self._consume_state_lock:
|
|
149
|
+
self.message_handler = handler
|
|
150
|
+
logger.info("消息处理器已设置")
|
|
151
|
+
|
|
152
|
+
# ------------------------------
|
|
153
|
+
# 连接状态操作(_connection_lock 专属保护)
|
|
154
|
+
# ------------------------------
|
|
155
|
+
async def _is_closed(self) -> bool:
|
|
156
|
+
"""检查客户端是否已关闭(线程安全)"""
|
|
157
|
+
async with self._connection_lock:
|
|
158
|
+
return self._closed
|
|
159
|
+
|
|
160
|
+
async def _mark_closed(self):
|
|
161
|
+
"""标记客户端已关闭(原子操作)"""
|
|
162
|
+
async with self._connection_lock:
|
|
163
|
+
self._closed = True
|
|
164
|
+
|
|
165
|
+
async def _get_connection_resources(self) -> tuple[Optional[AbstractChannel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
166
|
+
"""安全获取连接资源(channel/exchange/queue)"""
|
|
167
|
+
async with self._connection_lock:
|
|
168
|
+
return self.channel, self.exchange, self.queue
|
|
169
|
+
|
|
170
|
+
async def _reset_connection_state(self):
|
|
171
|
+
"""重置连接状态(用于重连时,原子操作)"""
|
|
172
|
+
async with self._connection_lock:
|
|
173
|
+
self._exchange_exists = False
|
|
174
|
+
self._queue_exists = False
|
|
175
|
+
self._queue_bound = False
|
|
176
|
+
self.channel = None
|
|
177
|
+
self.exchange = None
|
|
178
|
+
self.queue = None
|
|
179
|
+
self.actual_queue_name = None
|
|
180
|
+
|
|
181
|
+
async def _update_connection_resources(self, channel: AbstractChannel, exchange: AbstractExchange, queue: Optional[AbstractQueue] = None):
|
|
182
|
+
"""更新连接资源(原子操作)"""
|
|
183
|
+
async with self._connection_lock:
|
|
184
|
+
self.channel = channel
|
|
185
|
+
self.exchange = exchange
|
|
186
|
+
self.queue = queue
|
|
187
|
+
if queue:
|
|
188
|
+
self.actual_queue_name = queue.name
|
|
189
|
+
|
|
190
|
+
# ------------------------------
|
|
191
|
+
# 消息跟踪操作(_tracking_lock 专属保护)
|
|
192
|
+
# ------------------------------
|
|
193
|
+
async def _add_tracking_message(self, msg_id: str, delivery_tag: int, channel_number: Optional[int]):
|
|
194
|
+
"""添加消息跟踪记录(原子操作)"""
|
|
195
|
+
async with self._tracking_lock:
|
|
196
|
+
self._tracking_messages[msg_id] = {
|
|
197
|
+
'delivery_tag': delivery_tag,
|
|
198
|
+
'acked': False,
|
|
199
|
+
'channel_number': channel_number,
|
|
200
|
+
'start_time': asyncio.get_event_loop().time()
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
async def _mark_tracking_acked(self, msg_id: str):
|
|
204
|
+
"""标记消息已确认(原子操作)"""
|
|
205
|
+
async with self._tracking_lock:
|
|
206
|
+
if msg_id in self._tracking_messages:
|
|
207
|
+
self._tracking_messages[msg_id]['acked'] = True
|
|
208
|
+
|
|
209
|
+
async def _remove_tracking_message(self, msg_id: str):
|
|
210
|
+
"""删除消息跟踪记录(原子操作,避免KeyError)"""
|
|
211
|
+
async with self._tracking_lock:
|
|
212
|
+
if msg_id in self._tracking_messages:
|
|
213
|
+
del self._tracking_messages[msg_id]
|
|
214
|
+
logger.info(f"已删除消息跟踪信息: {msg_id}")
|
|
215
|
+
|
|
216
|
+
async def _check_duplicate_message(self, msg_id: str) -> bool:
|
|
217
|
+
"""检查消息是否重复处理(原子操作)"""
|
|
218
|
+
async with self._tracking_lock:
|
|
219
|
+
return msg_id in self._tracking_messages
|
|
220
|
+
|
|
221
|
+
async def _get_tracking_count(self) -> int:
|
|
222
|
+
"""获取当前跟踪的消息数(原子操作)"""
|
|
223
|
+
async with self._tracking_lock:
|
|
224
|
+
return len(self._tracking_messages)
|
|
225
|
+
|
|
226
|
+
async def _cleanup_acked_tracking_messages(self) -> int:
|
|
227
|
+
"""清理已确认的跟踪记录(原子操作,返回清理数量)"""
|
|
228
|
+
async with self._tracking_lock:
|
|
229
|
+
acked_ids = [
|
|
230
|
+
msg_id for msg_id, info in self._tracking_messages.items() if info.get('acked')]
|
|
231
|
+
for msg_id in acked_ids:
|
|
232
|
+
del self._tracking_messages[msg_id]
|
|
233
|
+
return len(acked_ids)
|
|
234
|
+
|
|
235
|
+
async def _clear_tracking_messages(self):
|
|
236
|
+
"""清空所有跟踪记录(原子操作)"""
|
|
237
|
+
async with self._tracking_lock:
|
|
238
|
+
self._tracking_messages.clear()
|
|
239
|
+
|
|
240
|
+
# ------------------------------
|
|
241
|
+
# 基础工具方法
|
|
242
|
+
# ------------------------------
|
|
127
243
|
async def _get_channel(self) -> AbstractChannel:
|
|
128
|
-
"""
|
|
244
|
+
"""从通道池获取通道(使用上下文管理器,自动归还)"""
|
|
129
245
|
if not self.connection_pool.channel_pool:
|
|
130
246
|
raise Exception("连接池未初始化,请先调用init_pools")
|
|
131
247
|
|
|
@@ -164,7 +280,7 @@ class RabbitMQClient:
|
|
|
164
280
|
return False
|
|
165
281
|
|
|
166
282
|
async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
|
|
167
|
-
"""
|
|
283
|
+
"""将队列绑定到交换机(带重试)"""
|
|
168
284
|
bind_routing_key = self.routing_key if self.routing_key else '#'
|
|
169
285
|
|
|
170
286
|
for attempt in range(MAX_RETRY_COUNT + 1):
|
|
@@ -177,7 +293,7 @@ class RabbitMQClient:
|
|
|
177
293
|
timeout=self.rpc_timeout
|
|
178
294
|
)
|
|
179
295
|
logger.info(
|
|
180
|
-
f"队列 '{
|
|
296
|
+
f"队列 '{queue.name}' 已绑定到交换机 '{exchange.name}',路由键: {bind_routing_key}")
|
|
181
297
|
return True
|
|
182
298
|
except Exception as e:
|
|
183
299
|
logger.warning(
|
|
@@ -186,6 +302,9 @@ class RabbitMQClient:
|
|
|
186
302
|
await asyncio.sleep(1)
|
|
187
303
|
return False
|
|
188
304
|
|
|
305
|
+
# ------------------------------
|
|
306
|
+
# 核心业务方法
|
|
307
|
+
# ------------------------------
|
|
189
308
|
async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
|
|
190
309
|
"""从连接池获取资源并初始化(交换机、队列)"""
|
|
191
310
|
logger.info(
|
|
@@ -193,32 +312,43 @@ class RabbitMQClient:
|
|
|
193
312
|
f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
|
|
194
313
|
)
|
|
195
314
|
|
|
196
|
-
|
|
315
|
+
# 检查是否已关闭
|
|
316
|
+
if await self._is_closed():
|
|
317
|
+
raise Exception("客户端已关闭,无法连接")
|
|
318
|
+
|
|
319
|
+
# 检查是否已连接(非强制重连则直接返回)
|
|
320
|
+
if await self.is_connected and not force_reconnect:
|
|
321
|
+
logger.info("已处于连接状态,无需重复连接")
|
|
197
322
|
return
|
|
198
323
|
|
|
324
|
+
# 取消现有重连任务
|
|
199
325
|
if self._reconnect_task and not self._reconnect_task.done():
|
|
200
326
|
self._reconnect_task.cancel()
|
|
327
|
+
try:
|
|
328
|
+
await self._reconnect_task
|
|
329
|
+
except asyncio.CancelledError:
|
|
330
|
+
logger.info("旧重连任务已取消")
|
|
201
331
|
|
|
202
|
-
#
|
|
203
|
-
self.
|
|
204
|
-
self.
|
|
205
|
-
self.
|
|
206
|
-
await self._set_is_consuming(False)
|
|
332
|
+
# 重置连接状态和跟踪记录
|
|
333
|
+
await self._reset_connection_state()
|
|
334
|
+
await self._clear_tracking_messages()
|
|
335
|
+
await self._set_consume_state(is_consuming=False)
|
|
207
336
|
|
|
208
337
|
retries = 0
|
|
209
338
|
last_exception = None
|
|
210
339
|
|
|
211
340
|
while retries < self.max_reconnection_attempts:
|
|
212
341
|
try:
|
|
213
|
-
|
|
214
|
-
await self.
|
|
342
|
+
# 获取新通道
|
|
343
|
+
channel = await self._get_channel()
|
|
344
|
+
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
215
345
|
|
|
216
346
|
# 处理交换机
|
|
217
|
-
exchange_exists = await self._check_exchange_exists(
|
|
347
|
+
exchange_exists = await self._check_exchange_exists(channel)
|
|
218
348
|
if not exchange_exists:
|
|
219
349
|
if self.create_if_not_exists:
|
|
220
|
-
|
|
221
|
-
|
|
350
|
+
exchange = await asyncio.wait_for(
|
|
351
|
+
channel.declare_exchange(
|
|
222
352
|
name=self.exchange_name,
|
|
223
353
|
type=self.exchange_type,
|
|
224
354
|
durable=self.durable,
|
|
@@ -231,20 +361,21 @@ class RabbitMQClient:
|
|
|
231
361
|
raise Exception(
|
|
232
362
|
f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
|
|
233
363
|
else:
|
|
234
|
-
|
|
364
|
+
exchange = await channel.get_exchange(self.exchange_name)
|
|
235
365
|
logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
|
|
236
366
|
|
|
237
367
|
# 处理队列
|
|
368
|
+
queue = None
|
|
238
369
|
if declare_queue and self.queue_name:
|
|
239
|
-
queue_exists = await self._check_queue_exists(
|
|
370
|
+
queue_exists = await self._check_queue_exists(channel)
|
|
240
371
|
|
|
241
372
|
if not queue_exists:
|
|
242
373
|
if not self.create_if_not_exists:
|
|
243
374
|
raise Exception(
|
|
244
375
|
f"队列 '{self.queue_name}' 不存在且不允许自动创建")
|
|
245
376
|
|
|
246
|
-
|
|
247
|
-
|
|
377
|
+
queue = await asyncio.wait_for(
|
|
378
|
+
channel.declare_queue(
|
|
248
379
|
name=self.queue_name,
|
|
249
380
|
durable=self.durable,
|
|
250
381
|
auto_delete=self.auto_delete,
|
|
@@ -252,28 +383,27 @@ class RabbitMQClient:
|
|
|
252
383
|
),
|
|
253
384
|
timeout=self.rpc_timeout
|
|
254
385
|
)
|
|
255
|
-
self.actual_queue_name = self.queue_name
|
|
256
386
|
logger.info(f"已创建队列 '{self.queue_name}'")
|
|
257
387
|
else:
|
|
258
|
-
|
|
259
|
-
self.actual_queue_name = self.queue_name
|
|
388
|
+
queue = await channel.get_queue(self.queue_name)
|
|
260
389
|
logger.info(f"使用已存在的队列 '{self.queue_name}'")
|
|
261
390
|
|
|
262
391
|
# 绑定队列到交换机
|
|
263
|
-
if
|
|
264
|
-
bound = await self._bind_queue(
|
|
392
|
+
if queue and exchange:
|
|
393
|
+
bound = await self._bind_queue(channel, queue, exchange)
|
|
265
394
|
if not bound:
|
|
266
|
-
raise Exception(f"队列 '{
|
|
267
|
-
else:
|
|
268
|
-
self.queue = None
|
|
269
|
-
self.actual_queue_name = None
|
|
270
|
-
logger.info(f"跳过队列 '{self.queue_name}' 的声明和绑定")
|
|
395
|
+
raise Exception(f"队列 '{queue.name}' 绑定到交换机失败")
|
|
271
396
|
|
|
272
|
-
|
|
397
|
+
# 更新连接资源
|
|
398
|
+
await self._update_connection_resources(channel, exchange, queue)
|
|
399
|
+
|
|
400
|
+
# 验证连接状态
|
|
401
|
+
if not await self.is_connected:
|
|
273
402
|
raise Exception("连接验证失败,状态异常")
|
|
274
403
|
|
|
275
|
-
#
|
|
276
|
-
|
|
404
|
+
# 重新开始消费(如果已设置处理器)
|
|
405
|
+
is_consuming, handler, _ = await self._get_consume_state()
|
|
406
|
+
if handler:
|
|
277
407
|
await self.start_consuming()
|
|
278
408
|
|
|
279
409
|
# 启动监控和保活任务
|
|
@@ -281,15 +411,12 @@ class RabbitMQClient:
|
|
|
281
411
|
self._start_keepalive()
|
|
282
412
|
|
|
283
413
|
self._update_activity_timestamp()
|
|
284
|
-
# 清理可能残留的跟踪记录
|
|
285
|
-
self._tracking_messages.clear()
|
|
286
414
|
logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
|
|
287
415
|
return
|
|
288
416
|
|
|
289
417
|
except Exception as e:
|
|
290
418
|
last_exception = e
|
|
291
419
|
logger.warning(f"资源初始化失败: {str(e)},重试中...")
|
|
292
|
-
self.channel = None
|
|
293
420
|
retries += 1
|
|
294
421
|
if retries < self.max_reconnection_attempts:
|
|
295
422
|
await asyncio.sleep(self.reconnection_delay)
|
|
@@ -299,127 +426,146 @@ class RabbitMQClient:
|
|
|
299
426
|
f"经过{self.max_reconnection_attempts}次重试后仍无法初始化客户端。最后错误: {str(last_exception)}")
|
|
300
427
|
|
|
301
428
|
def _start_monitoring(self) -> None:
|
|
302
|
-
"""
|
|
303
|
-
if self.
|
|
429
|
+
"""启动连接和消费监控任务(无锁,仅通过原子方法访问状态)"""
|
|
430
|
+
if self._monitor_task and not self._monitor_task.done():
|
|
304
431
|
return
|
|
305
432
|
|
|
306
433
|
async def monitor():
|
|
307
|
-
while not
|
|
434
|
+
while not await self._is_closed():
|
|
308
435
|
try:
|
|
309
436
|
# 检查通道状态
|
|
310
|
-
|
|
437
|
+
channel, _, _ = await self._get_connection_resources()
|
|
438
|
+
if channel and channel.is_closed:
|
|
311
439
|
logger.warning("检测到通道已关闭,尝试重建")
|
|
312
440
|
await self._recreate_channel()
|
|
313
441
|
continue
|
|
314
442
|
|
|
315
443
|
current_time = asyncio.get_event_loop().time()
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
# 检查消费停滞(仅当消费状态为 True 时才处理)
|
|
328
|
-
if self._is_consuming:
|
|
444
|
+
|
|
445
|
+
# 清理已确认的跟踪记录
|
|
446
|
+
cleaned_count = await self._cleanup_acked_tracking_messages()
|
|
447
|
+
if cleaned_count > 0:
|
|
448
|
+
logger.info(f"清理了 {cleaned_count} 条已确认消息记录")
|
|
449
|
+
|
|
450
|
+
# 检查消费停滞(仅当消费状态为True时)
|
|
451
|
+
is_consuming, _, _ = await self._get_consume_state()
|
|
452
|
+
if is_consuming:
|
|
453
|
+
tracking_count = await self._get_tracking_count()
|
|
329
454
|
if current_time - self._last_message_processed > self.consumption_stall_threshold:
|
|
330
|
-
if
|
|
455
|
+
if tracking_count > 0:
|
|
331
456
|
logger.warning(
|
|
332
|
-
f"消费停滞,但有 {
|
|
457
|
+
f"消费停滞,但有 {tracking_count} 个消息正在处理,暂不重启")
|
|
333
458
|
else:
|
|
334
459
|
logger.info("消费停滞且无消息处理,重启消费")
|
|
335
|
-
# 重启消费时增加异常捕获,确保状态回滚
|
|
336
460
|
try:
|
|
337
461
|
await self.stop_consuming()
|
|
338
462
|
await asyncio.sleep(1)
|
|
339
|
-
#
|
|
340
|
-
|
|
463
|
+
# 检查处理器是否存在
|
|
464
|
+
_, handler, _ = await self._get_consume_state()
|
|
465
|
+
if handler:
|
|
341
466
|
await self.start_consuming()
|
|
342
467
|
else:
|
|
343
468
|
logger.error("消费处理器已丢失,无法重启消费")
|
|
344
469
|
except Exception as e:
|
|
345
470
|
logger.error(
|
|
346
471
|
f"重启消费失败: {str(e)}", exc_info=True)
|
|
347
|
-
|
|
348
|
-
|
|
472
|
+
await self._set_consume_state(is_consuming=False)
|
|
473
|
+
|
|
349
474
|
except Exception as e:
|
|
350
475
|
logger.error(f"监控任务出错: {str(e)}", exc_info=True)
|
|
351
476
|
|
|
352
|
-
await asyncio.sleep(60)
|
|
477
|
+
await asyncio.sleep(60) # 监控间隔60秒
|
|
353
478
|
|
|
354
479
|
self._monitor_task = asyncio.create_task(monitor())
|
|
480
|
+
logger.info("监控任务已启动")
|
|
355
481
|
|
|
356
482
|
async def _recreate_channel(self) -> None:
|
|
357
|
-
|
|
358
|
-
|
|
483
|
+
"""重建通道并恢复资源(无锁嵌套)"""
|
|
484
|
+
# 先停止消费
|
|
485
|
+
await self._set_consume_state(is_consuming=False)
|
|
486
|
+
logger.info("开始重建通道...")
|
|
487
|
+
|
|
359
488
|
try:
|
|
360
|
-
|
|
361
|
-
await self.
|
|
489
|
+
# 获取新通道
|
|
490
|
+
channel = await self._get_channel()
|
|
491
|
+
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
492
|
+
|
|
493
|
+
# 重新获取交换机
|
|
494
|
+
exchange = await channel.get_exchange(self.exchange_name)
|
|
362
495
|
|
|
363
|
-
#
|
|
364
|
-
|
|
496
|
+
# 重新获取队列并绑定
|
|
497
|
+
queue = None
|
|
365
498
|
if self.queue_name:
|
|
366
|
-
|
|
367
|
-
if
|
|
368
|
-
bound = await self._bind_queue(
|
|
499
|
+
queue = await channel.get_queue(self.queue_name)
|
|
500
|
+
if queue and exchange:
|
|
501
|
+
bound = await self._bind_queue(channel, queue, exchange)
|
|
369
502
|
if not bound:
|
|
370
503
|
raise Exception("队列绑定失败,通道重建不完整")
|
|
371
504
|
|
|
372
|
-
#
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
505
|
+
# 更新连接资源
|
|
506
|
+
await self._update_connection_resources(channel, exchange, queue)
|
|
507
|
+
|
|
508
|
+
# 重新开始消费
|
|
509
|
+
_, handler, _ = await self._get_consume_state()
|
|
510
|
+
if handler:
|
|
376
511
|
await self.start_consuming()
|
|
377
512
|
|
|
513
|
+
# 清空跟踪记录
|
|
514
|
+
await self._clear_tracking_messages()
|
|
378
515
|
logger.info("通道已重建并恢复服务")
|
|
379
516
|
self._update_activity_timestamp()
|
|
380
517
|
except Exception as e:
|
|
381
518
|
logger.error(f"通道重建失败: {str(e)},触发重连", exc_info=True)
|
|
382
|
-
|
|
383
|
-
await self._set_is_consuming(False)
|
|
519
|
+
await self._set_consume_state(is_consuming=False)
|
|
384
520
|
await self.connect(force_reconnect=True)
|
|
385
521
|
|
|
386
522
|
def _start_keepalive(self) -> None:
|
|
387
|
-
"""
|
|
388
|
-
if self.
|
|
523
|
+
"""启动连接保活任务(无锁,仅通过原子方法访问状态)"""
|
|
524
|
+
if self._keepalive_task and not self._keepalive_task.done():
|
|
389
525
|
return
|
|
390
526
|
|
|
391
527
|
async def keepalive():
|
|
392
|
-
while not
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
try:
|
|
398
|
-
if self.channel.is_closed:
|
|
399
|
-
logger.warning("连接已关闭,触发重连")
|
|
400
|
-
await self.connect(force_reconnect=True)
|
|
401
|
-
return
|
|
402
|
-
|
|
403
|
-
# 轻量级操作保持连接活跃
|
|
404
|
-
await asyncio.wait_for(
|
|
405
|
-
self.channel.declare_exchange(
|
|
406
|
-
name=self.exchange_name,
|
|
407
|
-
type=self.exchange_type,
|
|
408
|
-
passive=True
|
|
409
|
-
),
|
|
410
|
-
timeout=5
|
|
411
|
-
)
|
|
412
|
-
self._update_activity_timestamp()
|
|
413
|
-
except Exception as e:
|
|
414
|
-
logger.warning(f"保活检查失败: {str(e)},触发重连")
|
|
528
|
+
while not await self._is_closed():
|
|
529
|
+
try:
|
|
530
|
+
# 检查连接状态
|
|
531
|
+
if not await self.is_connected:
|
|
532
|
+
logger.warning("保活任务检测到连接断开,触发重连")
|
|
415
533
|
await self.connect(force_reconnect=True)
|
|
534
|
+
await asyncio.sleep(5)
|
|
535
|
+
continue
|
|
536
|
+
|
|
537
|
+
current_time = asyncio.get_event_loop().time()
|
|
538
|
+
# 检查活动时间
|
|
539
|
+
if current_time - self._last_activity_timestamp > self.connection_pool.heartbeat * 2:
|
|
540
|
+
logger.info(
|
|
541
|
+
f"连接 {self.connection_pool.heartbeat*2}s 无活动,执行保活检查")
|
|
542
|
+
channel, exchange, _ = await self._get_connection_resources()
|
|
543
|
+
if channel and not channel.is_closed and exchange:
|
|
544
|
+
# 轻量级操作:检查交换机是否存在
|
|
545
|
+
await asyncio.wait_for(
|
|
546
|
+
channel.declare_exchange(
|
|
547
|
+
name=self.exchange_name,
|
|
548
|
+
type=self.exchange_type,
|
|
549
|
+
passive=True
|
|
550
|
+
),
|
|
551
|
+
timeout=5
|
|
552
|
+
)
|
|
553
|
+
self._update_activity_timestamp()
|
|
554
|
+
logger.info("保活检查成功")
|
|
555
|
+
else:
|
|
556
|
+
raise Exception("连接资源无效")
|
|
557
|
+
|
|
558
|
+
except Exception as e:
|
|
559
|
+
logger.warning(f"保活检查失败: {str(e)},触发重连")
|
|
560
|
+
await self.connect(force_reconnect=True)
|
|
416
561
|
|
|
417
562
|
await asyncio.sleep(self.connection_pool.heartbeat)
|
|
418
563
|
|
|
419
564
|
self._keepalive_task = asyncio.create_task(keepalive())
|
|
565
|
+
logger.info("保活任务已启动")
|
|
420
566
|
|
|
421
567
|
async def _schedule_reconnect(self) -> None:
|
|
422
|
-
"""
|
|
568
|
+
"""安排重新连接(无锁)"""
|
|
423
569
|
if self._reconnect_task and not self._reconnect_task.done():
|
|
424
570
|
return
|
|
425
571
|
|
|
@@ -428,38 +574,48 @@ class RabbitMQClient:
|
|
|
428
574
|
async def reconnect():
|
|
429
575
|
try:
|
|
430
576
|
await asyncio.sleep(self.reconnection_delay)
|
|
431
|
-
if not self.
|
|
577
|
+
if not await self._is_closed():
|
|
432
578
|
await self.connect(force_reconnect=True)
|
|
433
579
|
except Exception as e:
|
|
434
580
|
logger.error(f"重连任务失败: {str(e)}")
|
|
435
|
-
if not self.
|
|
581
|
+
if not await self._is_closed():
|
|
436
582
|
await self._schedule_reconnect()
|
|
437
583
|
|
|
438
584
|
self._reconnect_task = asyncio.create_task(reconnect())
|
|
439
585
|
|
|
440
586
|
async def close(self) -> None:
|
|
441
|
-
"""
|
|
442
|
-
self.
|
|
443
|
-
|
|
587
|
+
"""关闭客户端并释放资源(原子操作,无锁嵌套)"""
|
|
588
|
+
if await self._is_closed():
|
|
589
|
+
logger.info("客户端已关闭,无需重复操作")
|
|
590
|
+
return
|
|
591
|
+
|
|
592
|
+
logger.info("开始关闭RabbitMQ客户端...")
|
|
593
|
+
|
|
594
|
+
# 标记为已关闭
|
|
595
|
+
await self._mark_closed()
|
|
596
|
+
|
|
597
|
+
# 停止消费
|
|
598
|
+
await self.stop_consuming()
|
|
444
599
|
|
|
445
|
-
#
|
|
446
|
-
|
|
447
|
-
|
|
600
|
+
# 取消所有后台任务
|
|
601
|
+
tasks = [self._keepalive_task,
|
|
602
|
+
self._reconnect_task, self._monitor_task]
|
|
603
|
+
for task in tasks:
|
|
448
604
|
if task and not task.done():
|
|
449
605
|
task.cancel()
|
|
450
606
|
try:
|
|
451
607
|
await task
|
|
452
608
|
except asyncio.CancelledError:
|
|
453
|
-
|
|
609
|
+
logger.info(f"任务 {task.get_name()} 已取消")
|
|
454
610
|
|
|
455
|
-
#
|
|
456
|
-
self.
|
|
457
|
-
self.
|
|
458
|
-
self.
|
|
459
|
-
|
|
460
|
-
|
|
611
|
+
# 重置所有状态和资源
|
|
612
|
+
await self._reset_connection_state()
|
|
613
|
+
await self._clear_tracking_messages()
|
|
614
|
+
async with self._consume_state_lock:
|
|
615
|
+
self.message_handler = None
|
|
616
|
+
self._consumer_tag = None
|
|
461
617
|
|
|
462
|
-
logger.info("RabbitMQ
|
|
618
|
+
logger.info("RabbitMQ客户端已完全关闭")
|
|
463
619
|
|
|
464
620
|
async def publish(
|
|
465
621
|
self,
|
|
@@ -469,13 +625,14 @@ class RabbitMQClient:
|
|
|
469
625
|
headers: Optional[Dict[str, Any]] = None,
|
|
470
626
|
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
|
|
471
627
|
) -> None:
|
|
472
|
-
"""
|
|
473
|
-
if
|
|
474
|
-
|
|
475
|
-
await self.connect(force_reconnect=True)
|
|
628
|
+
"""发布消息(从池获取通道,自动重试,无锁冲突)"""
|
|
629
|
+
if await self._is_closed():
|
|
630
|
+
raise Exception("客户端已关闭,无法发布消息")
|
|
476
631
|
|
|
477
|
-
|
|
478
|
-
|
|
632
|
+
# 检查连接状态
|
|
633
|
+
if not await self.is_connected:
|
|
634
|
+
logger.warning("连接已断开,尝试重连后发布消息")
|
|
635
|
+
await self.connect(force_reconnect=True)
|
|
479
636
|
|
|
480
637
|
# 处理消息体
|
|
481
638
|
if isinstance(message_body, dict):
|
|
@@ -510,13 +667,12 @@ class RabbitMQClient:
|
|
|
510
667
|
raise Exception("消息未被服务器确认接收")
|
|
511
668
|
|
|
512
669
|
self._update_activity_timestamp()
|
|
513
|
-
logger.info(
|
|
670
|
+
logger.info(
|
|
671
|
+
f"消息已发布到交换机 '{self.exchange_name}'(路由键: {routing_key or self.routing_key or '#'})")
|
|
514
672
|
return
|
|
515
673
|
except (ConnectionClosed, ChannelInvalidStateError, asyncio.TimeoutError):
|
|
516
|
-
# 覆盖更多异常类型
|
|
517
674
|
retry_count += 1
|
|
518
675
|
logger.warning(f"连接异常,尝试重连后重新发布 (重试次数: {retry_count})")
|
|
519
|
-
# 主动刷新连接状态
|
|
520
676
|
await self.connect(force_reconnect=True)
|
|
521
677
|
except Exception as e:
|
|
522
678
|
retry_count += 1
|
|
@@ -526,89 +682,113 @@ class RabbitMQClient:
|
|
|
526
682
|
|
|
527
683
|
raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
|
|
528
684
|
|
|
529
|
-
async def
|
|
530
|
-
|
|
531
|
-
|
|
685
|
+
async def _safe_cancel_consumer(self, consumer_tag: ConsumerTag, queue: AbstractQueue) -> bool:
|
|
686
|
+
"""安全取消消费者(无锁,仅操作传入的局部变量)"""
|
|
687
|
+
try:
|
|
688
|
+
await asyncio.wait_for(
|
|
689
|
+
queue.cancel(consumer_tag),
|
|
690
|
+
timeout=self.rpc_timeout
|
|
691
|
+
)
|
|
692
|
+
logger.info(f"消费者 {consumer_tag} 已取消")
|
|
693
|
+
return True
|
|
694
|
+
except Exception as e:
|
|
695
|
+
logger.error(f"取消消费者 {consumer_tag} 异常: {str(e)}")
|
|
696
|
+
return False
|
|
532
697
|
|
|
533
698
|
async def start_consuming(self) -> ConsumerTag:
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
if not self.is_connected:
|
|
699
|
+
"""启动消费(无锁嵌套,通过原子方法获取/更新状态)"""
|
|
700
|
+
# 检查客户端状态
|
|
701
|
+
if await self._is_closed():
|
|
702
|
+
raise Exception("客户端已关闭,无法启动消费")
|
|
703
|
+
|
|
704
|
+
# 检查连接状态
|
|
705
|
+
if not await self.is_connected:
|
|
543
706
|
await self.connect()
|
|
544
707
|
|
|
545
|
-
|
|
546
|
-
|
|
708
|
+
# 获取消费状态和资源
|
|
709
|
+
is_consuming, handler, consumer_tag = await self._get_consume_state()
|
|
710
|
+
channel, exchange, queue = await self._get_connection_resources()
|
|
711
|
+
|
|
712
|
+
# 检查是否已在消费
|
|
713
|
+
if is_consuming and consumer_tag:
|
|
714
|
+
logger.info(f"已经在消费中,返回现有consumer_tag: {consumer_tag}")
|
|
715
|
+
return consumer_tag
|
|
547
716
|
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
717
|
+
# 检查必要条件
|
|
718
|
+
if not handler:
|
|
719
|
+
raise Exception("未设置消息处理函数,请先调用set_message_handler")
|
|
720
|
+
if not queue:
|
|
721
|
+
raise Exception("队列未初始化,无法开始消费")
|
|
722
|
+
if not channel or channel.is_closed:
|
|
723
|
+
raise Exception("通道无效,无法开始消费")
|
|
551
724
|
|
|
552
725
|
try:
|
|
553
|
-
|
|
726
|
+
# 启动消费
|
|
727
|
+
new_consumer_tag = await queue.consume(
|
|
554
728
|
self._message_wrapper,
|
|
555
729
|
no_ack=False # 手动确认消息
|
|
556
730
|
)
|
|
557
731
|
|
|
558
|
-
if not
|
|
732
|
+
if not new_consumer_tag:
|
|
559
733
|
raise Exception("未能获取到有效的consumer_tag")
|
|
560
734
|
|
|
561
|
-
|
|
735
|
+
# 更新消费状态
|
|
736
|
+
await self._set_consume_state(is_consuming=True, consumer_tag=new_consumer_tag)
|
|
562
737
|
logger.info(
|
|
563
|
-
f"消费者已启动,队列: {
|
|
564
|
-
return
|
|
738
|
+
f"消费者已启动,队列: {queue.name}, tag: {new_consumer_tag}")
|
|
739
|
+
return new_consumer_tag
|
|
565
740
|
except Exception as e:
|
|
566
|
-
#
|
|
567
|
-
await self.
|
|
741
|
+
# 异常时回滚状态
|
|
742
|
+
await self._set_consume_state(is_consuming=False)
|
|
568
743
|
logger.error(f"启动消费失败: {str(e)}", exc_info=True)
|
|
569
744
|
raise
|
|
570
745
|
|
|
571
|
-
async def _safe_cancel_consumer(self) -> bool:
|
|
572
|
-
if not self._consumer_tag or not self.queue or not self.channel:
|
|
573
|
-
return True
|
|
574
|
-
|
|
575
|
-
try:
|
|
576
|
-
await asyncio.wait_for(
|
|
577
|
-
self.queue.cancel(self._consumer_tag),
|
|
578
|
-
timeout=self.rpc_timeout
|
|
579
|
-
)
|
|
580
|
-
logger.info(f"消费者 {self._consumer_tag} 已取消")
|
|
581
|
-
return True
|
|
582
|
-
except Exception as e:
|
|
583
|
-
logger.error(f"取消消费者异常: {str(e)}")
|
|
584
|
-
return False
|
|
585
|
-
|
|
586
746
|
async def stop_consuming(self) -> None:
|
|
587
|
-
|
|
747
|
+
"""停止消费(无锁嵌套,通过原子方法获取/更新状态)"""
|
|
748
|
+
# 获取消费状态和资源
|
|
749
|
+
is_consuming, _, consumer_tag = await self._get_consume_state()
|
|
750
|
+
_, _, queue = await self._get_connection_resources()
|
|
751
|
+
|
|
752
|
+
if not is_consuming:
|
|
753
|
+
logger.info("未处于消费状态,无需停止")
|
|
588
754
|
return
|
|
589
755
|
|
|
590
|
-
|
|
756
|
+
logger.info(f"开始停止消费(consumer_tag: {consumer_tag})")
|
|
591
757
|
|
|
592
|
-
|
|
593
|
-
|
|
758
|
+
# 先更新消费状态为False
|
|
759
|
+
await self._set_consume_state(is_consuming=False)
|
|
760
|
+
|
|
761
|
+
# 取消消费者
|
|
762
|
+
if consumer_tag and queue and not await self._is_closed():
|
|
763
|
+
await self._safe_cancel_consumer(consumer_tag, queue)
|
|
594
764
|
|
|
595
765
|
# 等待所有正在处理的消息完成
|
|
596
|
-
|
|
597
|
-
|
|
766
|
+
tracking_count = await self._get_tracking_count()
|
|
767
|
+
if tracking_count > 0:
|
|
768
|
+
logger.info(f"等待 {tracking_count} 个正在处理的消息完成...")
|
|
598
769
|
wait_start = asyncio.get_event_loop().time()
|
|
599
|
-
while
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
770
|
+
while True:
|
|
771
|
+
# 检查是否超时或已关闭
|
|
772
|
+
if await self._is_closed() or asyncio.get_event_loop().time() - wait_start > 30:
|
|
773
|
+
timeout = asyncio.get_event_loop().time() - wait_start > 30
|
|
774
|
+
if timeout:
|
|
775
|
+
logger.warning("等待消息处理超时,强制清理跟踪记录")
|
|
776
|
+
await self._clear_tracking_messages()
|
|
777
|
+
break
|
|
778
|
+
# 检查跟踪记录是否为空
|
|
779
|
+
current_count = await self._get_tracking_count()
|
|
780
|
+
if current_count == 0:
|
|
603
781
|
break
|
|
604
782
|
await asyncio.sleep(1)
|
|
605
783
|
|
|
606
|
-
#
|
|
607
|
-
self.
|
|
608
|
-
|
|
609
|
-
|
|
784
|
+
# 清理消费状态
|
|
785
|
+
async with self._consume_state_lock:
|
|
786
|
+
self._consumer_tag = None
|
|
787
|
+
|
|
788
|
+
logger.info(f"已停止消费队列: {queue.name if queue else '未知'}")
|
|
610
789
|
|
|
611
790
|
async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
|
|
791
|
+
"""解析消息体(无锁,仅处理局部变量)"""
|
|
612
792
|
try:
|
|
613
793
|
body_str = message.body.decode('utf-8')
|
|
614
794
|
self._update_activity_timestamp()
|
|
@@ -617,101 +797,180 @@ class RabbitMQClient:
|
|
|
617
797
|
return json.loads(body_str)
|
|
618
798
|
return body_str
|
|
619
799
|
except json.JSONDecodeError:
|
|
620
|
-
logger.warning(
|
|
800
|
+
logger.warning(
|
|
801
|
+
f"消息 {message.message_id or id(message)} 解析JSON失败,返回原始字符串")
|
|
621
802
|
return body_str
|
|
622
803
|
except Exception as e:
|
|
623
|
-
logger.error(
|
|
804
|
+
logger.error(
|
|
805
|
+
f"消息 {message.message_id or id(message)} 解析出错: {str(e)}")
|
|
624
806
|
return message.body.decode('utf-8')
|
|
625
807
|
|
|
626
|
-
async def
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
808
|
+
async def _handle_business_retry(
|
|
809
|
+
self,
|
|
810
|
+
message: AbstractIncomingMessage,
|
|
811
|
+
error: Exception,
|
|
812
|
+
drop: bool = True
|
|
813
|
+
) -> None:
|
|
814
|
+
"""
|
|
815
|
+
封装业务失败重试逻辑:更新重试计数Header,延迟3秒重新发布
|
|
816
|
+
达到最大次数则标记失败(无锁,仅通过原子方法操作跟踪记录)
|
|
817
|
+
"""
|
|
818
|
+
# 获取当前重试次数
|
|
819
|
+
current_headers = message.headers or {}
|
|
820
|
+
retry_count = current_headers.get('x-retry-count', 0)
|
|
821
|
+
retry_count += 1
|
|
822
|
+
message_id = message.message_id or str(id(message))
|
|
631
823
|
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
824
|
+
error_msg = f"[{type(error).__name__}] {str(error)}"[:200]
|
|
825
|
+
|
|
826
|
+
# 打印错误日志
|
|
827
|
+
logger.error(
|
|
828
|
+
f"消息 {message_id} 处理出错(第{retry_count}次重试): {error_msg}",
|
|
829
|
+
exc_info=True
|
|
830
|
+
)
|
|
831
|
+
|
|
832
|
+
# 达到最大重试次数:ack标记失败
|
|
833
|
+
if drop and retry_count >= MAX_RETRY_COUNT:
|
|
834
|
+
logger.error(
|
|
835
|
+
f"消息 {message_id} 已达到最大重试次数{MAX_RETRY_COUNT},标记为失败")
|
|
836
|
+
# 标记跟踪记录为已确认
|
|
837
|
+
await self._mark_tracking_acked(message_id)
|
|
838
|
+
await message.ack()
|
|
839
|
+
self._update_activity_timestamp()
|
|
840
|
+
return
|
|
841
|
+
|
|
842
|
+
# 构造新消息Header
|
|
843
|
+
new_headers = current_headers.copy()
|
|
844
|
+
new_headers['x-retry-count'] = retry_count
|
|
845
|
+
new_headers['x-retry-error'] = error_msg
|
|
846
|
+
|
|
847
|
+
# 提交异步任务,延迟3秒后重新发布
|
|
848
|
+
asyncio.create_task(
|
|
849
|
+
self._delayed_republish(
|
|
850
|
+
message, new_headers, retry_count, message_id)
|
|
851
|
+
)
|
|
852
|
+
|
|
853
|
+
async def _delayed_republish(
|
|
854
|
+
self,
|
|
855
|
+
message: AbstractIncomingMessage,
|
|
856
|
+
new_headers: Dict[str, Any],
|
|
857
|
+
retry_count: int,
|
|
858
|
+
message_id: str
|
|
859
|
+
) -> None:
|
|
860
|
+
"""延迟发布重试消息(无锁,仅通过原子方法操作资源)"""
|
|
861
|
+
try:
|
|
862
|
+
# 延迟3秒重试
|
|
863
|
+
await asyncio.sleep(3)
|
|
864
|
+
|
|
865
|
+
# 检查客户端状态
|
|
866
|
+
if await self._is_closed():
|
|
867
|
+
logger.warning(f"客户端已关闭,放弃消息 {message_id} 的重试发布")
|
|
868
|
+
return
|
|
869
|
+
|
|
870
|
+
# 获取交换机
|
|
871
|
+
_, exchange, _ = await self._get_connection_resources()
|
|
872
|
+
if not exchange:
|
|
873
|
+
raise Exception("交换机未初始化,无法发布重试消息")
|
|
874
|
+
|
|
875
|
+
# 构造新消息
|
|
876
|
+
new_message = Message(
|
|
877
|
+
body=message.body,
|
|
878
|
+
content_type=message.content_type,
|
|
879
|
+
headers=new_headers,
|
|
880
|
+
delivery_mode=message.delivery_mode
|
|
881
|
+
)
|
|
882
|
+
|
|
883
|
+
# 重新发布消息
|
|
884
|
+
await exchange.publish(
|
|
885
|
+
new_message,
|
|
886
|
+
routing_key=self.routing_key or '#',
|
|
887
|
+
mandatory=True,
|
|
888
|
+
timeout=5.0
|
|
889
|
+
)
|
|
890
|
+
self._update_activity_timestamp()
|
|
891
|
+
logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
|
|
892
|
+
|
|
893
|
+
# 拒绝原始消息(不重新入队)
|
|
894
|
+
await message.reject(requeue=False)
|
|
895
|
+
# 标记跟踪记录为已确认
|
|
896
|
+
await self._mark_tracking_acked(message_id)
|
|
897
|
+
|
|
898
|
+
except Exception as e:
|
|
899
|
+
logger.error(
|
|
900
|
+
f"消息 {message_id} 延迟发布失败(错误:{str(e)}),触发requeue兜底",
|
|
901
|
+
exc_info=True
|
|
636
902
|
)
|
|
903
|
+
# 发布失败兜底:requeue原始消息
|
|
904
|
+
await message.reject(requeue=True)
|
|
905
|
+
|
|
906
|
+
async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
|
|
907
|
+
"""消息处理包装器(无锁嵌套,仅通过原子方法操作状态)"""
|
|
908
|
+
message_id = message.message_id or str(id(message))
|
|
909
|
+
max_check_attempts = 3
|
|
910
|
+
check_interval = 1
|
|
911
|
+
|
|
912
|
+
# 重试检查消费状态(处理极端并发场景)
|
|
913
|
+
for attempt in range(max_check_attempts):
|
|
914
|
+
is_consuming, handler, _ = await self._get_consume_state()
|
|
915
|
+
if is_consuming and handler:
|
|
916
|
+
break
|
|
917
|
+
if attempt < max_check_attempts - 1:
|
|
918
|
+
logger.debug(
|
|
919
|
+
f"消息 {message_id} 处理状态检查重试(第{attempt+1}次): "
|
|
920
|
+
f"handler={'存在' if handler else '不存在'}, "
|
|
921
|
+
f"is_consuming={is_consuming}"
|
|
922
|
+
)
|
|
923
|
+
await asyncio.sleep(check_interval)
|
|
924
|
+
|
|
925
|
+
# 最终状态判断:状态异常则拒绝消息
|
|
926
|
+
is_consuming, handler, _ = await self._get_consume_state()
|
|
927
|
+
if not is_consuming or not handler:
|
|
928
|
+
err_msg = f"消息 {message_id} 拒绝处理:handler={'存在' if handler else '不存在'}, is_consuming={is_consuming}"
|
|
929
|
+
logger.warning(err_msg)
|
|
637
930
|
try:
|
|
638
|
-
await
|
|
639
|
-
await asyncio.sleep(3)
|
|
931
|
+
await self._handle_business_retry(message, Exception(err_msg), drop=False)
|
|
640
932
|
except Exception as e:
|
|
641
|
-
logger.error(f"
|
|
933
|
+
logger.error(f"消息 {message_id} 拒绝处理失败: {e}")
|
|
642
934
|
return
|
|
643
935
|
|
|
644
|
-
|
|
645
|
-
if
|
|
936
|
+
# 检查重复处理
|
|
937
|
+
if await self._check_duplicate_message(message_id):
|
|
646
938
|
logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
|
|
647
939
|
await message.ack()
|
|
648
940
|
return
|
|
649
941
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
'channel_number': self.channel.number if self.channel else None,
|
|
655
|
-
'start_time': start_time
|
|
656
|
-
}
|
|
942
|
+
# 添加跟踪记录
|
|
943
|
+
channel, _, _ = await self._get_connection_resources()
|
|
944
|
+
channel_number = channel.number if channel else None
|
|
945
|
+
await self._add_tracking_message(message_id, message.delivery_tag, channel_number)
|
|
657
946
|
|
|
658
947
|
try:
|
|
659
948
|
logger.info(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
|
|
660
|
-
print(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
|
|
661
949
|
|
|
950
|
+
# 解析消息
|
|
662
951
|
parsed_data = await self._parse_message(message)
|
|
663
|
-
|
|
952
|
+
# 转换为MQMsgModel
|
|
953
|
+
if isinstance(parsed_data, dict):
|
|
954
|
+
msg_model = MQMsgModel(**parsed_data)
|
|
955
|
+
else:
|
|
956
|
+
msg_model = MQMsgModel(data=parsed_data)
|
|
957
|
+
|
|
958
|
+
# 调用业务处理器
|
|
959
|
+
await handler(msg_model, message)
|
|
664
960
|
|
|
961
|
+
# 处理成功:标记跟踪记录并确认消息
|
|
962
|
+
await self._mark_tracking_acked(message_id)
|
|
665
963
|
await message.ack()
|
|
666
|
-
self._tracking_messages[message_id]['acked'] = True
|
|
667
964
|
self._update_activity_timestamp()
|
|
668
965
|
self._update_message_processed_timestamp()
|
|
669
966
|
logger.info(f"消息 {message_id} 处理完成并确认")
|
|
670
967
|
|
|
671
968
|
except Exception as e:
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
retry_count += 1
|
|
675
|
-
|
|
676
|
-
logger.error(
|
|
677
|
-
f"消息 {message_id} 处理出错(第{retry_count}次重试): {str(e)}",
|
|
678
|
-
exc_info=True
|
|
679
|
-
)
|
|
680
|
-
|
|
681
|
-
if retry_count >= MAX_RETRY_COUNT:
|
|
682
|
-
logger.error(
|
|
683
|
-
f"消息 {message_id} 已达到最大重试次数{MAX_RETRY_COUNT},标记为失败")
|
|
684
|
-
await message.ack()
|
|
685
|
-
self._tracking_messages[message_id]['acked'] = True
|
|
686
|
-
self._update_activity_timestamp()
|
|
687
|
-
return
|
|
688
|
-
|
|
689
|
-
new_headers = current_headers.copy()
|
|
690
|
-
new_headers['x-retry-count'] = retry_count
|
|
691
|
-
|
|
692
|
-
new_message = Message(
|
|
693
|
-
body=message.body,
|
|
694
|
-
content_type=message.content_type,
|
|
695
|
-
headers=new_headers,
|
|
696
|
-
delivery_mode=message.delivery_mode
|
|
697
|
-
)
|
|
698
|
-
|
|
699
|
-
await message.reject(requeue=False)
|
|
700
|
-
self._tracking_messages[message_id]['acked'] = True
|
|
701
|
-
|
|
702
|
-
if self.exchange:
|
|
703
|
-
await self.exchange.publish(
|
|
704
|
-
new_message,
|
|
705
|
-
routing_key=self.routing_key or '#',
|
|
706
|
-
mandatory=True,
|
|
707
|
-
timeout=5.0
|
|
708
|
-
)
|
|
709
|
-
self._update_activity_timestamp()
|
|
710
|
-
logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
|
|
969
|
+
# 业务处理失败:触发重试逻辑
|
|
970
|
+
await self._handle_business_retry(message, e)
|
|
711
971
|
finally:
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
logger.info(f"已删除消息跟踪信息: {message_id}")
|
|
972
|
+
# 清理跟踪记录
|
|
973
|
+
await self._remove_tracking_message(message_id)
|
|
715
974
|
|
|
716
975
|
async def __aenter__(self):
|
|
717
976
|
await self.connect()
|