sycommon-python-lib 0.1.29__py3-none-any.whl → 0.1.38__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/models/mqlistener_config.py +1 -0
- sycommon/rabbitmq/rabbitmq_client.py +575 -258
- sycommon/rabbitmq/rabbitmq_service.py +255 -131
- sycommon/services.py +64 -22
- sycommon/synacos/example.py +153 -0
- sycommon/synacos/example2.py +129 -0
- sycommon/synacos/feign.py +51 -436
- sycommon/synacos/feign_client.py +317 -0
- sycommon/synacos/nacos_service.py +1 -1
- sycommon/synacos/param.py +75 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.38.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.38.dist-info}/RECORD +15 -11
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.38.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.38.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.38.dist-info}/top_level.txt +0 -0
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
from aio_pika.pool import Pool
|
|
2
|
+
from aio_pika.abc import AbstractRobustConnection
|
|
3
|
+
from aio_pika import connect_robust, Channel
|
|
4
|
+
from typing import Optional, List
|
|
1
5
|
import asyncio
|
|
2
6
|
import json
|
|
3
7
|
from typing import Callable, Coroutine, Optional, Dict, Any, Union, Set
|
|
@@ -25,6 +29,7 @@ class RabbitMQClient:
|
|
|
25
29
|
"""
|
|
26
30
|
RabbitMQ客户端(基于连接池),支持集群多节点配置
|
|
27
31
|
提供自动故障转移、连接恢复和消息可靠性保障
|
|
32
|
+
采用细粒度锁设计,彻底避免死锁隐患
|
|
28
33
|
"""
|
|
29
34
|
|
|
30
35
|
def __init__(
|
|
@@ -45,11 +50,6 @@ class RabbitMQClient:
|
|
|
45
50
|
prefetch_count: int = 2,
|
|
46
51
|
consumption_stall_threshold: int = 10
|
|
47
52
|
):
|
|
48
|
-
"""
|
|
49
|
-
初始化RabbitMQ客户端(依赖连接池)
|
|
50
|
-
|
|
51
|
-
:param connection_pool: 连接池实例
|
|
52
|
-
"""
|
|
53
53
|
# 连接池依赖
|
|
54
54
|
self.connection_pool = connection_pool
|
|
55
55
|
|
|
@@ -75,18 +75,18 @@ class RabbitMQClient:
|
|
|
75
75
|
# 消息处理参数
|
|
76
76
|
self.consumption_stall_threshold = consumption_stall_threshold
|
|
77
77
|
|
|
78
|
-
#
|
|
78
|
+
# 通道和资源对象(由 _connection_lock 保护)
|
|
79
79
|
self.channel: Optional[AbstractChannel] = None
|
|
80
80
|
self.exchange: Optional[AbstractExchange] = None
|
|
81
81
|
self.queue: Optional[AbstractQueue] = None
|
|
82
82
|
|
|
83
|
-
#
|
|
83
|
+
# 状态跟踪(按类型拆分锁保护)
|
|
84
84
|
self.actual_queue_name: Optional[str] = None
|
|
85
|
-
self._exchange_exists = False
|
|
86
|
-
self._queue_exists = False
|
|
87
|
-
self._queue_bound = False
|
|
88
|
-
self.
|
|
89
|
-
|
|
85
|
+
self._exchange_exists = False # 由 _connection_lock 保护
|
|
86
|
+
self._queue_exists = False # 由 _connection_lock 保护
|
|
87
|
+
self._queue_bound = False # 由 _connection_lock 保护
|
|
88
|
+
self._closed = False # 由 _connection_lock 保护
|
|
89
|
+
# 由 _consume_state_lock 保护:通过 consumer_tag 存在性判断是否在消费
|
|
90
90
|
self._consumer_tag: Optional[ConsumerTag] = None
|
|
91
91
|
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
92
92
|
self._last_message_processed = asyncio.get_event_loop().time()
|
|
@@ -95,44 +95,167 @@ class RabbitMQClient:
|
|
|
95
95
|
self.message_handler: Optional[Callable[
|
|
96
96
|
[Union[Dict[str, Any], str], AbstractIncomingMessage],
|
|
97
97
|
Coroutine[Any, Any, None]
|
|
98
|
-
]] = None
|
|
98
|
+
]] = None # 由 _consume_state_lock 保护
|
|
99
99
|
self._consuming_task: Optional[asyncio.Task] = None
|
|
100
100
|
self._reconnect_task: Optional[asyncio.Task] = None
|
|
101
101
|
self._keepalive_task: Optional[asyncio.Task] = None
|
|
102
102
|
self._monitor_task: Optional[asyncio.Task] = None
|
|
103
103
|
|
|
104
|
-
#
|
|
105
|
-
self.
|
|
104
|
+
# 消息处理跟踪(由 _tracking_lock 保护)
|
|
105
|
+
self._tracking_messages: Dict[str, Dict[str, Any]] = {}
|
|
106
|
+
|
|
107
|
+
# 细粒度锁(核心设计:按资源类型拆分,避免嵌套)
|
|
108
|
+
# 保护消费状态(message_handler、_consumer_tag)
|
|
109
|
+
self._consume_state_lock = asyncio.Lock()
|
|
110
|
+
self._tracking_lock = asyncio.Lock() # 保护消息跟踪记录(_tracking_messages)
|
|
111
|
+
# 保护连接/资源状态(channel、exchange、queue、_closed等)
|
|
112
|
+
self._connection_lock = asyncio.Lock()
|
|
106
113
|
|
|
107
114
|
@property
|
|
108
|
-
def is_connected(self) -> bool:
|
|
109
|
-
"""
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
115
|
+
async def is_connected(self) -> bool:
|
|
116
|
+
"""异步属性:检查当前通道是否有效(线程安全)"""
|
|
117
|
+
async with self._connection_lock:
|
|
118
|
+
return (not self._closed and
|
|
119
|
+
self.channel is not None and
|
|
120
|
+
not self.channel.is_closed and
|
|
121
|
+
self.exchange is not None)
|
|
114
122
|
|
|
115
123
|
def _update_activity_timestamp(self) -> None:
|
|
116
|
-
"""
|
|
124
|
+
"""更新最后活动时间戳(非共享状态,无需锁)"""
|
|
117
125
|
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
118
126
|
|
|
119
127
|
def _update_message_processed_timestamp(self) -> None:
|
|
120
|
-
"""
|
|
128
|
+
"""更新最后消息处理时间戳(非共享状态,无需锁)"""
|
|
121
129
|
self._last_message_processed = asyncio.get_event_loop().time()
|
|
122
130
|
|
|
131
|
+
# ------------------------------
|
|
132
|
+
# 消费状态操作(_consume_state_lock 专属保护)
|
|
133
|
+
# ------------------------------
|
|
134
|
+
async def _get_consume_state(self) -> tuple[bool, Optional[Callable], Optional[ConsumerTag]]:
|
|
135
|
+
"""安全获取消费相关状态(一次性获取,避免多次加锁)"""
|
|
136
|
+
async with self._consume_state_lock:
|
|
137
|
+
# 通过 _consumer_tag 是否存在判断是否在消费
|
|
138
|
+
is_consuming = self._consumer_tag is not None
|
|
139
|
+
return is_consuming, self.message_handler, self._consumer_tag
|
|
140
|
+
|
|
141
|
+
async def _set_consumer_tag(self, consumer_tag: Optional[ConsumerTag] = None):
|
|
142
|
+
"""安全更新消费者标签(替代原 _set_consume_state)"""
|
|
143
|
+
async with self._consume_state_lock:
|
|
144
|
+
old_tag = self._consumer_tag
|
|
145
|
+
self._consumer_tag = consumer_tag
|
|
146
|
+
old_is_consuming = old_tag is not None
|
|
147
|
+
new_is_consuming = consumer_tag is not None
|
|
148
|
+
if old_is_consuming != new_is_consuming:
|
|
149
|
+
logger.info(f"消费状态变更: {old_is_consuming} → {new_is_consuming}")
|
|
150
|
+
|
|
151
|
+
async def set_message_handler(self, handler):
|
|
152
|
+
"""设置消息处理器(加锁保护,避免并发修改)"""
|
|
153
|
+
async with self._consume_state_lock:
|
|
154
|
+
self.message_handler = handler
|
|
155
|
+
logger.info("消息处理器已设置")
|
|
156
|
+
|
|
157
|
+
# ------------------------------
|
|
158
|
+
# 连接状态操作(_connection_lock 专属保护)
|
|
159
|
+
# ------------------------------
|
|
160
|
+
async def _is_closed(self) -> bool:
|
|
161
|
+
"""检查客户端是否已关闭(线程安全)"""
|
|
162
|
+
async with self._connection_lock:
|
|
163
|
+
return self._closed
|
|
164
|
+
|
|
165
|
+
async def _mark_closed(self):
|
|
166
|
+
"""标记客户端已关闭(原子操作)"""
|
|
167
|
+
async with self._connection_lock:
|
|
168
|
+
self._closed = True
|
|
169
|
+
|
|
170
|
+
async def _get_connection_resources(self) -> tuple[Optional[AbstractChannel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
171
|
+
"""安全获取连接资源(channel/exchange/queue)"""
|
|
172
|
+
async with self._connection_lock:
|
|
173
|
+
return self.channel, self.exchange, self.queue
|
|
174
|
+
|
|
175
|
+
async def _reset_connection_state(self):
|
|
176
|
+
"""重置连接状态(用于重连时,原子操作)"""
|
|
177
|
+
async with self._connection_lock:
|
|
178
|
+
self._exchange_exists = False
|
|
179
|
+
self._queue_exists = False
|
|
180
|
+
self._queue_bound = False
|
|
181
|
+
self.channel = None
|
|
182
|
+
self.exchange = None
|
|
183
|
+
self.queue = None
|
|
184
|
+
self.actual_queue_name = None
|
|
185
|
+
|
|
186
|
+
async def _update_connection_resources(self, channel: AbstractChannel, exchange: AbstractExchange, queue: Optional[AbstractQueue] = None):
|
|
187
|
+
"""更新连接资源(原子操作)"""
|
|
188
|
+
async with self._connection_lock:
|
|
189
|
+
self.channel = channel
|
|
190
|
+
self.exchange = exchange
|
|
191
|
+
self.queue = queue
|
|
192
|
+
if queue:
|
|
193
|
+
self.actual_queue_name = queue.name
|
|
194
|
+
|
|
195
|
+
# ------------------------------
|
|
196
|
+
# 消息跟踪操作(_tracking_lock 专属保护)
|
|
197
|
+
# ------------------------------
|
|
198
|
+
async def _add_tracking_message(self, msg_id: str, delivery_tag: int, channel_number: Optional[int]):
|
|
199
|
+
"""添加消息跟踪记录(原子操作)"""
|
|
200
|
+
async with self._tracking_lock:
|
|
201
|
+
self._tracking_messages[msg_id] = {
|
|
202
|
+
'delivery_tag': delivery_tag,
|
|
203
|
+
'acked': False,
|
|
204
|
+
'channel_number': channel_number,
|
|
205
|
+
'start_time': asyncio.get_event_loop().time()
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async def _mark_tracking_acked(self, msg_id: str):
|
|
209
|
+
"""标记消息已确认(原子操作)"""
|
|
210
|
+
async with self._tracking_lock:
|
|
211
|
+
if msg_id in self._tracking_messages:
|
|
212
|
+
self._tracking_messages[msg_id]['acked'] = True
|
|
213
|
+
|
|
214
|
+
async def _remove_tracking_message(self, msg_id: str):
|
|
215
|
+
"""删除消息跟踪记录(原子操作,避免KeyError)"""
|
|
216
|
+
async with self._tracking_lock:
|
|
217
|
+
if msg_id in self._tracking_messages:
|
|
218
|
+
del self._tracking_messages[msg_id]
|
|
219
|
+
logger.info(f"已删除消息跟踪信息: {msg_id}")
|
|
220
|
+
|
|
221
|
+
async def _check_duplicate_message(self, msg_id: str) -> bool:
|
|
222
|
+
"""检查消息是否重复处理(原子操作)"""
|
|
223
|
+
async with self._tracking_lock:
|
|
224
|
+
return msg_id in self._tracking_messages
|
|
225
|
+
|
|
226
|
+
async def _get_tracking_count(self) -> int:
|
|
227
|
+
"""获取当前跟踪的消息数(原子操作)"""
|
|
228
|
+
async with self._tracking_lock:
|
|
229
|
+
return len(self._tracking_messages)
|
|
230
|
+
|
|
231
|
+
async def _cleanup_acked_tracking_messages(self) -> int:
|
|
232
|
+
"""清理已确认的跟踪记录(原子操作,返回清理数量)"""
|
|
233
|
+
async with self._tracking_lock:
|
|
234
|
+
acked_ids = [
|
|
235
|
+
msg_id for msg_id, info in self._tracking_messages.items() if info.get('acked')]
|
|
236
|
+
for msg_id in acked_ids:
|
|
237
|
+
del self._tracking_messages[msg_id]
|
|
238
|
+
return len(acked_ids)
|
|
239
|
+
|
|
240
|
+
async def _clear_tracking_messages(self):
|
|
241
|
+
"""清空所有跟踪记录(原子操作)"""
|
|
242
|
+
async with self._tracking_lock:
|
|
243
|
+
self._tracking_messages.clear()
|
|
244
|
+
|
|
245
|
+
# ------------------------------
|
|
246
|
+
# 基础工具方法
|
|
247
|
+
# ------------------------------
|
|
123
248
|
async def _get_channel(self) -> AbstractChannel:
|
|
124
|
-
"""
|
|
249
|
+
"""从通道池获取通道(使用上下文管理器,自动归还)"""
|
|
125
250
|
if not self.connection_pool.channel_pool:
|
|
126
251
|
raise Exception("连接池未初始化,请先调用init_pools")
|
|
127
252
|
|
|
128
|
-
# 使用async with获取通道,并通过变量返回
|
|
129
253
|
async with self.connection_pool.channel_pool.acquire() as channel:
|
|
130
254
|
return channel
|
|
131
255
|
|
|
132
256
|
async def _check_exchange_exists(self, channel: AbstractChannel) -> bool:
|
|
133
257
|
"""检查交换机是否存在"""
|
|
134
258
|
try:
|
|
135
|
-
# 使用被动模式检查交换机
|
|
136
259
|
await asyncio.wait_for(
|
|
137
260
|
channel.declare_exchange(
|
|
138
261
|
name=self.exchange_name,
|
|
@@ -150,7 +273,6 @@ class RabbitMQClient:
|
|
|
150
273
|
if not self.queue_name:
|
|
151
274
|
return False
|
|
152
275
|
try:
|
|
153
|
-
# 使用被动模式检查队列
|
|
154
276
|
await asyncio.wait_for(
|
|
155
277
|
channel.declare_queue(
|
|
156
278
|
name=self.queue_name,
|
|
@@ -163,7 +285,7 @@ class RabbitMQClient:
|
|
|
163
285
|
return False
|
|
164
286
|
|
|
165
287
|
async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
|
|
166
|
-
"""
|
|
288
|
+
"""将队列绑定到交换机(带重试)"""
|
|
167
289
|
bind_routing_key = self.routing_key if self.routing_key else '#'
|
|
168
290
|
|
|
169
291
|
for attempt in range(MAX_RETRY_COUNT + 1):
|
|
@@ -176,7 +298,7 @@ class RabbitMQClient:
|
|
|
176
298
|
timeout=self.rpc_timeout
|
|
177
299
|
)
|
|
178
300
|
logger.info(
|
|
179
|
-
f"队列 '{
|
|
301
|
+
f"队列 '{queue.name}' 已绑定到交换机 '{exchange.name}',路由键: {bind_routing_key}")
|
|
180
302
|
return True
|
|
181
303
|
except Exception as e:
|
|
182
304
|
logger.warning(
|
|
@@ -185,44 +307,53 @@ class RabbitMQClient:
|
|
|
185
307
|
await asyncio.sleep(1)
|
|
186
308
|
return False
|
|
187
309
|
|
|
310
|
+
# ------------------------------
|
|
311
|
+
# 核心业务方法
|
|
312
|
+
# ------------------------------
|
|
188
313
|
async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
|
|
189
|
-
"""
|
|
190
|
-
|
|
191
|
-
"""
|
|
192
|
-
logger.debug(
|
|
314
|
+
"""从连接池获取资源并初始化(交换机、队列)"""
|
|
315
|
+
logger.info(
|
|
193
316
|
f"连接参数 - force_reconnect={force_reconnect}, "
|
|
194
317
|
f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
|
|
195
318
|
)
|
|
196
319
|
|
|
197
|
-
#
|
|
198
|
-
if self.
|
|
320
|
+
# 检查是否已关闭
|
|
321
|
+
if await self._is_closed():
|
|
322
|
+
raise Exception("客户端已关闭,无法连接")
|
|
323
|
+
|
|
324
|
+
# 检查是否已连接(非强制重连则直接返回)
|
|
325
|
+
if await self.is_connected and not force_reconnect:
|
|
326
|
+
logger.info("已处于连接状态,无需重复连接")
|
|
199
327
|
return
|
|
200
328
|
|
|
201
|
-
#
|
|
329
|
+
# 取消现有重连任务
|
|
202
330
|
if self._reconnect_task and not self._reconnect_task.done():
|
|
203
331
|
self._reconnect_task.cancel()
|
|
332
|
+
try:
|
|
333
|
+
await self._reconnect_task
|
|
334
|
+
except asyncio.CancelledError:
|
|
335
|
+
logger.info("旧重连任务已取消")
|
|
204
336
|
|
|
205
|
-
#
|
|
206
|
-
self.
|
|
207
|
-
self.
|
|
208
|
-
self.
|
|
337
|
+
# 重置连接状态和跟踪记录
|
|
338
|
+
await self._reset_connection_state()
|
|
339
|
+
await self._clear_tracking_messages()
|
|
340
|
+
await self._set_consumer_tag(None) # 重置消费者标签(停止消费)
|
|
209
341
|
|
|
210
342
|
retries = 0
|
|
211
343
|
last_exception = None
|
|
212
344
|
|
|
213
345
|
while retries < self.max_reconnection_attempts:
|
|
214
346
|
try:
|
|
215
|
-
#
|
|
216
|
-
|
|
217
|
-
await
|
|
347
|
+
# 获取新通道
|
|
348
|
+
channel = await self._get_channel()
|
|
349
|
+
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
218
350
|
|
|
219
351
|
# 处理交换机
|
|
220
|
-
exchange_exists = await self._check_exchange_exists(
|
|
352
|
+
exchange_exists = await self._check_exchange_exists(channel)
|
|
221
353
|
if not exchange_exists:
|
|
222
354
|
if self.create_if_not_exists:
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
self.channel.declare_exchange(
|
|
355
|
+
exchange = await asyncio.wait_for(
|
|
356
|
+
channel.declare_exchange(
|
|
226
357
|
name=self.exchange_name,
|
|
227
358
|
type=self.exchange_type,
|
|
228
359
|
durable=self.durable,
|
|
@@ -235,22 +366,21 @@ class RabbitMQClient:
|
|
|
235
366
|
raise Exception(
|
|
236
367
|
f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
|
|
237
368
|
else:
|
|
238
|
-
|
|
239
|
-
self.exchange = await self.channel.get_exchange(self.exchange_name)
|
|
369
|
+
exchange = await channel.get_exchange(self.exchange_name)
|
|
240
370
|
logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
|
|
241
371
|
|
|
242
372
|
# 处理队列
|
|
373
|
+
queue = None
|
|
243
374
|
if declare_queue and self.queue_name:
|
|
244
|
-
queue_exists = await self._check_queue_exists(
|
|
375
|
+
queue_exists = await self._check_queue_exists(channel)
|
|
245
376
|
|
|
246
377
|
if not queue_exists:
|
|
247
378
|
if not self.create_if_not_exists:
|
|
248
379
|
raise Exception(
|
|
249
380
|
f"队列 '{self.queue_name}' 不存在且不允许自动创建")
|
|
250
381
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
self.channel.declare_queue(
|
|
382
|
+
queue = await asyncio.wait_for(
|
|
383
|
+
channel.declare_queue(
|
|
254
384
|
name=self.queue_name,
|
|
255
385
|
durable=self.durable,
|
|
256
386
|
auto_delete=self.auto_delete,
|
|
@@ -258,46 +388,46 @@ class RabbitMQClient:
|
|
|
258
388
|
),
|
|
259
389
|
timeout=self.rpc_timeout
|
|
260
390
|
)
|
|
261
|
-
self.actual_queue_name = self.queue_name
|
|
262
391
|
logger.info(f"已创建队列 '{self.queue_name}'")
|
|
263
392
|
else:
|
|
264
|
-
|
|
265
|
-
self.queue = await self.channel.get_queue(self.queue_name)
|
|
266
|
-
self.actual_queue_name = self.queue_name
|
|
393
|
+
queue = await channel.get_queue(self.queue_name)
|
|
267
394
|
logger.info(f"使用已存在的队列 '{self.queue_name}'")
|
|
268
395
|
|
|
269
396
|
# 绑定队列到交换机
|
|
270
|
-
if
|
|
271
|
-
bound = await self._bind_queue(
|
|
397
|
+
if queue and exchange:
|
|
398
|
+
bound = await self._bind_queue(channel, queue, exchange)
|
|
272
399
|
if not bound:
|
|
273
|
-
raise Exception(f"队列 '{
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
self.actual_queue_name = None
|
|
278
|
-
logger.debug(f"跳过队列 '{self.queue_name}' 的声明和绑定")
|
|
400
|
+
raise Exception(f"队列 '{queue.name}' 绑定到交换机失败")
|
|
401
|
+
|
|
402
|
+
# 更新连接资源
|
|
403
|
+
await self._update_connection_resources(channel, exchange, queue)
|
|
279
404
|
|
|
280
405
|
# 验证连接状态
|
|
281
|
-
if not self.is_connected:
|
|
406
|
+
if not await self.is_connected:
|
|
282
407
|
raise Exception("连接验证失败,状态异常")
|
|
283
408
|
|
|
284
|
-
#
|
|
285
|
-
|
|
409
|
+
# 重新开始消费(如果已设置处理器且之前在消费)
|
|
410
|
+
_, handler, consumer_tag = await self._get_consume_state()
|
|
411
|
+
if handler and consumer_tag: # 有处理器且之前有消费标签,说明需要恢复消费
|
|
286
412
|
await self.start_consuming()
|
|
287
413
|
|
|
288
|
-
#
|
|
414
|
+
# 启动监控和保活任务
|
|
289
415
|
self._start_monitoring()
|
|
290
416
|
self._start_keepalive()
|
|
291
417
|
|
|
292
418
|
self._update_activity_timestamp()
|
|
419
|
+
|
|
420
|
+
# 首次启动时延迟1秒(避免初始化未完成就接收消息)
|
|
421
|
+
if not force_reconnect:
|
|
422
|
+
logger.info("客户端初始化成功,延迟1秒接收消息(解决启动时序问题)")
|
|
423
|
+
await asyncio.sleep(1)
|
|
424
|
+
|
|
293
425
|
logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
|
|
294
426
|
return
|
|
295
427
|
|
|
296
428
|
except Exception as e:
|
|
297
429
|
last_exception = e
|
|
298
430
|
logger.warning(f"资源初始化失败: {str(e)},重试中...")
|
|
299
|
-
# 释放当前通道(放回池并重新获取)
|
|
300
|
-
self.channel = None
|
|
301
431
|
retries += 1
|
|
302
432
|
if retries < self.max_reconnection_attempts:
|
|
303
433
|
await asyncio.sleep(self.reconnection_delay)
|
|
@@ -307,102 +437,146 @@ class RabbitMQClient:
|
|
|
307
437
|
f"经过{self.max_reconnection_attempts}次重试后仍无法初始化客户端。最后错误: {str(last_exception)}")
|
|
308
438
|
|
|
309
439
|
def _start_monitoring(self) -> None:
|
|
310
|
-
"""
|
|
311
|
-
if self.
|
|
440
|
+
"""启动连接和消费监控任务(无锁,仅通过原子方法访问状态)"""
|
|
441
|
+
if self._monitor_task and not self._monitor_task.done():
|
|
312
442
|
return
|
|
313
443
|
|
|
314
444
|
async def monitor():
|
|
315
|
-
while not
|
|
445
|
+
while not await self._is_closed():
|
|
316
446
|
try:
|
|
317
447
|
# 检查通道状态
|
|
318
|
-
|
|
448
|
+
channel, _, _ = await self._get_connection_resources()
|
|
449
|
+
if channel and channel.is_closed:
|
|
319
450
|
logger.warning("检测到通道已关闭,尝试重建")
|
|
320
451
|
await self._recreate_channel()
|
|
321
452
|
continue
|
|
322
453
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
454
|
+
current_time = asyncio.get_event_loop().time()
|
|
455
|
+
|
|
456
|
+
# 清理已确认的跟踪记录
|
|
457
|
+
cleaned_count = await self._cleanup_acked_tracking_messages()
|
|
458
|
+
if cleaned_count > 0:
|
|
459
|
+
logger.info(f"清理了 {cleaned_count} 条已确认消息记录")
|
|
460
|
+
|
|
461
|
+
# 检查消费停滞(仅当有消费者标签时)
|
|
462
|
+
_, handler, consumer_tag = await self._get_consume_state()
|
|
463
|
+
if consumer_tag: # 有消费标签说明正在消费
|
|
464
|
+
tracking_count = await self._get_tracking_count()
|
|
326
465
|
if current_time - self._last_message_processed > self.consumption_stall_threshold:
|
|
327
|
-
if
|
|
466
|
+
if tracking_count > 0:
|
|
328
467
|
logger.warning(
|
|
329
|
-
f"消费停滞,但有 {
|
|
468
|
+
f"消费停滞,但有 {tracking_count} 个消息正在处理,暂不重启")
|
|
330
469
|
else:
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
470
|
+
logger.info("消费停滞且无消息处理,重启消费")
|
|
471
|
+
try:
|
|
472
|
+
await self.stop_consuming()
|
|
473
|
+
await asyncio.sleep(1)
|
|
474
|
+
# 检查处理器是否存在
|
|
475
|
+
_, handler, _ = await self._get_consume_state()
|
|
476
|
+
if handler:
|
|
477
|
+
await self.start_consuming()
|
|
478
|
+
else:
|
|
479
|
+
logger.error("消费处理器已丢失,无法重启消费")
|
|
480
|
+
except Exception as e:
|
|
481
|
+
logger.error(
|
|
482
|
+
f"重启消费失败: {str(e)}", exc_info=True)
|
|
483
|
+
await self._set_consumer_tag(None)
|
|
484
|
+
|
|
335
485
|
except Exception as e:
|
|
336
|
-
logger.error(f"监控任务出错: {str(e)}")
|
|
337
|
-
await asyncio.sleep(1)
|
|
486
|
+
logger.error(f"监控任务出错: {str(e)}", exc_info=True)
|
|
338
487
|
|
|
339
|
-
await asyncio.sleep(
|
|
488
|
+
await asyncio.sleep(60) # 监控间隔60秒
|
|
340
489
|
|
|
341
490
|
self._monitor_task = asyncio.create_task(monitor())
|
|
491
|
+
logger.info("监控任务已启动")
|
|
342
492
|
|
|
343
493
|
async def _recreate_channel(self) -> None:
|
|
494
|
+
"""重建通道并恢复资源(无锁嵌套)"""
|
|
495
|
+
# 先停止消费
|
|
496
|
+
await self._set_consumer_tag(None)
|
|
497
|
+
logger.info("开始重建通道...")
|
|
498
|
+
|
|
344
499
|
try:
|
|
345
|
-
#
|
|
346
|
-
|
|
347
|
-
await
|
|
500
|
+
# 获取新通道
|
|
501
|
+
channel = await self._get_channel()
|
|
502
|
+
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
348
503
|
|
|
349
504
|
# 重新获取交换机
|
|
350
|
-
|
|
505
|
+
exchange = await channel.get_exchange(self.exchange_name)
|
|
351
506
|
|
|
352
|
-
#
|
|
507
|
+
# 重新获取队列并绑定
|
|
508
|
+
queue = None
|
|
353
509
|
if self.queue_name:
|
|
354
|
-
|
|
355
|
-
if
|
|
356
|
-
await self._bind_queue(
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
510
|
+
queue = await channel.get_queue(self.queue_name)
|
|
511
|
+
if queue and exchange:
|
|
512
|
+
bound = await self._bind_queue(channel, queue, exchange)
|
|
513
|
+
if not bound:
|
|
514
|
+
raise Exception("队列绑定失败,通道重建不完整")
|
|
515
|
+
|
|
516
|
+
# 更新连接资源
|
|
517
|
+
await self._update_connection_resources(channel, exchange, queue)
|
|
518
|
+
|
|
519
|
+
# 重新开始消费(如果有处理器)
|
|
520
|
+
_, handler, _ = await self._get_consume_state()
|
|
521
|
+
if handler:
|
|
360
522
|
await self.start_consuming()
|
|
361
523
|
|
|
524
|
+
# 清空跟踪记录
|
|
525
|
+
await self._clear_tracking_messages()
|
|
362
526
|
logger.info("通道已重建并恢复服务")
|
|
363
527
|
self._update_activity_timestamp()
|
|
364
528
|
except Exception as e:
|
|
365
|
-
logger.error(f"通道重建失败: {str(e)},触发重连")
|
|
529
|
+
logger.error(f"通道重建失败: {str(e)},触发重连", exc_info=True)
|
|
530
|
+
await self._set_consumer_tag(None)
|
|
366
531
|
await self.connect(force_reconnect=True)
|
|
367
532
|
|
|
368
533
|
def _start_keepalive(self) -> None:
|
|
369
|
-
"""
|
|
370
|
-
if self.
|
|
534
|
+
"""启动连接保活任务(无锁,仅通过原子方法访问状态)"""
|
|
535
|
+
if self._keepalive_task and not self._keepalive_task.done():
|
|
371
536
|
return
|
|
372
537
|
|
|
373
538
|
async def keepalive():
|
|
374
|
-
while not
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
f"连接 {self.connection_pool.heartbeat*1.5}s 无活动,执行保活检查")
|
|
380
|
-
try:
|
|
381
|
-
if self.channel.is_closed:
|
|
382
|
-
logger.warning("连接已关闭,触发重连")
|
|
383
|
-
await self.connect(force_reconnect=True)
|
|
384
|
-
return
|
|
385
|
-
|
|
386
|
-
# 轻量级操作保持连接活跃
|
|
387
|
-
await asyncio.wait_for(
|
|
388
|
-
self.channel.declare_exchange(
|
|
389
|
-
name=self.exchange_name,
|
|
390
|
-
type=self.exchange_type,
|
|
391
|
-
passive=True
|
|
392
|
-
),
|
|
393
|
-
timeout=5
|
|
394
|
-
)
|
|
395
|
-
self._update_activity_timestamp()
|
|
396
|
-
except Exception as e:
|
|
397
|
-
logger.warning(f"保活检查失败: {str(e)},触发重连")
|
|
539
|
+
while not await self._is_closed():
|
|
540
|
+
try:
|
|
541
|
+
# 检查连接状态
|
|
542
|
+
if not await self.is_connected:
|
|
543
|
+
logger.warning("保活任务检测到连接断开,触发重连")
|
|
398
544
|
await self.connect(force_reconnect=True)
|
|
545
|
+
await asyncio.sleep(5)
|
|
546
|
+
continue
|
|
547
|
+
|
|
548
|
+
current_time = asyncio.get_event_loop().time()
|
|
549
|
+
# 检查活动时间
|
|
550
|
+
if current_time - self._last_activity_timestamp > self.connection_pool.heartbeat * 2:
|
|
551
|
+
logger.info(
|
|
552
|
+
f"连接 {self.connection_pool.heartbeat*2}s 无活动,执行保活检查")
|
|
553
|
+
channel, exchange, _ = await self._get_connection_resources()
|
|
554
|
+
if channel and not channel.is_closed and exchange:
|
|
555
|
+
# 轻量级操作:检查交换机是否存在
|
|
556
|
+
await asyncio.wait_for(
|
|
557
|
+
channel.declare_exchange(
|
|
558
|
+
name=self.exchange_name,
|
|
559
|
+
type=self.exchange_type,
|
|
560
|
+
passive=True
|
|
561
|
+
),
|
|
562
|
+
timeout=5
|
|
563
|
+
)
|
|
564
|
+
self._update_activity_timestamp()
|
|
565
|
+
logger.info("保活检查成功")
|
|
566
|
+
else:
|
|
567
|
+
raise Exception("连接资源无效")
|
|
399
568
|
|
|
400
|
-
|
|
569
|
+
except Exception as e:
|
|
570
|
+
logger.warning(f"保活检查失败: {str(e)},触发重连")
|
|
571
|
+
await self.connect(force_reconnect=True)
|
|
572
|
+
|
|
573
|
+
await asyncio.sleep(self.connection_pool.heartbeat)
|
|
401
574
|
|
|
402
575
|
self._keepalive_task = asyncio.create_task(keepalive())
|
|
576
|
+
logger.info("保活任务已启动")
|
|
403
577
|
|
|
404
578
|
async def _schedule_reconnect(self) -> None:
|
|
405
|
-
"""
|
|
579
|
+
"""安排重新连接(无锁)"""
|
|
406
580
|
if self._reconnect_task and not self._reconnect_task.done():
|
|
407
581
|
return
|
|
408
582
|
|
|
@@ -411,38 +585,48 @@ class RabbitMQClient:
|
|
|
411
585
|
async def reconnect():
|
|
412
586
|
try:
|
|
413
587
|
await asyncio.sleep(self.reconnection_delay)
|
|
414
|
-
if not self.
|
|
588
|
+
if not await self._is_closed():
|
|
415
589
|
await self.connect(force_reconnect=True)
|
|
416
590
|
except Exception as e:
|
|
417
591
|
logger.error(f"重连任务失败: {str(e)}")
|
|
418
|
-
if not self.
|
|
592
|
+
if not await self._is_closed():
|
|
419
593
|
await self._schedule_reconnect()
|
|
420
594
|
|
|
421
595
|
self._reconnect_task = asyncio.create_task(reconnect())
|
|
422
596
|
|
|
423
597
|
async def close(self) -> None:
|
|
424
|
-
"""
|
|
425
|
-
self.
|
|
426
|
-
|
|
598
|
+
"""关闭客户端并释放资源(原子操作,无锁嵌套)"""
|
|
599
|
+
if await self._is_closed():
|
|
600
|
+
logger.info("客户端已关闭,无需重复操作")
|
|
601
|
+
return
|
|
602
|
+
|
|
603
|
+
logger.info("开始关闭RabbitMQ客户端...")
|
|
427
604
|
|
|
428
|
-
#
|
|
429
|
-
|
|
430
|
-
|
|
605
|
+
# 标记为已关闭
|
|
606
|
+
await self._mark_closed()
|
|
607
|
+
|
|
608
|
+
# 停止消费
|
|
609
|
+
await self.stop_consuming()
|
|
610
|
+
|
|
611
|
+
# 取消所有后台任务
|
|
612
|
+
tasks = [self._keepalive_task,
|
|
613
|
+
self._reconnect_task, self._monitor_task]
|
|
614
|
+
for task in tasks:
|
|
431
615
|
if task and not task.done():
|
|
432
616
|
task.cancel()
|
|
433
617
|
try:
|
|
434
618
|
await task
|
|
435
619
|
except asyncio.CancelledError:
|
|
436
|
-
|
|
620
|
+
logger.info(f"任务 {task.get_name()} 已取消")
|
|
437
621
|
|
|
438
|
-
#
|
|
439
|
-
self.
|
|
440
|
-
self.
|
|
441
|
-
self.
|
|
442
|
-
|
|
443
|
-
|
|
622
|
+
# 重置所有状态和资源
|
|
623
|
+
await self._reset_connection_state()
|
|
624
|
+
await self._clear_tracking_messages()
|
|
625
|
+
async with self._consume_state_lock:
|
|
626
|
+
self.message_handler = None
|
|
627
|
+
self._consumer_tag = None
|
|
444
628
|
|
|
445
|
-
logger.info("RabbitMQ
|
|
629
|
+
logger.info("RabbitMQ客户端已完全关闭")
|
|
446
630
|
|
|
447
631
|
async def publish(
|
|
448
632
|
self,
|
|
@@ -452,13 +636,14 @@ class RabbitMQClient:
|
|
|
452
636
|
headers: Optional[Dict[str, Any]] = None,
|
|
453
637
|
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
|
|
454
638
|
) -> None:
|
|
455
|
-
"""
|
|
456
|
-
if
|
|
457
|
-
|
|
458
|
-
await self.connect(force_reconnect=True)
|
|
639
|
+
"""发布消息(从池获取通道,自动重试,无锁冲突)"""
|
|
640
|
+
if await self._is_closed():
|
|
641
|
+
raise Exception("客户端已关闭,无法发布消息")
|
|
459
642
|
|
|
460
|
-
|
|
461
|
-
|
|
643
|
+
# 检查连接状态
|
|
644
|
+
if not await self.is_connected:
|
|
645
|
+
logger.warning("连接已断开,尝试重连后发布消息")
|
|
646
|
+
await self.connect(force_reconnect=True)
|
|
462
647
|
|
|
463
648
|
# 处理消息体
|
|
464
649
|
if isinstance(message_body, dict):
|
|
@@ -481,24 +666,24 @@ class RabbitMQClient:
|
|
|
481
666
|
max_retries = 2
|
|
482
667
|
while retry_count < max_retries:
|
|
483
668
|
try:
|
|
484
|
-
# 从池获取新通道用于发布(避免长时间占用消费通道)
|
|
485
669
|
async with self.connection_pool.channel_pool.acquire() as publish_channel:
|
|
486
670
|
exchange = await publish_channel.get_exchange(self.exchange_name)
|
|
487
671
|
confirmed = await exchange.publish(
|
|
488
672
|
message,
|
|
489
673
|
routing_key=routing_key or self.routing_key or '#',
|
|
490
674
|
mandatory=True,
|
|
491
|
-
timeout=5
|
|
675
|
+
timeout=5
|
|
492
676
|
)
|
|
493
677
|
if not confirmed:
|
|
494
678
|
raise Exception("消息未被服务器确认接收")
|
|
495
679
|
|
|
496
680
|
self._update_activity_timestamp()
|
|
497
|
-
logger.
|
|
681
|
+
logger.info(
|
|
682
|
+
f"消息已发布到交换机 '{self.exchange_name}'(路由键: {routing_key or self.routing_key or '#'})")
|
|
498
683
|
return
|
|
499
|
-
except (ConnectionClosed, ChannelInvalidStateError):
|
|
684
|
+
except (ConnectionClosed, ChannelInvalidStateError, asyncio.TimeoutError):
|
|
500
685
|
retry_count += 1
|
|
501
|
-
logger.warning(f"
|
|
686
|
+
logger.warning(f"连接异常,尝试重连后重新发布 (重试次数: {retry_count})")
|
|
502
687
|
await self.connect(force_reconnect=True)
|
|
503
688
|
except Exception as e:
|
|
504
689
|
retry_count += 1
|
|
@@ -508,88 +693,122 @@ class RabbitMQClient:
|
|
|
508
693
|
|
|
509
694
|
raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
|
|
510
695
|
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
696
|
+
async def _safe_cancel_consumer(self, consumer_tag: ConsumerTag, queue: AbstractQueue) -> bool:
|
|
697
|
+
"""安全取消消费者(无锁,仅操作传入的局部变量)"""
|
|
698
|
+
try:
|
|
699
|
+
await asyncio.wait_for(
|
|
700
|
+
queue.cancel(consumer_tag),
|
|
701
|
+
timeout=self.rpc_timeout
|
|
702
|
+
)
|
|
703
|
+
logger.info(f"消费者 {consumer_tag} 已取消")
|
|
704
|
+
return True
|
|
705
|
+
except Exception as e:
|
|
706
|
+
logger.error(f"取消消费者 {consumer_tag} 异常: {str(e)}")
|
|
707
|
+
return False
|
|
514
708
|
|
|
515
709
|
async def start_consuming(self) -> ConsumerTag:
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
if not self.is_connected:
|
|
710
|
+
"""启动消费(无锁嵌套,通过原子方法获取/更新状态)"""
|
|
711
|
+
# 检查客户端状态
|
|
712
|
+
if await self._is_closed():
|
|
713
|
+
raise Exception("客户端已关闭,无法启动消费")
|
|
714
|
+
|
|
715
|
+
# 检查连接状态(确保通道完全就绪)
|
|
716
|
+
if not await self.is_connected:
|
|
525
717
|
await self.connect()
|
|
526
718
|
|
|
527
|
-
|
|
719
|
+
# 确保通道未关闭(解决启动时通道初始化滞后问题)
|
|
720
|
+
channel, _, _ = await self._get_connection_resources()
|
|
721
|
+
max_wait_attempts = 5
|
|
722
|
+
wait_interval = 0.5
|
|
723
|
+
for attempt in range(max_wait_attempts):
|
|
724
|
+
if channel and not channel.is_closed:
|
|
725
|
+
break
|
|
726
|
+
logger.debug(f"等待通道就绪(第{attempt+1}/{max_wait_attempts}次)")
|
|
727
|
+
await asyncio.sleep(wait_interval)
|
|
728
|
+
channel, _, _ = await self._get_connection_resources()
|
|
729
|
+
if not channel or channel.is_closed:
|
|
730
|
+
raise Exception("通道初始化失败,无法启动消费")
|
|
731
|
+
|
|
732
|
+
# 获取消费状态和资源
|
|
733
|
+
_, handler, consumer_tag = await self._get_consume_state()
|
|
734
|
+
_, exchange, queue = await self._get_connection_resources()
|
|
735
|
+
|
|
736
|
+
# 检查是否已在消费(通过 consumer_tag 是否存在判断)
|
|
737
|
+
if consumer_tag:
|
|
738
|
+
logger.info(f"已经在消费中,返回现有consumer_tag: {consumer_tag}")
|
|
739
|
+
return consumer_tag
|
|
740
|
+
|
|
741
|
+
# 检查必要条件
|
|
742
|
+
if not handler:
|
|
743
|
+
raise Exception("未设置消息处理函数,请先调用set_message_handler")
|
|
744
|
+
if not queue:
|
|
528
745
|
raise Exception("队列未初始化,无法开始消费")
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
raise Exception("未设置消息处理函数")
|
|
532
|
-
|
|
533
|
-
self._is_consuming = True
|
|
534
|
-
logger.info(f"开始消费队列: {self.actual_queue_name}")
|
|
746
|
+
if not channel or channel.is_closed:
|
|
747
|
+
raise Exception("通道无效,无法开始消费")
|
|
535
748
|
|
|
536
749
|
try:
|
|
537
|
-
#
|
|
538
|
-
|
|
750
|
+
# 启动消费
|
|
751
|
+
new_consumer_tag = await queue.consume(
|
|
539
752
|
self._message_wrapper,
|
|
540
753
|
no_ack=False # 手动确认消息
|
|
541
754
|
)
|
|
542
755
|
|
|
543
|
-
|
|
544
|
-
if not self._consumer_tag:
|
|
756
|
+
if not new_consumer_tag:
|
|
545
757
|
raise Exception("未能获取到有效的consumer_tag")
|
|
546
758
|
|
|
759
|
+
# 更新消费状态(设置消费者标签)
|
|
760
|
+
await self._set_consumer_tag(new_consumer_tag)
|
|
547
761
|
logger.info(
|
|
548
|
-
f"消费者已启动,队列: {
|
|
549
|
-
return
|
|
762
|
+
f"消费者已启动,队列: {queue.name}, tag: {new_consumer_tag}")
|
|
763
|
+
return new_consumer_tag
|
|
550
764
|
except Exception as e:
|
|
551
|
-
|
|
765
|
+
# 异常时回滚状态
|
|
766
|
+
await self._set_consumer_tag(None)
|
|
552
767
|
logger.error(f"启动消费失败: {str(e)}", exc_info=True)
|
|
553
768
|
raise
|
|
554
769
|
|
|
555
|
-
async def _safe_cancel_consumer(self) -> bool:
|
|
556
|
-
if not self._consumer_tag or not self.queue or not self.channel:
|
|
557
|
-
return True
|
|
558
|
-
|
|
559
|
-
try:
|
|
560
|
-
await asyncio.wait_for(
|
|
561
|
-
self.queue.cancel(self._consumer_tag),
|
|
562
|
-
timeout=self.rpc_timeout
|
|
563
|
-
)
|
|
564
|
-
logger.info(f"消费者 {self._consumer_tag} 已取消")
|
|
565
|
-
return True
|
|
566
|
-
except Exception as e:
|
|
567
|
-
logger.error(f"取消消费者异常: {str(e)}")
|
|
568
|
-
return False
|
|
569
|
-
|
|
570
770
|
async def stop_consuming(self) -> None:
|
|
571
|
-
|
|
771
|
+
"""停止消费(无锁嵌套,通过原子方法获取/更新状态)"""
|
|
772
|
+
# 获取消费状态和资源
|
|
773
|
+
_, _, consumer_tag = await self._get_consume_state()
|
|
774
|
+
_, _, queue = await self._get_connection_resources()
|
|
775
|
+
|
|
776
|
+
if not consumer_tag: # 无消费标签说明未在消费
|
|
777
|
+
logger.info("未处于消费状态,无需停止")
|
|
572
778
|
return
|
|
573
779
|
|
|
574
|
-
|
|
780
|
+
logger.info(f"开始停止消费(consumer_tag: {consumer_tag})")
|
|
575
781
|
|
|
576
|
-
|
|
577
|
-
|
|
782
|
+
# 先清除消费标签
|
|
783
|
+
await self._set_consumer_tag(None)
|
|
578
784
|
|
|
579
|
-
#
|
|
580
|
-
if self.
|
|
581
|
-
|
|
582
|
-
f"等待 {len(self._processing_message_ids)} 个正在处理的消息完成...")
|
|
583
|
-
while self._processing_message_ids and not self._closed:
|
|
584
|
-
await asyncio.sleep(0.1)
|
|
785
|
+
# 取消消费者
|
|
786
|
+
if consumer_tag and queue and not await self._is_closed():
|
|
787
|
+
await self._safe_cancel_consumer(consumer_tag, queue)
|
|
585
788
|
|
|
586
|
-
#
|
|
587
|
-
|
|
588
|
-
|
|
789
|
+
# 等待所有正在处理的消息完成
|
|
790
|
+
tracking_count = await self._get_tracking_count()
|
|
791
|
+
if tracking_count > 0:
|
|
792
|
+
logger.info(f"等待 {tracking_count} 个正在处理的消息完成...")
|
|
793
|
+
wait_start = asyncio.get_event_loop().time()
|
|
794
|
+
while True:
|
|
795
|
+
# 检查是否超时或已关闭
|
|
796
|
+
if await self._is_closed() or asyncio.get_event_loop().time() - wait_start > 30:
|
|
797
|
+
timeout = asyncio.get_event_loop().time() - wait_start > 30
|
|
798
|
+
if timeout:
|
|
799
|
+
logger.warning("等待消息处理超时,强制清理跟踪记录")
|
|
800
|
+
await self._clear_tracking_messages()
|
|
801
|
+
break
|
|
802
|
+
# 检查跟踪记录是否为空
|
|
803
|
+
current_count = await self._get_tracking_count()
|
|
804
|
+
if current_count == 0:
|
|
805
|
+
break
|
|
806
|
+
await asyncio.sleep(1)
|
|
589
807
|
|
|
590
|
-
logger.info(f"已停止消费队列: {
|
|
808
|
+
logger.info(f"已停止消费队列: {queue.name if queue else '未知'}")
|
|
591
809
|
|
|
592
810
|
async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
|
|
811
|
+
"""解析消息体(无锁,仅处理局部变量)"""
|
|
593
812
|
try:
|
|
594
813
|
body_str = message.body.decode('utf-8')
|
|
595
814
|
self._update_activity_timestamp()
|
|
@@ -598,61 +817,82 @@ class RabbitMQClient:
|
|
|
598
817
|
return json.loads(body_str)
|
|
599
818
|
return body_str
|
|
600
819
|
except json.JSONDecodeError:
|
|
601
|
-
logger.warning(
|
|
820
|
+
logger.warning(
|
|
821
|
+
f"消息 {message.message_id or id(message)} 解析JSON失败,返回原始字符串")
|
|
602
822
|
return body_str
|
|
603
823
|
except Exception as e:
|
|
604
|
-
logger.error(
|
|
824
|
+
logger.error(
|
|
825
|
+
f"消息 {message.message_id or id(message)} 解析出错: {str(e)}")
|
|
605
826
|
return message.body.decode('utf-8')
|
|
606
827
|
|
|
607
|
-
async def
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
828
|
+
async def _handle_business_retry(
|
|
829
|
+
self,
|
|
830
|
+
message: AbstractIncomingMessage,
|
|
831
|
+
error: Exception,
|
|
832
|
+
drop: bool = True
|
|
833
|
+
) -> None:
|
|
834
|
+
"""
|
|
835
|
+
封装业务失败重试逻辑:更新重试计数Header,延迟3秒重新发布
|
|
836
|
+
达到最大次数则标记失败(无锁,仅通过原子方法操作跟踪记录)
|
|
837
|
+
"""
|
|
838
|
+
# 获取当前重试次数
|
|
839
|
+
current_headers = message.headers or {}
|
|
840
|
+
retry_count = current_headers.get('x-retry-count', 0)
|
|
841
|
+
retry_count += 1
|
|
617
842
|
message_id = message.message_id or str(id(message))
|
|
618
|
-
if message_id in self._processing_message_ids:
|
|
619
|
-
logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
|
|
620
|
-
await message.ack()
|
|
621
|
-
return
|
|
622
|
-
|
|
623
|
-
self._processing_message_ids.add(message_id)
|
|
624
843
|
|
|
625
|
-
|
|
626
|
-
logger.debug(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
|
|
844
|
+
error_msg = f"[{type(error).__name__}] {str(error)}"[:200]
|
|
627
845
|
|
|
628
|
-
|
|
629
|
-
|
|
846
|
+
# 打印错误日志
|
|
847
|
+
logger.error(
|
|
848
|
+
f"消息 {message_id} 处理出错(第{retry_count}次重试): {error_msg}",
|
|
849
|
+
exc_info=True
|
|
850
|
+
)
|
|
630
851
|
|
|
852
|
+
# 达到最大重试次数:ack标记失败
|
|
853
|
+
if drop and retry_count >= MAX_RETRY_COUNT:
|
|
854
|
+
logger.error(
|
|
855
|
+
f"消息 {message_id} 已达到最大重试次数{MAX_RETRY_COUNT},标记为失败")
|
|
856
|
+
# 标记跟踪记录为已确认
|
|
857
|
+
await self._mark_tracking_acked(message_id)
|
|
631
858
|
await message.ack()
|
|
632
859
|
self._update_activity_timestamp()
|
|
633
|
-
|
|
634
|
-
logger.debug(f"消息 {message_id} 处理完成并确认")
|
|
860
|
+
return
|
|
635
861
|
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
862
|
+
# 构造新消息Header
|
|
863
|
+
new_headers = current_headers.copy()
|
|
864
|
+
new_headers['x-retry-count'] = retry_count
|
|
865
|
+
new_headers['x-retry-error'] = error_msg
|
|
640
866
|
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
867
|
+
# 提交异步任务,延迟3秒后重新发布
|
|
868
|
+
asyncio.create_task(
|
|
869
|
+
self._delayed_republish(
|
|
870
|
+
message, new_headers, retry_count, message_id)
|
|
871
|
+
)
|
|
645
872
|
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
873
|
+
async def _delayed_republish(
|
|
874
|
+
self,
|
|
875
|
+
message: AbstractIncomingMessage,
|
|
876
|
+
new_headers: Dict[str, Any],
|
|
877
|
+
retry_count: int,
|
|
878
|
+
message_id: str
|
|
879
|
+
) -> None:
|
|
880
|
+
"""延迟发布重试消息(无锁,仅通过原子方法操作资源)"""
|
|
881
|
+
try:
|
|
882
|
+
# 延迟3秒重试
|
|
883
|
+
await asyncio.sleep(3)
|
|
884
|
+
|
|
885
|
+
# 检查客户端状态
|
|
886
|
+
if await self._is_closed():
|
|
887
|
+
logger.warning(f"客户端已关闭,放弃消息 {message_id} 的重试发布")
|
|
651
888
|
return
|
|
652
889
|
|
|
653
|
-
|
|
654
|
-
|
|
890
|
+
# 获取交换机
|
|
891
|
+
_, exchange, _ = await self._get_connection_resources()
|
|
892
|
+
if not exchange:
|
|
893
|
+
raise Exception("交换机未初始化,无法发布重试消息")
|
|
655
894
|
|
|
895
|
+
# 构造新消息
|
|
656
896
|
new_message = Message(
|
|
657
897
|
body=message.body,
|
|
658
898
|
content_type=message.content_type,
|
|
@@ -660,20 +900,97 @@ class RabbitMQClient:
|
|
|
660
900
|
delivery_mode=message.delivery_mode
|
|
661
901
|
)
|
|
662
902
|
|
|
903
|
+
# 重新发布消息
|
|
904
|
+
await exchange.publish(
|
|
905
|
+
new_message,
|
|
906
|
+
routing_key=self.routing_key or '#',
|
|
907
|
+
mandatory=True,
|
|
908
|
+
timeout=5.0
|
|
909
|
+
)
|
|
910
|
+
self._update_activity_timestamp()
|
|
911
|
+
logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
|
|
912
|
+
|
|
913
|
+
# 拒绝原始消息(不重新入队)
|
|
663
914
|
await message.reject(requeue=False)
|
|
915
|
+
# 标记跟踪记录为已确认
|
|
916
|
+
await self._mark_tracking_acked(message_id)
|
|
917
|
+
|
|
918
|
+
except Exception as e:
|
|
919
|
+
logger.error(
|
|
920
|
+
f"消息 {message_id} 延迟发布失败(错误:{str(e)}),触发requeue兜底",
|
|
921
|
+
exc_info=True
|
|
922
|
+
)
|
|
923
|
+
# 发布失败兜底:requeue原始消息
|
|
924
|
+
await message.reject(requeue=True)
|
|
664
925
|
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
926
|
+
async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
|
|
927
|
+
"""消息处理包装器(无锁嵌套,仅通过原子方法操作状态)"""
|
|
928
|
+
message_id = message.message_id or str(id(message))
|
|
929
|
+
max_check_attempts = 3
|
|
930
|
+
check_interval = 1
|
|
931
|
+
|
|
932
|
+
# 重试检查消费状态(处理极端并发场景)
|
|
933
|
+
for attempt in range(max_check_attempts):
|
|
934
|
+
_, handler, consumer_tag = await self._get_consume_state()
|
|
935
|
+
if consumer_tag and handler: # 有消费标签且有处理器才继续
|
|
936
|
+
break
|
|
937
|
+
if attempt < max_check_attempts - 1:
|
|
938
|
+
logger.debug(
|
|
939
|
+
f"消息 {message_id} 处理状态检查重试(第{attempt+1}次): "
|
|
940
|
+
f"handler={'存在' if handler else '不存在'}, "
|
|
941
|
+
f"consumer_tag={'存在' if consumer_tag else '不存在'}"
|
|
671
942
|
)
|
|
672
|
-
|
|
673
|
-
|
|
943
|
+
await asyncio.sleep(check_interval)
|
|
944
|
+
|
|
945
|
+
# 最终状态判断:状态异常则拒绝消息
|
|
946
|
+
_, handler, consumer_tag = await self._get_consume_state()
|
|
947
|
+
if not consumer_tag or not handler:
|
|
948
|
+
err_msg = f"消息 {message_id} 拒绝处理:handler={'存在' if handler else '不存在'}, consumer_tag={'存在' if consumer_tag else '不存在'}"
|
|
949
|
+
logger.warning(err_msg)
|
|
950
|
+
try:
|
|
951
|
+
await self._handle_business_retry(message, Exception(err_msg), drop=False)
|
|
952
|
+
except Exception as e:
|
|
953
|
+
logger.error(f"消息 {message_id} 拒绝处理失败: {e}")
|
|
954
|
+
return
|
|
955
|
+
|
|
956
|
+
# 检查重复处理
|
|
957
|
+
if await self._check_duplicate_message(message_id):
|
|
958
|
+
logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
|
|
959
|
+
await message.ack()
|
|
960
|
+
return
|
|
961
|
+
|
|
962
|
+
# 添加跟踪记录
|
|
963
|
+
channel, _, _ = await self._get_connection_resources()
|
|
964
|
+
channel_number = channel.number if channel else None
|
|
965
|
+
await self._add_tracking_message(message_id, message.delivery_tag, channel_number)
|
|
966
|
+
|
|
967
|
+
try:
|
|
968
|
+
logger.info(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
|
|
969
|
+
|
|
970
|
+
# 解析消息
|
|
971
|
+
parsed_data = await self._parse_message(message)
|
|
972
|
+
# 转换为MQMsgModel
|
|
973
|
+
if isinstance(parsed_data, dict):
|
|
974
|
+
msg_model = MQMsgModel(**parsed_data)
|
|
975
|
+
else:
|
|
976
|
+
msg_model = MQMsgModel(data=parsed_data)
|
|
977
|
+
|
|
978
|
+
# 调用业务处理器
|
|
979
|
+
await handler(msg_model, message)
|
|
980
|
+
|
|
981
|
+
# 处理成功:标记跟踪记录并确认消息
|
|
982
|
+
await self._mark_tracking_acked(message_id)
|
|
983
|
+
await message.ack()
|
|
984
|
+
self._update_activity_timestamp()
|
|
985
|
+
self._update_message_processed_timestamp()
|
|
986
|
+
logger.info(f"消息 {message_id} 处理完成并确认")
|
|
987
|
+
|
|
988
|
+
except Exception as e:
|
|
989
|
+
# 业务处理失败:触发重试逻辑
|
|
990
|
+
await self._handle_business_retry(message, e)
|
|
674
991
|
finally:
|
|
675
|
-
|
|
676
|
-
|
|
992
|
+
# 清理跟踪记录
|
|
993
|
+
await self._remove_tracking_message(message_id)
|
|
677
994
|
|
|
678
995
|
async def __aenter__(self):
|
|
679
996
|
await self.connect()
|