sycommon-python-lib 0.1.38__py3-none-any.whl → 0.1.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/rabbitmq/rabbitmq_client.py +199 -899
- sycommon/rabbitmq/rabbitmq_pool.py +141 -65
- sycommon/rabbitmq/rabbitmq_service.py +19 -19
- {sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/RECORD +8 -8
- {sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/top_level.txt +0 -0
|
@@ -1,23 +1,19 @@
|
|
|
1
|
-
from aio_pika
|
|
2
|
-
from
|
|
3
|
-
from aio_pika import connect_robust, Channel
|
|
4
|
-
from typing import Optional, List
|
|
1
|
+
from aio_pika import Channel
|
|
2
|
+
from typing import Optional
|
|
5
3
|
import asyncio
|
|
6
4
|
import json
|
|
7
|
-
from typing import Callable, Coroutine,
|
|
5
|
+
from typing import Callable, Coroutine, Dict, Any, Union
|
|
8
6
|
from aio_pika import Message, DeliveryMode, ExchangeType
|
|
9
7
|
from aio_pika.abc import (
|
|
10
|
-
AbstractChannel,
|
|
11
8
|
AbstractExchange,
|
|
12
9
|
AbstractQueue,
|
|
13
10
|
AbstractIncomingMessage,
|
|
14
11
|
ConsumerTag,
|
|
15
12
|
)
|
|
16
|
-
from
|
|
17
|
-
|
|
13
|
+
from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
|
|
18
14
|
from sycommon.logging.kafka_log import SYLogger
|
|
19
15
|
from sycommon.models.mqmsg_model import MQMsgModel
|
|
20
|
-
|
|
16
|
+
|
|
21
17
|
|
|
22
18
|
# 最大重试次数限制
|
|
23
19
|
MAX_RETRY_COUNT = 3
|
|
@@ -27,9 +23,8 @@ logger = SYLogger
|
|
|
27
23
|
|
|
28
24
|
class RabbitMQClient:
|
|
29
25
|
"""
|
|
30
|
-
RabbitMQ
|
|
31
|
-
|
|
32
|
-
采用细粒度锁设计,彻底避免死锁隐患
|
|
26
|
+
RabbitMQ客户端(基于连接池),支持集群、自动重连、消息发布/消费
|
|
27
|
+
依赖 aio_pika 的内置重连机制,移除手动重连逻辑
|
|
33
28
|
"""
|
|
34
29
|
|
|
35
30
|
def __init__(
|
|
@@ -43,958 +38,263 @@ class RabbitMQClient:
|
|
|
43
38
|
auto_delete: bool = False,
|
|
44
39
|
auto_parse_json: bool = True,
|
|
45
40
|
create_if_not_exists: bool = True,
|
|
46
|
-
connection_timeout: int = 10,
|
|
47
41
|
rpc_timeout: int = 10,
|
|
48
|
-
reconnection_delay: int = 1,
|
|
49
|
-
max_reconnection_attempts: int = 5,
|
|
50
42
|
prefetch_count: int = 2,
|
|
51
|
-
consumption_stall_threshold: int =
|
|
43
|
+
consumption_stall_threshold: int = 60,
|
|
52
44
|
):
|
|
53
|
-
# 连接池依赖
|
|
54
45
|
self.connection_pool = connection_pool
|
|
46
|
+
if not self.connection_pool._initialized:
|
|
47
|
+
raise RuntimeError("连接池未初始化,请先调用 connection_pool.init_pools()")
|
|
55
48
|
|
|
56
|
-
#
|
|
49
|
+
# 交换机配置
|
|
57
50
|
self.exchange_name = exchange_name
|
|
58
|
-
self.exchange_type = ExchangeType(exchange_type)
|
|
51
|
+
self.exchange_type = ExchangeType(exchange_type.lower())
|
|
52
|
+
# 队列配置
|
|
59
53
|
self.queue_name = queue_name
|
|
60
54
|
self.routing_key = routing_key
|
|
61
55
|
self.durable = durable
|
|
62
56
|
self.auto_delete = auto_delete
|
|
63
|
-
|
|
64
|
-
# 行为控制参数
|
|
65
57
|
self.auto_parse_json = auto_parse_json
|
|
66
58
|
self.create_if_not_exists = create_if_not_exists
|
|
67
|
-
|
|
59
|
+
|
|
60
|
+
# 运行时配置
|
|
68
61
|
self.rpc_timeout = rpc_timeout
|
|
69
62
|
self.prefetch_count = prefetch_count
|
|
70
|
-
|
|
71
|
-
# 重连参数
|
|
72
|
-
self.reconnection_delay = reconnection_delay
|
|
73
|
-
self.max_reconnection_attempts = max_reconnection_attempts
|
|
74
|
-
|
|
75
|
-
# 消息处理参数
|
|
76
63
|
self.consumption_stall_threshold = consumption_stall_threshold
|
|
77
64
|
|
|
78
|
-
#
|
|
79
|
-
self.
|
|
80
|
-
self.
|
|
81
|
-
self.
|
|
82
|
-
|
|
83
|
-
# 状态跟踪(按类型拆分锁保护)
|
|
84
|
-
self.actual_queue_name: Optional[str] = None
|
|
85
|
-
self._exchange_exists = False # 由 _connection_lock 保护
|
|
86
|
-
self._queue_exists = False # 由 _connection_lock 保护
|
|
87
|
-
self._queue_bound = False # 由 _connection_lock 保护
|
|
88
|
-
self._closed = False # 由 _connection_lock 保护
|
|
89
|
-
# 由 _consume_state_lock 保护:通过 consumer_tag 存在性判断是否在消费
|
|
65
|
+
# 内部状态
|
|
66
|
+
self._channel: Optional[Channel] = None
|
|
67
|
+
self._exchange: Optional[AbstractExchange] = None
|
|
68
|
+
self._queue: Optional[AbstractQueue] = None
|
|
90
69
|
self._consumer_tag: Optional[ConsumerTag] = None
|
|
91
|
-
self.
|
|
92
|
-
self.
|
|
93
|
-
|
|
94
|
-
#
|
|
95
|
-
self.
|
|
96
|
-
|
|
97
|
-
Coroutine[Any, Any, None]
|
|
98
|
-
]] = None # 由 _consume_state_lock 保护
|
|
99
|
-
self._consuming_task: Optional[asyncio.Task] = None
|
|
100
|
-
self._reconnect_task: Optional[asyncio.Task] = None
|
|
101
|
-
self._keepalive_task: Optional[asyncio.Task] = None
|
|
102
|
-
self._monitor_task: Optional[asyncio.Task] = None
|
|
103
|
-
|
|
104
|
-
# 消息处理跟踪(由 _tracking_lock 保护)
|
|
105
|
-
self._tracking_messages: Dict[str, Dict[str, Any]] = {}
|
|
106
|
-
|
|
107
|
-
# 细粒度锁(核心设计:按资源类型拆分,避免嵌套)
|
|
108
|
-
# 保护消费状态(message_handler、_consumer_tag)
|
|
109
|
-
self._consume_state_lock = asyncio.Lock()
|
|
110
|
-
self._tracking_lock = asyncio.Lock() # 保护消息跟踪记录(_tracking_messages)
|
|
111
|
-
# 保护连接/资源状态(channel、exchange、queue、_closed等)
|
|
112
|
-
self._connection_lock = asyncio.Lock()
|
|
70
|
+
self._message_handler: Optional[Callable] = None
|
|
71
|
+
self._closed = False
|
|
72
|
+
|
|
73
|
+
# 细粒度锁
|
|
74
|
+
self._consume_lock = asyncio.Lock()
|
|
75
|
+
self._connect_lock = asyncio.Lock()
|
|
113
76
|
|
|
114
77
|
@property
|
|
115
78
|
async def is_connected(self) -> bool:
|
|
116
|
-
"""
|
|
117
|
-
|
|
118
|
-
return (not self._closed and
|
|
119
|
-
self.channel is not None and
|
|
120
|
-
not self.channel.is_closed and
|
|
121
|
-
self.exchange is not None)
|
|
122
|
-
|
|
123
|
-
def _update_activity_timestamp(self) -> None:
|
|
124
|
-
"""更新最后活动时间戳(非共享状态,无需锁)"""
|
|
125
|
-
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
126
|
-
|
|
127
|
-
def _update_message_processed_timestamp(self) -> None:
|
|
128
|
-
"""更新最后消息处理时间戳(非共享状态,无需锁)"""
|
|
129
|
-
self._last_message_processed = asyncio.get_event_loop().time()
|
|
130
|
-
|
|
131
|
-
# ------------------------------
|
|
132
|
-
# 消费状态操作(_consume_state_lock 专属保护)
|
|
133
|
-
# ------------------------------
|
|
134
|
-
async def _get_consume_state(self) -> tuple[bool, Optional[Callable], Optional[ConsumerTag]]:
|
|
135
|
-
"""安全获取消费相关状态(一次性获取,避免多次加锁)"""
|
|
136
|
-
async with self._consume_state_lock:
|
|
137
|
-
# 通过 _consumer_tag 是否存在判断是否在消费
|
|
138
|
-
is_consuming = self._consumer_tag is not None
|
|
139
|
-
return is_consuming, self.message_handler, self._consumer_tag
|
|
140
|
-
|
|
141
|
-
async def _set_consumer_tag(self, consumer_tag: Optional[ConsumerTag] = None):
|
|
142
|
-
"""安全更新消费者标签(替代原 _set_consume_state)"""
|
|
143
|
-
async with self._consume_state_lock:
|
|
144
|
-
old_tag = self._consumer_tag
|
|
145
|
-
self._consumer_tag = consumer_tag
|
|
146
|
-
old_is_consuming = old_tag is not None
|
|
147
|
-
new_is_consuming = consumer_tag is not None
|
|
148
|
-
if old_is_consuming != new_is_consuming:
|
|
149
|
-
logger.info(f"消费状态变更: {old_is_consuming} → {new_is_consuming}")
|
|
150
|
-
|
|
151
|
-
async def set_message_handler(self, handler):
|
|
152
|
-
"""设置消息处理器(加锁保护,避免并发修改)"""
|
|
153
|
-
async with self._consume_state_lock:
|
|
154
|
-
self.message_handler = handler
|
|
155
|
-
logger.info("消息处理器已设置")
|
|
156
|
-
|
|
157
|
-
# ------------------------------
|
|
158
|
-
# 连接状态操作(_connection_lock 专属保护)
|
|
159
|
-
# ------------------------------
|
|
160
|
-
async def _is_closed(self) -> bool:
|
|
161
|
-
"""检查客户端是否已关闭(线程安全)"""
|
|
162
|
-
async with self._connection_lock:
|
|
163
|
-
return self._closed
|
|
164
|
-
|
|
165
|
-
async def _mark_closed(self):
|
|
166
|
-
"""标记客户端已关闭(原子操作)"""
|
|
167
|
-
async with self._connection_lock:
|
|
168
|
-
self._closed = True
|
|
169
|
-
|
|
170
|
-
async def _get_connection_resources(self) -> tuple[Optional[AbstractChannel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
171
|
-
"""安全获取连接资源(channel/exchange/queue)"""
|
|
172
|
-
async with self._connection_lock:
|
|
173
|
-
return self.channel, self.exchange, self.queue
|
|
174
|
-
|
|
175
|
-
async def _reset_connection_state(self):
|
|
176
|
-
"""重置连接状态(用于重连时,原子操作)"""
|
|
177
|
-
async with self._connection_lock:
|
|
178
|
-
self._exchange_exists = False
|
|
179
|
-
self._queue_exists = False
|
|
180
|
-
self._queue_bound = False
|
|
181
|
-
self.channel = None
|
|
182
|
-
self.exchange = None
|
|
183
|
-
self.queue = None
|
|
184
|
-
self.actual_queue_name = None
|
|
185
|
-
|
|
186
|
-
async def _update_connection_resources(self, channel: AbstractChannel, exchange: AbstractExchange, queue: Optional[AbstractQueue] = None):
|
|
187
|
-
"""更新连接资源(原子操作)"""
|
|
188
|
-
async with self._connection_lock:
|
|
189
|
-
self.channel = channel
|
|
190
|
-
self.exchange = exchange
|
|
191
|
-
self.queue = queue
|
|
192
|
-
if queue:
|
|
193
|
-
self.actual_queue_name = queue.name
|
|
194
|
-
|
|
195
|
-
# ------------------------------
|
|
196
|
-
# 消息跟踪操作(_tracking_lock 专属保护)
|
|
197
|
-
# ------------------------------
|
|
198
|
-
async def _add_tracking_message(self, msg_id: str, delivery_tag: int, channel_number: Optional[int]):
|
|
199
|
-
"""添加消息跟踪记录(原子操作)"""
|
|
200
|
-
async with self._tracking_lock:
|
|
201
|
-
self._tracking_messages[msg_id] = {
|
|
202
|
-
'delivery_tag': delivery_tag,
|
|
203
|
-
'acked': False,
|
|
204
|
-
'channel_number': channel_number,
|
|
205
|
-
'start_time': asyncio.get_event_loop().time()
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
async def _mark_tracking_acked(self, msg_id: str):
|
|
209
|
-
"""标记消息已确认(原子操作)"""
|
|
210
|
-
async with self._tracking_lock:
|
|
211
|
-
if msg_id in self._tracking_messages:
|
|
212
|
-
self._tracking_messages[msg_id]['acked'] = True
|
|
213
|
-
|
|
214
|
-
async def _remove_tracking_message(self, msg_id: str):
|
|
215
|
-
"""删除消息跟踪记录(原子操作,避免KeyError)"""
|
|
216
|
-
async with self._tracking_lock:
|
|
217
|
-
if msg_id in self._tracking_messages:
|
|
218
|
-
del self._tracking_messages[msg_id]
|
|
219
|
-
logger.info(f"已删除消息跟踪信息: {msg_id}")
|
|
220
|
-
|
|
221
|
-
async def _check_duplicate_message(self, msg_id: str) -> bool:
|
|
222
|
-
"""检查消息是否重复处理(原子操作)"""
|
|
223
|
-
async with self._tracking_lock:
|
|
224
|
-
return msg_id in self._tracking_messages
|
|
225
|
-
|
|
226
|
-
async def _get_tracking_count(self) -> int:
|
|
227
|
-
"""获取当前跟踪的消息数(原子操作)"""
|
|
228
|
-
async with self._tracking_lock:
|
|
229
|
-
return len(self._tracking_messages)
|
|
230
|
-
|
|
231
|
-
async def _cleanup_acked_tracking_messages(self) -> int:
|
|
232
|
-
"""清理已确认的跟踪记录(原子操作,返回清理数量)"""
|
|
233
|
-
async with self._tracking_lock:
|
|
234
|
-
acked_ids = [
|
|
235
|
-
msg_id for msg_id, info in self._tracking_messages.items() if info.get('acked')]
|
|
236
|
-
for msg_id in acked_ids:
|
|
237
|
-
del self._tracking_messages[msg_id]
|
|
238
|
-
return len(acked_ids)
|
|
239
|
-
|
|
240
|
-
async def _clear_tracking_messages(self):
|
|
241
|
-
"""清空所有跟踪记录(原子操作)"""
|
|
242
|
-
async with self._tracking_lock:
|
|
243
|
-
self._tracking_messages.clear()
|
|
244
|
-
|
|
245
|
-
# ------------------------------
|
|
246
|
-
# 基础工具方法
|
|
247
|
-
# ------------------------------
|
|
248
|
-
async def _get_channel(self) -> AbstractChannel:
|
|
249
|
-
"""从通道池获取通道(使用上下文管理器,自动归还)"""
|
|
250
|
-
if not self.connection_pool.channel_pool:
|
|
251
|
-
raise Exception("连接池未初始化,请先调用init_pools")
|
|
252
|
-
|
|
253
|
-
async with self.connection_pool.channel_pool.acquire() as channel:
|
|
254
|
-
return channel
|
|
255
|
-
|
|
256
|
-
async def _check_exchange_exists(self, channel: AbstractChannel) -> bool:
|
|
257
|
-
"""检查交换机是否存在"""
|
|
258
|
-
try:
|
|
259
|
-
await asyncio.wait_for(
|
|
260
|
-
channel.declare_exchange(
|
|
261
|
-
name=self.exchange_name,
|
|
262
|
-
type=self.exchange_type,
|
|
263
|
-
passive=True
|
|
264
|
-
),
|
|
265
|
-
timeout=self.rpc_timeout
|
|
266
|
-
)
|
|
267
|
-
return True
|
|
268
|
-
except Exception:
|
|
269
|
-
return False
|
|
270
|
-
|
|
271
|
-
async def _check_queue_exists(self, channel: AbstractChannel) -> bool:
|
|
272
|
-
"""检查队列是否存在"""
|
|
273
|
-
if not self.queue_name:
|
|
79
|
+
"""异步检查客户端连接状态(属性,不是函数)"""
|
|
80
|
+
if self._closed:
|
|
274
81
|
return False
|
|
275
82
|
try:
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
timeout=self.rpc_timeout
|
|
282
|
-
)
|
|
283
|
-
return True
|
|
284
|
-
except Exception:
|
|
285
|
-
return False
|
|
286
|
-
|
|
287
|
-
async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
|
|
288
|
-
"""将队列绑定到交换机(带重试)"""
|
|
289
|
-
bind_routing_key = self.routing_key if self.routing_key else '#'
|
|
290
|
-
|
|
291
|
-
for attempt in range(MAX_RETRY_COUNT + 1):
|
|
292
|
-
try:
|
|
293
|
-
await asyncio.wait_for(
|
|
294
|
-
queue.bind(
|
|
295
|
-
exchange,
|
|
296
|
-
routing_key=bind_routing_key
|
|
297
|
-
),
|
|
298
|
-
timeout=self.rpc_timeout
|
|
299
|
-
)
|
|
300
|
-
logger.info(
|
|
301
|
-
f"队列 '{queue.name}' 已绑定到交换机 '{exchange.name}',路由键: {bind_routing_key}")
|
|
83
|
+
# 检查通道是否有效
|
|
84
|
+
if self._channel and not self._channel.is_closed:
|
|
85
|
+
# 检查交换机和队列(如果需要)
|
|
86
|
+
if self.create_if_not_exists and self.queue_name:
|
|
87
|
+
return bool(self._exchange and self._queue)
|
|
302
88
|
return True
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
await asyncio.sleep(1)
|
|
308
|
-
return False
|
|
309
|
-
|
|
310
|
-
# ------------------------------
|
|
311
|
-
# 核心业务方法
|
|
312
|
-
# ------------------------------
|
|
313
|
-
async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
|
|
314
|
-
"""从连接池获取资源并初始化(交换机、队列)"""
|
|
315
|
-
logger.info(
|
|
316
|
-
f"连接参数 - force_reconnect={force_reconnect}, "
|
|
317
|
-
f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
|
|
318
|
-
)
|
|
89
|
+
return False
|
|
90
|
+
except Exception as e:
|
|
91
|
+
logger.warning(f"检查连接状态失败: {str(e)}")
|
|
92
|
+
return False
|
|
319
93
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
94
|
+
async def _get_connection_resources(self) -> tuple[Optional[Channel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
95
|
+
"""原子获取连接资源(通道、交换机、队列)"""
|
|
96
|
+
async with self._connect_lock:
|
|
97
|
+
return self._channel, self._exchange, self._queue
|
|
323
98
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
99
|
+
async def connect(self) -> None:
|
|
100
|
+
"""建立连接并初始化交换机/队列(使用新的通道获取方式)"""
|
|
101
|
+
if self._closed:
|
|
102
|
+
raise RuntimeError("客户端已关闭,无法重新连接")
|
|
328
103
|
|
|
329
|
-
|
|
330
|
-
if self._reconnect_task and not self._reconnect_task.done():
|
|
331
|
-
self._reconnect_task.cancel()
|
|
104
|
+
async with self._connect_lock:
|
|
332
105
|
try:
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
106
|
+
# 关键修复:使用连接池的 acquire_channel 方法获取通道
|
|
107
|
+
self._channel = await self.connection_pool.acquire_channel()
|
|
108
|
+
# 设置预取计数
|
|
109
|
+
await self._channel.set_qos(prefetch_count=self.prefetch_count)
|
|
336
110
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
while retries < self.max_reconnection_attempts:
|
|
346
|
-
try:
|
|
347
|
-
# 获取新通道
|
|
348
|
-
channel = await self._get_channel()
|
|
349
|
-
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
350
|
-
|
|
351
|
-
# 处理交换机
|
|
352
|
-
exchange_exists = await self._check_exchange_exists(channel)
|
|
353
|
-
if not exchange_exists:
|
|
354
|
-
if self.create_if_not_exists:
|
|
355
|
-
exchange = await asyncio.wait_for(
|
|
356
|
-
channel.declare_exchange(
|
|
357
|
-
name=self.exchange_name,
|
|
358
|
-
type=self.exchange_type,
|
|
359
|
-
durable=self.durable,
|
|
360
|
-
auto_delete=self.auto_delete
|
|
361
|
-
),
|
|
362
|
-
timeout=self.rpc_timeout
|
|
363
|
-
)
|
|
364
|
-
logger.info(f"已创建交换机 '{self.exchange_name}'")
|
|
365
|
-
else:
|
|
366
|
-
raise Exception(
|
|
367
|
-
f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
|
|
368
|
-
else:
|
|
369
|
-
exchange = await channel.get_exchange(self.exchange_name)
|
|
370
|
-
logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
|
|
371
|
-
|
|
372
|
-
# 处理队列
|
|
373
|
-
queue = None
|
|
374
|
-
if declare_queue and self.queue_name:
|
|
375
|
-
queue_exists = await self._check_queue_exists(channel)
|
|
376
|
-
|
|
377
|
-
if not queue_exists:
|
|
378
|
-
if not self.create_if_not_exists:
|
|
379
|
-
raise Exception(
|
|
380
|
-
f"队列 '{self.queue_name}' 不存在且不允许自动创建")
|
|
381
|
-
|
|
382
|
-
queue = await asyncio.wait_for(
|
|
383
|
-
channel.declare_queue(
|
|
384
|
-
name=self.queue_name,
|
|
385
|
-
durable=self.durable,
|
|
386
|
-
auto_delete=self.auto_delete,
|
|
387
|
-
exclusive=False
|
|
388
|
-
),
|
|
389
|
-
timeout=self.rpc_timeout
|
|
390
|
-
)
|
|
391
|
-
logger.info(f"已创建队列 '{self.queue_name}'")
|
|
392
|
-
else:
|
|
393
|
-
queue = await channel.get_queue(self.queue_name)
|
|
394
|
-
logger.info(f"使用已存在的队列 '{self.queue_name}'")
|
|
111
|
+
# 声明交换机
|
|
112
|
+
self._exchange = await self._channel.declare_exchange(
|
|
113
|
+
name=self.exchange_name,
|
|
114
|
+
type=self.exchange_type,
|
|
115
|
+
durable=self.durable,
|
|
116
|
+
auto_delete=self.auto_delete,
|
|
117
|
+
passive=not self.create_if_not_exists # 不创建时使用passive模式检查
|
|
118
|
+
)
|
|
395
119
|
|
|
120
|
+
# 声明队列(如果配置了队列名)
|
|
121
|
+
if self.queue_name:
|
|
122
|
+
self._queue = await self._channel.declare_queue(
|
|
123
|
+
name=self.queue_name,
|
|
124
|
+
durable=self.durable,
|
|
125
|
+
auto_delete=self.auto_delete,
|
|
126
|
+
passive=not self.create_if_not_exists
|
|
127
|
+
)
|
|
396
128
|
# 绑定队列到交换机
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
# 重新开始消费(如果已设置处理器且之前在消费)
|
|
410
|
-
_, handler, consumer_tag = await self._get_consume_state()
|
|
411
|
-
if handler and consumer_tag: # 有处理器且之前有消费标签,说明需要恢复消费
|
|
412
|
-
await self.start_consuming()
|
|
413
|
-
|
|
414
|
-
# 启动监控和保活任务
|
|
415
|
-
self._start_monitoring()
|
|
416
|
-
self._start_keepalive()
|
|
417
|
-
|
|
418
|
-
self._update_activity_timestamp()
|
|
419
|
-
|
|
420
|
-
# 首次启动时延迟1秒(避免初始化未完成就接收消息)
|
|
421
|
-
if not force_reconnect:
|
|
422
|
-
logger.info("客户端初始化成功,延迟1秒接收消息(解决启动时序问题)")
|
|
423
|
-
await asyncio.sleep(1)
|
|
424
|
-
|
|
425
|
-
logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
|
|
426
|
-
return
|
|
129
|
+
await self._queue.bind(
|
|
130
|
+
exchange=self._exchange,
|
|
131
|
+
routing_key=self.routing_key or self.queue_name
|
|
132
|
+
)
|
|
133
|
+
logger.info(
|
|
134
|
+
f"队列 '{self.queue_name}' 已声明并绑定到交换机 '{self.exchange_name}' "
|
|
135
|
+
f"(exchange_type: {self.exchange_type}, routing_key: {self.routing_key})"
|
|
136
|
+
)
|
|
137
|
+
else:
|
|
138
|
+
logger.info(
|
|
139
|
+
f"未配置队列名,仅初始化交换机 '{self.exchange_name}' (exchange_type: {self.exchange_type})")
|
|
427
140
|
|
|
141
|
+
logger.info(f"RabbitMQ客户端连接成功(exchange: {self.exchange_name})")
|
|
428
142
|
except Exception as e:
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
async def monitor():
|
|
445
|
-
while not await self._is_closed():
|
|
446
|
-
try:
|
|
447
|
-
# 检查通道状态
|
|
448
|
-
channel, _, _ = await self._get_connection_resources()
|
|
449
|
-
if channel and channel.is_closed:
|
|
450
|
-
logger.warning("检测到通道已关闭,尝试重建")
|
|
451
|
-
await self._recreate_channel()
|
|
452
|
-
continue
|
|
453
|
-
|
|
454
|
-
current_time = asyncio.get_event_loop().time()
|
|
455
|
-
|
|
456
|
-
# 清理已确认的跟踪记录
|
|
457
|
-
cleaned_count = await self._cleanup_acked_tracking_messages()
|
|
458
|
-
if cleaned_count > 0:
|
|
459
|
-
logger.info(f"清理了 {cleaned_count} 条已确认消息记录")
|
|
460
|
-
|
|
461
|
-
# 检查消费停滞(仅当有消费者标签时)
|
|
462
|
-
_, handler, consumer_tag = await self._get_consume_state()
|
|
463
|
-
if consumer_tag: # 有消费标签说明正在消费
|
|
464
|
-
tracking_count = await self._get_tracking_count()
|
|
465
|
-
if current_time - self._last_message_processed > self.consumption_stall_threshold:
|
|
466
|
-
if tracking_count > 0:
|
|
467
|
-
logger.warning(
|
|
468
|
-
f"消费停滞,但有 {tracking_count} 个消息正在处理,暂不重启")
|
|
469
|
-
else:
|
|
470
|
-
logger.info("消费停滞且无消息处理,重启消费")
|
|
471
|
-
try:
|
|
472
|
-
await self.stop_consuming()
|
|
473
|
-
await asyncio.sleep(1)
|
|
474
|
-
# 检查处理器是否存在
|
|
475
|
-
_, handler, _ = await self._get_consume_state()
|
|
476
|
-
if handler:
|
|
477
|
-
await self.start_consuming()
|
|
478
|
-
else:
|
|
479
|
-
logger.error("消费处理器已丢失,无法重启消费")
|
|
480
|
-
except Exception as e:
|
|
481
|
-
logger.error(
|
|
482
|
-
f"重启消费失败: {str(e)}", exc_info=True)
|
|
483
|
-
await self._set_consumer_tag(None)
|
|
143
|
+
logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
|
|
144
|
+
# 清理异常状态
|
|
145
|
+
if self._channel:
|
|
146
|
+
await self.connection_pool.release_channel(self._channel)
|
|
147
|
+
self._channel = None
|
|
148
|
+
self._exchange = None
|
|
149
|
+
self._queue = None
|
|
150
|
+
raise
|
|
151
|
+
|
|
152
|
+
async def set_message_handler(self, handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]):
|
|
153
|
+
"""设置消息处理器(消费消息时使用)"""
|
|
154
|
+
async with self._consume_lock:
|
|
155
|
+
self._message_handler = handler
|
|
156
|
+
logger.info("消息处理器已设置")
|
|
484
157
|
|
|
485
|
-
|
|
486
|
-
|
|
158
|
+
async def start_consuming(self) -> Optional[ConsumerTag]:
|
|
159
|
+
"""启动消费(返回消费者标签)"""
|
|
160
|
+
if self._closed:
|
|
161
|
+
logger.warning("客户端已关闭,无法启动消费")
|
|
162
|
+
return None
|
|
487
163
|
|
|
488
|
-
|
|
164
|
+
async with self._consume_lock:
|
|
165
|
+
# 检查前置条件
|
|
166
|
+
if not self._message_handler:
|
|
167
|
+
raise RuntimeError("未设置消息处理器,请先调用 set_message_handler")
|
|
489
168
|
|
|
490
|
-
|
|
491
|
-
|
|
169
|
+
# 确保连接已建立
|
|
170
|
+
if not await self.is_connected:
|
|
171
|
+
await self.connect()
|
|
492
172
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
logger.info("开始重建通道...")
|
|
173
|
+
# 确保队列已初始化
|
|
174
|
+
_, _, queue = await self._get_connection_resources()
|
|
175
|
+
if not queue:
|
|
176
|
+
raise RuntimeError("队列未初始化,无法启动消费")
|
|
498
177
|
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
channel = await self._get_channel()
|
|
502
|
-
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
503
|
-
|
|
504
|
-
# 重新获取交换机
|
|
505
|
-
exchange = await channel.get_exchange(self.exchange_name)
|
|
506
|
-
|
|
507
|
-
# 重新获取队列并绑定
|
|
508
|
-
queue = None
|
|
509
|
-
if self.queue_name:
|
|
510
|
-
queue = await channel.get_queue(self.queue_name)
|
|
511
|
-
if queue and exchange:
|
|
512
|
-
bound = await self._bind_queue(channel, queue, exchange)
|
|
513
|
-
if not bound:
|
|
514
|
-
raise Exception("队列绑定失败,通道重建不完整")
|
|
515
|
-
|
|
516
|
-
# 更新连接资源
|
|
517
|
-
await self._update_connection_resources(channel, exchange, queue)
|
|
518
|
-
|
|
519
|
-
# 重新开始消费(如果有处理器)
|
|
520
|
-
_, handler, _ = await self._get_consume_state()
|
|
521
|
-
if handler:
|
|
522
|
-
await self.start_consuming()
|
|
523
|
-
|
|
524
|
-
# 清空跟踪记录
|
|
525
|
-
await self._clear_tracking_messages()
|
|
526
|
-
logger.info("通道已重建并恢复服务")
|
|
527
|
-
self._update_activity_timestamp()
|
|
528
|
-
except Exception as e:
|
|
529
|
-
logger.error(f"通道重建失败: {str(e)},触发重连", exc_info=True)
|
|
530
|
-
await self._set_consumer_tag(None)
|
|
531
|
-
await self.connect(force_reconnect=True)
|
|
532
|
-
|
|
533
|
-
def _start_keepalive(self) -> None:
|
|
534
|
-
"""启动连接保活任务(无锁,仅通过原子方法访问状态)"""
|
|
535
|
-
if self._keepalive_task and not self._keepalive_task.done():
|
|
536
|
-
return
|
|
537
|
-
|
|
538
|
-
async def keepalive():
|
|
539
|
-
while not await self._is_closed():
|
|
178
|
+
# 定义消费回调
|
|
179
|
+
async def consume_callback(message: AbstractIncomingMessage):
|
|
540
180
|
try:
|
|
541
|
-
#
|
|
542
|
-
if
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
current_time = asyncio.get_event_loop().time()
|
|
549
|
-
# 检查活动时间
|
|
550
|
-
if current_time - self._last_activity_timestamp > self.connection_pool.heartbeat * 2:
|
|
551
|
-
logger.info(
|
|
552
|
-
f"连接 {self.connection_pool.heartbeat*2}s 无活动,执行保活检查")
|
|
553
|
-
channel, exchange, _ = await self._get_connection_resources()
|
|
554
|
-
if channel and not channel.is_closed and exchange:
|
|
555
|
-
# 轻量级操作:检查交换机是否存在
|
|
556
|
-
await asyncio.wait_for(
|
|
557
|
-
channel.declare_exchange(
|
|
558
|
-
name=self.exchange_name,
|
|
559
|
-
type=self.exchange_type,
|
|
560
|
-
passive=True
|
|
561
|
-
),
|
|
562
|
-
timeout=5
|
|
563
|
-
)
|
|
564
|
-
self._update_activity_timestamp()
|
|
565
|
-
logger.info("保活检查成功")
|
|
566
|
-
else:
|
|
567
|
-
raise Exception("连接资源无效")
|
|
181
|
+
# 解析消息
|
|
182
|
+
if self.auto_parse_json:
|
|
183
|
+
body = json.loads(message.body.decode('utf-8'))
|
|
184
|
+
parsed_data = MQMsgModel(**body)
|
|
185
|
+
else:
|
|
186
|
+
parsed_data = MQMsgModel(
|
|
187
|
+
msg=message.body.decode('utf-8'))
|
|
568
188
|
|
|
189
|
+
# 调用处理器
|
|
190
|
+
await self._message_handler(parsed_data, message)
|
|
191
|
+
# 手动确认消息
|
|
192
|
+
await message.ack()
|
|
569
193
|
except Exception as e:
|
|
570
|
-
logger.
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
"""安排重新连接(无锁)"""
|
|
580
|
-
if self._reconnect_task and not self._reconnect_task.done():
|
|
581
|
-
return
|
|
582
|
-
|
|
583
|
-
logger.info(f"将在 {self.reconnection_delay} 秒后尝试重新连接...")
|
|
584
|
-
|
|
585
|
-
async def reconnect():
|
|
586
|
-
try:
|
|
587
|
-
await asyncio.sleep(self.reconnection_delay)
|
|
588
|
-
if not await self._is_closed():
|
|
589
|
-
await self.connect(force_reconnect=True)
|
|
590
|
-
except Exception as e:
|
|
591
|
-
logger.error(f"重连任务失败: {str(e)}")
|
|
592
|
-
if not await self._is_closed():
|
|
593
|
-
await self._schedule_reconnect()
|
|
594
|
-
|
|
595
|
-
self._reconnect_task = asyncio.create_task(reconnect())
|
|
596
|
-
|
|
597
|
-
async def close(self) -> None:
|
|
598
|
-
"""关闭客户端并释放资源(原子操作,无锁嵌套)"""
|
|
599
|
-
if await self._is_closed():
|
|
600
|
-
logger.info("客户端已关闭,无需重复操作")
|
|
601
|
-
return
|
|
602
|
-
|
|
603
|
-
logger.info("开始关闭RabbitMQ客户端...")
|
|
604
|
-
|
|
605
|
-
# 标记为已关闭
|
|
606
|
-
await self._mark_closed()
|
|
607
|
-
|
|
608
|
-
# 停止消费
|
|
609
|
-
await self.stop_consuming()
|
|
610
|
-
|
|
611
|
-
# 取消所有后台任务
|
|
612
|
-
tasks = [self._keepalive_task,
|
|
613
|
-
self._reconnect_task, self._monitor_task]
|
|
614
|
-
for task in tasks:
|
|
615
|
-
if task and not task.done():
|
|
616
|
-
task.cancel()
|
|
617
|
-
try:
|
|
618
|
-
await task
|
|
619
|
-
except asyncio.CancelledError:
|
|
620
|
-
logger.info(f"任务 {task.get_name()} 已取消")
|
|
194
|
+
logger.error(
|
|
195
|
+
f"处理消息失败 (delivery_tag: {message.delivery_tag}): {str(e)}", exc_info=True)
|
|
196
|
+
# 消费失败时重新入队(最多重试3次)
|
|
197
|
+
if message.redelivered:
|
|
198
|
+
logger.warning(
|
|
199
|
+
f"消息已重试过,拒绝入队 (delivery_tag: {message.delivery_tag})")
|
|
200
|
+
await message.reject(requeue=False)
|
|
201
|
+
else:
|
|
202
|
+
await message.nack(requeue=True)
|
|
621
203
|
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
self.
|
|
627
|
-
self._consumer_tag = None
|
|
204
|
+
# 启动消费
|
|
205
|
+
self._consumer_tag = await queue.consume(consume_callback)
|
|
206
|
+
logger.info(
|
|
207
|
+
f"开始消费队列 '{queue.name}',consumer_tag: {self._consumer_tag}")
|
|
208
|
+
return self._consumer_tag
|
|
628
209
|
|
|
629
|
-
|
|
210
|
+
async def stop_consuming(self) -> None:
|
|
211
|
+
"""停止消费"""
|
|
212
|
+
async with self._consume_lock:
|
|
213
|
+
if self._consumer_tag and not self._closed:
|
|
214
|
+
_, _, queue = await self._get_connection_resources()
|
|
215
|
+
if queue and not queue.is_closed:
|
|
216
|
+
try:
|
|
217
|
+
await queue.cancel(self._consumer_tag)
|
|
218
|
+
logger.info(f"停止消费,consumer_tag: {self._consumer_tag}")
|
|
219
|
+
except Exception as e:
|
|
220
|
+
logger.error(f"停止消费失败: {str(e)}")
|
|
221
|
+
self._consumer_tag = None
|
|
630
222
|
|
|
631
223
|
async def publish(
|
|
632
224
|
self,
|
|
633
|
-
message_body: Union[str, Dict[str, Any]],
|
|
634
|
-
routing_key: Optional[str] = None,
|
|
635
|
-
content_type: str = "application/json",
|
|
225
|
+
message_body: Union[str, Dict[str, Any], MQMsgModel],
|
|
636
226
|
headers: Optional[Dict[str, Any]] = None,
|
|
227
|
+
content_type: str = "application/json",
|
|
637
228
|
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
|
|
638
229
|
) -> None:
|
|
639
|
-
"""
|
|
640
|
-
if
|
|
641
|
-
raise
|
|
230
|
+
"""发布消息(修复 is_closed 问题)"""
|
|
231
|
+
if self._closed:
|
|
232
|
+
raise RuntimeError("客户端已关闭,无法发布消息")
|
|
642
233
|
|
|
643
|
-
#
|
|
234
|
+
# 确保连接已建立
|
|
644
235
|
if not await self.is_connected:
|
|
645
|
-
|
|
646
|
-
await self.connect(force_reconnect=True)
|
|
236
|
+
await self.connect()
|
|
647
237
|
|
|
648
238
|
# 处理消息体
|
|
649
|
-
if isinstance(message_body,
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
239
|
+
if isinstance(message_body, MQMsgModel):
|
|
240
|
+
body = json.dumps(message_body.__dict__,
|
|
241
|
+
ensure_ascii=False).encode('utf-8')
|
|
242
|
+
elif isinstance(message_body, dict):
|
|
243
|
+
body = json.dumps(message_body, ensure_ascii=False).encode('utf-8')
|
|
653
244
|
else:
|
|
654
|
-
|
|
245
|
+
body = str(message_body).encode('utf-8')
|
|
655
246
|
|
|
656
|
-
#
|
|
247
|
+
# 创建消息
|
|
657
248
|
message = Message(
|
|
658
|
-
body=
|
|
659
|
-
content_type=content_type,
|
|
249
|
+
body=body,
|
|
660
250
|
headers=headers or {},
|
|
251
|
+
content_type=content_type,
|
|
661
252
|
delivery_mode=delivery_mode
|
|
662
253
|
)
|
|
663
254
|
|
|
664
|
-
#
|
|
665
|
-
retry_count = 0
|
|
666
|
-
max_retries = 2
|
|
667
|
-
while retry_count < max_retries:
|
|
668
|
-
try:
|
|
669
|
-
async with self.connection_pool.channel_pool.acquire() as publish_channel:
|
|
670
|
-
exchange = await publish_channel.get_exchange(self.exchange_name)
|
|
671
|
-
confirmed = await exchange.publish(
|
|
672
|
-
message,
|
|
673
|
-
routing_key=routing_key or self.routing_key or '#',
|
|
674
|
-
mandatory=True,
|
|
675
|
-
timeout=5
|
|
676
|
-
)
|
|
677
|
-
if not confirmed:
|
|
678
|
-
raise Exception("消息未被服务器确认接收")
|
|
679
|
-
|
|
680
|
-
self._update_activity_timestamp()
|
|
681
|
-
logger.info(
|
|
682
|
-
f"消息已发布到交换机 '{self.exchange_name}'(路由键: {routing_key or self.routing_key or '#'})")
|
|
683
|
-
return
|
|
684
|
-
except (ConnectionClosed, ChannelInvalidStateError, asyncio.TimeoutError):
|
|
685
|
-
retry_count += 1
|
|
686
|
-
logger.warning(f"连接异常,尝试重连后重新发布 (重试次数: {retry_count})")
|
|
687
|
-
await self.connect(force_reconnect=True)
|
|
688
|
-
except Exception as e:
|
|
689
|
-
retry_count += 1
|
|
690
|
-
logger.error(f"消息发布失败 (重试次数: {retry_count}): {str(e)}")
|
|
691
|
-
if retry_count < max_retries:
|
|
692
|
-
await asyncio.sleep(1)
|
|
693
|
-
|
|
694
|
-
raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
|
|
695
|
-
|
|
696
|
-
async def _safe_cancel_consumer(self, consumer_tag: ConsumerTag, queue: AbstractQueue) -> bool:
|
|
697
|
-
"""安全取消消费者(无锁,仅操作传入的局部变量)"""
|
|
255
|
+
# 发布消息
|
|
698
256
|
try:
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
# 检查连接状态(确保通道完全就绪)
|
|
716
|
-
if not await self.is_connected:
|
|
717
|
-
await self.connect()
|
|
718
|
-
|
|
719
|
-
# 确保通道未关闭(解决启动时通道初始化滞后问题)
|
|
720
|
-
channel, _, _ = await self._get_connection_resources()
|
|
721
|
-
max_wait_attempts = 5
|
|
722
|
-
wait_interval = 0.5
|
|
723
|
-
for attempt in range(max_wait_attempts):
|
|
724
|
-
if channel and not channel.is_closed:
|
|
725
|
-
break
|
|
726
|
-
logger.debug(f"等待通道就绪(第{attempt+1}/{max_wait_attempts}次)")
|
|
727
|
-
await asyncio.sleep(wait_interval)
|
|
728
|
-
channel, _, _ = await self._get_connection_resources()
|
|
729
|
-
if not channel or channel.is_closed:
|
|
730
|
-
raise Exception("通道初始化失败,无法启动消费")
|
|
731
|
-
|
|
732
|
-
# 获取消费状态和资源
|
|
733
|
-
_, handler, consumer_tag = await self._get_consume_state()
|
|
734
|
-
_, exchange, queue = await self._get_connection_resources()
|
|
735
|
-
|
|
736
|
-
# 检查是否已在消费(通过 consumer_tag 是否存在判断)
|
|
737
|
-
if consumer_tag:
|
|
738
|
-
logger.info(f"已经在消费中,返回现有consumer_tag: {consumer_tag}")
|
|
739
|
-
return consumer_tag
|
|
740
|
-
|
|
741
|
-
# 检查必要条件
|
|
742
|
-
if not handler:
|
|
743
|
-
raise Exception("未设置消息处理函数,请先调用set_message_handler")
|
|
744
|
-
if not queue:
|
|
745
|
-
raise Exception("队列未初始化,无法开始消费")
|
|
746
|
-
if not channel or channel.is_closed:
|
|
747
|
-
raise Exception("通道无效,无法开始消费")
|
|
748
|
-
|
|
749
|
-
try:
|
|
750
|
-
# 启动消费
|
|
751
|
-
new_consumer_tag = await queue.consume(
|
|
752
|
-
self._message_wrapper,
|
|
753
|
-
no_ack=False # 手动确认消息
|
|
754
|
-
)
|
|
755
|
-
|
|
756
|
-
if not new_consumer_tag:
|
|
757
|
-
raise Exception("未能获取到有效的consumer_tag")
|
|
758
|
-
|
|
759
|
-
# 更新消费状态(设置消费者标签)
|
|
760
|
-
await self._set_consumer_tag(new_consumer_tag)
|
|
761
|
-
logger.info(
|
|
762
|
-
f"消费者已启动,队列: {queue.name}, tag: {new_consumer_tag}")
|
|
763
|
-
return new_consumer_tag
|
|
257
|
+
async with self._connect_lock:
|
|
258
|
+
if not self._exchange:
|
|
259
|
+
# 交换机未初始化,重新声明
|
|
260
|
+
logger.warning("交换机未初始化,重新声明")
|
|
261
|
+
self._exchange = await self._channel.declare_exchange(
|
|
262
|
+
name=self.exchange_name,
|
|
263
|
+
type=self.exchange_type,
|
|
264
|
+
durable=self.durable,
|
|
265
|
+
auto_delete=self.auto_delete
|
|
266
|
+
)
|
|
267
|
+
await self._exchange.publish(
|
|
268
|
+
message=message,
|
|
269
|
+
routing_key=self.routing_key or self.queue_name or "#"
|
|
270
|
+
)
|
|
271
|
+
logger.debug(f"消息发布成功(routing_key: {self.routing_key})")
|
|
764
272
|
except Exception as e:
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
273
|
+
logger.error(f"发布消息失败: {str(e)}", exc_info=True)
|
|
274
|
+
# 发布失败时清理状态,下次自动重连
|
|
275
|
+
self._exchange = None
|
|
768
276
|
raise
|
|
769
277
|
|
|
770
|
-
async def
|
|
771
|
-
"""
|
|
772
|
-
|
|
773
|
-
_, _, consumer_tag = await self._get_consume_state()
|
|
774
|
-
_, _, queue = await self._get_connection_resources()
|
|
775
|
-
|
|
776
|
-
if not consumer_tag: # 无消费标签说明未在消费
|
|
777
|
-
logger.info("未处于消费状态,无需停止")
|
|
778
|
-
return
|
|
779
|
-
|
|
780
|
-
logger.info(f"开始停止消费(consumer_tag: {consumer_tag})")
|
|
781
|
-
|
|
782
|
-
# 先清除消费标签
|
|
783
|
-
await self._set_consumer_tag(None)
|
|
784
|
-
|
|
785
|
-
# 取消消费者
|
|
786
|
-
if consumer_tag and queue and not await self._is_closed():
|
|
787
|
-
await self._safe_cancel_consumer(consumer_tag, queue)
|
|
788
|
-
|
|
789
|
-
# 等待所有正在处理的消息完成
|
|
790
|
-
tracking_count = await self._get_tracking_count()
|
|
791
|
-
if tracking_count > 0:
|
|
792
|
-
logger.info(f"等待 {tracking_count} 个正在处理的消息完成...")
|
|
793
|
-
wait_start = asyncio.get_event_loop().time()
|
|
794
|
-
while True:
|
|
795
|
-
# 检查是否超时或已关闭
|
|
796
|
-
if await self._is_closed() or asyncio.get_event_loop().time() - wait_start > 30:
|
|
797
|
-
timeout = asyncio.get_event_loop().time() - wait_start > 30
|
|
798
|
-
if timeout:
|
|
799
|
-
logger.warning("等待消息处理超时,强制清理跟踪记录")
|
|
800
|
-
await self._clear_tracking_messages()
|
|
801
|
-
break
|
|
802
|
-
# 检查跟踪记录是否为空
|
|
803
|
-
current_count = await self._get_tracking_count()
|
|
804
|
-
if current_count == 0:
|
|
805
|
-
break
|
|
806
|
-
await asyncio.sleep(1)
|
|
807
|
-
|
|
808
|
-
logger.info(f"已停止消费队列: {queue.name if queue else '未知'}")
|
|
809
|
-
|
|
810
|
-
async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
|
|
811
|
-
"""解析消息体(无锁,仅处理局部变量)"""
|
|
812
|
-
try:
|
|
813
|
-
body_str = message.body.decode('utf-8')
|
|
814
|
-
self._update_activity_timestamp()
|
|
815
|
-
|
|
816
|
-
if self.auto_parse_json:
|
|
817
|
-
return json.loads(body_str)
|
|
818
|
-
return body_str
|
|
819
|
-
except json.JSONDecodeError:
|
|
820
|
-
logger.warning(
|
|
821
|
-
f"消息 {message.message_id or id(message)} 解析JSON失败,返回原始字符串")
|
|
822
|
-
return body_str
|
|
823
|
-
except Exception as e:
|
|
824
|
-
logger.error(
|
|
825
|
-
f"消息 {message.message_id or id(message)} 解析出错: {str(e)}")
|
|
826
|
-
return message.body.decode('utf-8')
|
|
827
|
-
|
|
828
|
-
async def _handle_business_retry(
|
|
829
|
-
self,
|
|
830
|
-
message: AbstractIncomingMessage,
|
|
831
|
-
error: Exception,
|
|
832
|
-
drop: bool = True
|
|
833
|
-
) -> None:
|
|
834
|
-
"""
|
|
835
|
-
封装业务失败重试逻辑:更新重试计数Header,延迟3秒重新发布
|
|
836
|
-
达到最大次数则标记失败(无锁,仅通过原子方法操作跟踪记录)
|
|
837
|
-
"""
|
|
838
|
-
# 获取当前重试次数
|
|
839
|
-
current_headers = message.headers or {}
|
|
840
|
-
retry_count = current_headers.get('x-retry-count', 0)
|
|
841
|
-
retry_count += 1
|
|
842
|
-
message_id = message.message_id or str(id(message))
|
|
843
|
-
|
|
844
|
-
error_msg = f"[{type(error).__name__}] {str(error)}"[:200]
|
|
845
|
-
|
|
846
|
-
# 打印错误日志
|
|
847
|
-
logger.error(
|
|
848
|
-
f"消息 {message_id} 处理出错(第{retry_count}次重试): {error_msg}",
|
|
849
|
-
exc_info=True
|
|
850
|
-
)
|
|
851
|
-
|
|
852
|
-
# 达到最大重试次数:ack标记失败
|
|
853
|
-
if drop and retry_count >= MAX_RETRY_COUNT:
|
|
854
|
-
logger.error(
|
|
855
|
-
f"消息 {message_id} 已达到最大重试次数{MAX_RETRY_COUNT},标记为失败")
|
|
856
|
-
# 标记跟踪记录为已确认
|
|
857
|
-
await self._mark_tracking_acked(message_id)
|
|
858
|
-
await message.ack()
|
|
859
|
-
self._update_activity_timestamp()
|
|
860
|
-
return
|
|
861
|
-
|
|
862
|
-
# 构造新消息Header
|
|
863
|
-
new_headers = current_headers.copy()
|
|
864
|
-
new_headers['x-retry-count'] = retry_count
|
|
865
|
-
new_headers['x-retry-error'] = error_msg
|
|
866
|
-
|
|
867
|
-
# 提交异步任务,延迟3秒后重新发布
|
|
868
|
-
asyncio.create_task(
|
|
869
|
-
self._delayed_republish(
|
|
870
|
-
message, new_headers, retry_count, message_id)
|
|
871
|
-
)
|
|
872
|
-
|
|
873
|
-
async def _delayed_republish(
|
|
874
|
-
self,
|
|
875
|
-
message: AbstractIncomingMessage,
|
|
876
|
-
new_headers: Dict[str, Any],
|
|
877
|
-
retry_count: int,
|
|
878
|
-
message_id: str
|
|
879
|
-
) -> None:
|
|
880
|
-
"""延迟发布重试消息(无锁,仅通过原子方法操作资源)"""
|
|
881
|
-
try:
|
|
882
|
-
# 延迟3秒重试
|
|
883
|
-
await asyncio.sleep(3)
|
|
884
|
-
|
|
885
|
-
# 检查客户端状态
|
|
886
|
-
if await self._is_closed():
|
|
887
|
-
logger.warning(f"客户端已关闭,放弃消息 {message_id} 的重试发布")
|
|
888
|
-
return
|
|
889
|
-
|
|
890
|
-
# 获取交换机
|
|
891
|
-
_, exchange, _ = await self._get_connection_resources()
|
|
892
|
-
if not exchange:
|
|
893
|
-
raise Exception("交换机未初始化,无法发布重试消息")
|
|
894
|
-
|
|
895
|
-
# 构造新消息
|
|
896
|
-
new_message = Message(
|
|
897
|
-
body=message.body,
|
|
898
|
-
content_type=message.content_type,
|
|
899
|
-
headers=new_headers,
|
|
900
|
-
delivery_mode=message.delivery_mode
|
|
901
|
-
)
|
|
902
|
-
|
|
903
|
-
# 重新发布消息
|
|
904
|
-
await exchange.publish(
|
|
905
|
-
new_message,
|
|
906
|
-
routing_key=self.routing_key or '#',
|
|
907
|
-
mandatory=True,
|
|
908
|
-
timeout=5.0
|
|
909
|
-
)
|
|
910
|
-
self._update_activity_timestamp()
|
|
911
|
-
logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
|
|
912
|
-
|
|
913
|
-
# 拒绝原始消息(不重新入队)
|
|
914
|
-
await message.reject(requeue=False)
|
|
915
|
-
# 标记跟踪记录为已确认
|
|
916
|
-
await self._mark_tracking_acked(message_id)
|
|
917
|
-
|
|
918
|
-
except Exception as e:
|
|
919
|
-
logger.error(
|
|
920
|
-
f"消息 {message_id} 延迟发布失败(错误:{str(e)}),触发requeue兜底",
|
|
921
|
-
exc_info=True
|
|
922
|
-
)
|
|
923
|
-
# 发布失败兜底:requeue原始消息
|
|
924
|
-
await message.reject(requeue=True)
|
|
925
|
-
|
|
926
|
-
async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
|
|
927
|
-
"""消息处理包装器(无锁嵌套,仅通过原子方法操作状态)"""
|
|
928
|
-
message_id = message.message_id or str(id(message))
|
|
929
|
-
max_check_attempts = 3
|
|
930
|
-
check_interval = 1
|
|
931
|
-
|
|
932
|
-
# 重试检查消费状态(处理极端并发场景)
|
|
933
|
-
for attempt in range(max_check_attempts):
|
|
934
|
-
_, handler, consumer_tag = await self._get_consume_state()
|
|
935
|
-
if consumer_tag and handler: # 有消费标签且有处理器才继续
|
|
936
|
-
break
|
|
937
|
-
if attempt < max_check_attempts - 1:
|
|
938
|
-
logger.debug(
|
|
939
|
-
f"消息 {message_id} 处理状态检查重试(第{attempt+1}次): "
|
|
940
|
-
f"handler={'存在' if handler else '不存在'}, "
|
|
941
|
-
f"consumer_tag={'存在' if consumer_tag else '不存在'}"
|
|
942
|
-
)
|
|
943
|
-
await asyncio.sleep(check_interval)
|
|
944
|
-
|
|
945
|
-
# 最终状态判断:状态异常则拒绝消息
|
|
946
|
-
_, handler, consumer_tag = await self._get_consume_state()
|
|
947
|
-
if not consumer_tag or not handler:
|
|
948
|
-
err_msg = f"消息 {message_id} 拒绝处理:handler={'存在' if handler else '不存在'}, consumer_tag={'存在' if consumer_tag else '不存在'}"
|
|
949
|
-
logger.warning(err_msg)
|
|
950
|
-
try:
|
|
951
|
-
await self._handle_business_retry(message, Exception(err_msg), drop=False)
|
|
952
|
-
except Exception as e:
|
|
953
|
-
logger.error(f"消息 {message_id} 拒绝处理失败: {e}")
|
|
278
|
+
async def close(self) -> None:
|
|
279
|
+
"""关闭客户端(释放通道)"""
|
|
280
|
+
if self._closed:
|
|
954
281
|
return
|
|
955
282
|
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
|
|
959
|
-
await message.ack()
|
|
960
|
-
return
|
|
283
|
+
self._closed = True
|
|
284
|
+
logger.info("开始关闭RabbitMQ客户端...")
|
|
961
285
|
|
|
962
|
-
#
|
|
963
|
-
|
|
964
|
-
channel_number = channel.number if channel else None
|
|
965
|
-
await self._add_tracking_message(message_id, message.delivery_tag, channel_number)
|
|
286
|
+
# 先停止消费
|
|
287
|
+
await self.stop_consuming()
|
|
966
288
|
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
# 调用业务处理器
|
|
979
|
-
await handler(msg_model, message)
|
|
980
|
-
|
|
981
|
-
# 处理成功:标记跟踪记录并确认消息
|
|
982
|
-
await self._mark_tracking_acked(message_id)
|
|
983
|
-
await message.ack()
|
|
984
|
-
self._update_activity_timestamp()
|
|
985
|
-
self._update_message_processed_timestamp()
|
|
986
|
-
logger.info(f"消息 {message_id} 处理完成并确认")
|
|
289
|
+
# 释放通道到连接池
|
|
290
|
+
async with self._connect_lock:
|
|
291
|
+
if self._channel and not self._channel.is_closed:
|
|
292
|
+
try:
|
|
293
|
+
await self.connection_pool.release_channel(self._channel)
|
|
294
|
+
except Exception as e:
|
|
295
|
+
logger.warning(f"释放通道失败: {str(e)}")
|
|
296
|
+
self._channel = None
|
|
297
|
+
self._exchange = None
|
|
298
|
+
self._queue = None
|
|
987
299
|
|
|
988
|
-
|
|
989
|
-
# 业务处理失败:触发重试逻辑
|
|
990
|
-
await self._handle_business_retry(message, e)
|
|
991
|
-
finally:
|
|
992
|
-
# 清理跟踪记录
|
|
993
|
-
await self._remove_tracking_message(message_id)
|
|
994
|
-
|
|
995
|
-
async def __aenter__(self):
|
|
996
|
-
await self.connect()
|
|
997
|
-
return self
|
|
998
|
-
|
|
999
|
-
async def __aexit__(self, exc_type, exc, tb):
|
|
1000
|
-
await self.close()
|
|
300
|
+
logger.info("RabbitMQ客户端已关闭")
|