sycommon-python-lib 0.1.38__py3-none-any.whl → 0.1.40__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,23 +1,19 @@
1
- from aio_pika.pool import Pool
2
- from aio_pika.abc import AbstractRobustConnection
3
- from aio_pika import connect_robust, Channel
4
- from typing import Optional, List
1
+ from aio_pika import Channel
2
+ from typing import Optional
5
3
  import asyncio
6
4
  import json
7
- from typing import Callable, Coroutine, Optional, Dict, Any, Union, Set
5
+ from typing import Callable, Coroutine, Dict, Any, Union
8
6
  from aio_pika import Message, DeliveryMode, ExchangeType
9
7
  from aio_pika.abc import (
10
- AbstractChannel,
11
8
  AbstractExchange,
12
9
  AbstractQueue,
13
10
  AbstractIncomingMessage,
14
11
  ConsumerTag,
15
12
  )
16
- from aiormq.exceptions import ChannelInvalidStateError, ConnectionClosed
17
-
13
+ from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
18
14
  from sycommon.logging.kafka_log import SYLogger
19
15
  from sycommon.models.mqmsg_model import MQMsgModel
20
- from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
16
+
21
17
 
22
18
  # 最大重试次数限制
23
19
  MAX_RETRY_COUNT = 3
@@ -27,9 +23,8 @@ logger = SYLogger
27
23
 
28
24
  class RabbitMQClient:
29
25
  """
30
- RabbitMQ客户端(基于连接池),支持集群多节点配置
31
- 提供自动故障转移、连接恢复和消息可靠性保障
32
- 采用细粒度锁设计,彻底避免死锁隐患
26
+ RabbitMQ客户端(基于连接池),支持集群、自动重连、消息发布/消费
27
+ 依赖 aio_pika 的内置重连机制,移除手动重连逻辑
33
28
  """
34
29
 
35
30
  def __init__(
@@ -43,958 +38,263 @@ class RabbitMQClient:
43
38
  auto_delete: bool = False,
44
39
  auto_parse_json: bool = True,
45
40
  create_if_not_exists: bool = True,
46
- connection_timeout: int = 10,
47
41
  rpc_timeout: int = 10,
48
- reconnection_delay: int = 1,
49
- max_reconnection_attempts: int = 5,
50
42
  prefetch_count: int = 2,
51
- consumption_stall_threshold: int = 10
43
+ consumption_stall_threshold: int = 60,
52
44
  ):
53
- # 连接池依赖
54
45
  self.connection_pool = connection_pool
46
+ if not self.connection_pool._initialized:
47
+ raise RuntimeError("连接池未初始化,请先调用 connection_pool.init_pools()")
55
48
 
56
- # 交换器和队列参数
49
+ # 交换机配置
57
50
  self.exchange_name = exchange_name
58
- self.exchange_type = ExchangeType(exchange_type)
51
+ self.exchange_type = ExchangeType(exchange_type.lower())
52
+ # 队列配置
59
53
  self.queue_name = queue_name
60
54
  self.routing_key = routing_key
61
55
  self.durable = durable
62
56
  self.auto_delete = auto_delete
63
-
64
- # 行为控制参数
65
57
  self.auto_parse_json = auto_parse_json
66
58
  self.create_if_not_exists = create_if_not_exists
67
- self.connection_timeout = connection_timeout
59
+
60
+ # 运行时配置
68
61
  self.rpc_timeout = rpc_timeout
69
62
  self.prefetch_count = prefetch_count
70
-
71
- # 重连参数
72
- self.reconnection_delay = reconnection_delay
73
- self.max_reconnection_attempts = max_reconnection_attempts
74
-
75
- # 消息处理参数
76
63
  self.consumption_stall_threshold = consumption_stall_threshold
77
64
 
78
- # 通道和资源对象(由 _connection_lock 保护)
79
- self.channel: Optional[AbstractChannel] = None
80
- self.exchange: Optional[AbstractExchange] = None
81
- self.queue: Optional[AbstractQueue] = None
82
-
83
- # 状态跟踪(按类型拆分锁保护)
84
- self.actual_queue_name: Optional[str] = None
85
- self._exchange_exists = False # 由 _connection_lock 保护
86
- self._queue_exists = False # 由 _connection_lock 保护
87
- self._queue_bound = False # 由 _connection_lock 保护
88
- self._closed = False # 由 _connection_lock 保护
89
- # 由 _consume_state_lock 保护:通过 consumer_tag 存在性判断是否在消费
65
+ # 内部状态
66
+ self._channel: Optional[Channel] = None
67
+ self._exchange: Optional[AbstractExchange] = None
68
+ self._queue: Optional[AbstractQueue] = None
90
69
  self._consumer_tag: Optional[ConsumerTag] = None
91
- self._last_activity_timestamp = asyncio.get_event_loop().time()
92
- self._last_message_processed = asyncio.get_event_loop().time()
93
-
94
- # 任务和处理器
95
- self.message_handler: Optional[Callable[
96
- [Union[Dict[str, Any], str], AbstractIncomingMessage],
97
- Coroutine[Any, Any, None]
98
- ]] = None # 由 _consume_state_lock 保护
99
- self._consuming_task: Optional[asyncio.Task] = None
100
- self._reconnect_task: Optional[asyncio.Task] = None
101
- self._keepalive_task: Optional[asyncio.Task] = None
102
- self._monitor_task: Optional[asyncio.Task] = None
103
-
104
- # 消息处理跟踪(由 _tracking_lock 保护)
105
- self._tracking_messages: Dict[str, Dict[str, Any]] = {}
106
-
107
- # 细粒度锁(核心设计:按资源类型拆分,避免嵌套)
108
- # 保护消费状态(message_handler、_consumer_tag)
109
- self._consume_state_lock = asyncio.Lock()
110
- self._tracking_lock = asyncio.Lock() # 保护消息跟踪记录(_tracking_messages)
111
- # 保护连接/资源状态(channel、exchange、queue、_closed等)
112
- self._connection_lock = asyncio.Lock()
70
+ self._message_handler: Optional[Callable] = None
71
+ self._closed = False
72
+
73
+ # 细粒度锁
74
+ self._consume_lock = asyncio.Lock()
75
+ self._connect_lock = asyncio.Lock()
113
76
 
114
77
  @property
115
78
  async def is_connected(self) -> bool:
116
- """异步属性:检查当前通道是否有效(线程安全)"""
117
- async with self._connection_lock:
118
- return (not self._closed and
119
- self.channel is not None and
120
- not self.channel.is_closed and
121
- self.exchange is not None)
122
-
123
- def _update_activity_timestamp(self) -> None:
124
- """更新最后活动时间戳(非共享状态,无需锁)"""
125
- self._last_activity_timestamp = asyncio.get_event_loop().time()
126
-
127
- def _update_message_processed_timestamp(self) -> None:
128
- """更新最后消息处理时间戳(非共享状态,无需锁)"""
129
- self._last_message_processed = asyncio.get_event_loop().time()
130
-
131
- # ------------------------------
132
- # 消费状态操作(_consume_state_lock 专属保护)
133
- # ------------------------------
134
- async def _get_consume_state(self) -> tuple[bool, Optional[Callable], Optional[ConsumerTag]]:
135
- """安全获取消费相关状态(一次性获取,避免多次加锁)"""
136
- async with self._consume_state_lock:
137
- # 通过 _consumer_tag 是否存在判断是否在消费
138
- is_consuming = self._consumer_tag is not None
139
- return is_consuming, self.message_handler, self._consumer_tag
140
-
141
- async def _set_consumer_tag(self, consumer_tag: Optional[ConsumerTag] = None):
142
- """安全更新消费者标签(替代原 _set_consume_state)"""
143
- async with self._consume_state_lock:
144
- old_tag = self._consumer_tag
145
- self._consumer_tag = consumer_tag
146
- old_is_consuming = old_tag is not None
147
- new_is_consuming = consumer_tag is not None
148
- if old_is_consuming != new_is_consuming:
149
- logger.info(f"消费状态变更: {old_is_consuming} → {new_is_consuming}")
150
-
151
- async def set_message_handler(self, handler):
152
- """设置消息处理器(加锁保护,避免并发修改)"""
153
- async with self._consume_state_lock:
154
- self.message_handler = handler
155
- logger.info("消息处理器已设置")
156
-
157
- # ------------------------------
158
- # 连接状态操作(_connection_lock 专属保护)
159
- # ------------------------------
160
- async def _is_closed(self) -> bool:
161
- """检查客户端是否已关闭(线程安全)"""
162
- async with self._connection_lock:
163
- return self._closed
164
-
165
- async def _mark_closed(self):
166
- """标记客户端已关闭(原子操作)"""
167
- async with self._connection_lock:
168
- self._closed = True
169
-
170
- async def _get_connection_resources(self) -> tuple[Optional[AbstractChannel], Optional[AbstractExchange], Optional[AbstractQueue]]:
171
- """安全获取连接资源(channel/exchange/queue)"""
172
- async with self._connection_lock:
173
- return self.channel, self.exchange, self.queue
174
-
175
- async def _reset_connection_state(self):
176
- """重置连接状态(用于重连时,原子操作)"""
177
- async with self._connection_lock:
178
- self._exchange_exists = False
179
- self._queue_exists = False
180
- self._queue_bound = False
181
- self.channel = None
182
- self.exchange = None
183
- self.queue = None
184
- self.actual_queue_name = None
185
-
186
- async def _update_connection_resources(self, channel: AbstractChannel, exchange: AbstractExchange, queue: Optional[AbstractQueue] = None):
187
- """更新连接资源(原子操作)"""
188
- async with self._connection_lock:
189
- self.channel = channel
190
- self.exchange = exchange
191
- self.queue = queue
192
- if queue:
193
- self.actual_queue_name = queue.name
194
-
195
- # ------------------------------
196
- # 消息跟踪操作(_tracking_lock 专属保护)
197
- # ------------------------------
198
- async def _add_tracking_message(self, msg_id: str, delivery_tag: int, channel_number: Optional[int]):
199
- """添加消息跟踪记录(原子操作)"""
200
- async with self._tracking_lock:
201
- self._tracking_messages[msg_id] = {
202
- 'delivery_tag': delivery_tag,
203
- 'acked': False,
204
- 'channel_number': channel_number,
205
- 'start_time': asyncio.get_event_loop().time()
206
- }
207
-
208
- async def _mark_tracking_acked(self, msg_id: str):
209
- """标记消息已确认(原子操作)"""
210
- async with self._tracking_lock:
211
- if msg_id in self._tracking_messages:
212
- self._tracking_messages[msg_id]['acked'] = True
213
-
214
- async def _remove_tracking_message(self, msg_id: str):
215
- """删除消息跟踪记录(原子操作,避免KeyError)"""
216
- async with self._tracking_lock:
217
- if msg_id in self._tracking_messages:
218
- del self._tracking_messages[msg_id]
219
- logger.info(f"已删除消息跟踪信息: {msg_id}")
220
-
221
- async def _check_duplicate_message(self, msg_id: str) -> bool:
222
- """检查消息是否重复处理(原子操作)"""
223
- async with self._tracking_lock:
224
- return msg_id in self._tracking_messages
225
-
226
- async def _get_tracking_count(self) -> int:
227
- """获取当前跟踪的消息数(原子操作)"""
228
- async with self._tracking_lock:
229
- return len(self._tracking_messages)
230
-
231
- async def _cleanup_acked_tracking_messages(self) -> int:
232
- """清理已确认的跟踪记录(原子操作,返回清理数量)"""
233
- async with self._tracking_lock:
234
- acked_ids = [
235
- msg_id for msg_id, info in self._tracking_messages.items() if info.get('acked')]
236
- for msg_id in acked_ids:
237
- del self._tracking_messages[msg_id]
238
- return len(acked_ids)
239
-
240
- async def _clear_tracking_messages(self):
241
- """清空所有跟踪记录(原子操作)"""
242
- async with self._tracking_lock:
243
- self._tracking_messages.clear()
244
-
245
- # ------------------------------
246
- # 基础工具方法
247
- # ------------------------------
248
- async def _get_channel(self) -> AbstractChannel:
249
- """从通道池获取通道(使用上下文管理器,自动归还)"""
250
- if not self.connection_pool.channel_pool:
251
- raise Exception("连接池未初始化,请先调用init_pools")
252
-
253
- async with self.connection_pool.channel_pool.acquire() as channel:
254
- return channel
255
-
256
- async def _check_exchange_exists(self, channel: AbstractChannel) -> bool:
257
- """检查交换机是否存在"""
258
- try:
259
- await asyncio.wait_for(
260
- channel.declare_exchange(
261
- name=self.exchange_name,
262
- type=self.exchange_type,
263
- passive=True
264
- ),
265
- timeout=self.rpc_timeout
266
- )
267
- return True
268
- except Exception:
269
- return False
270
-
271
- async def _check_queue_exists(self, channel: AbstractChannel) -> bool:
272
- """检查队列是否存在"""
273
- if not self.queue_name:
79
+ """异步检查客户端连接状态(属性,不是函数)"""
80
+ if self._closed:
274
81
  return False
275
82
  try:
276
- await asyncio.wait_for(
277
- channel.declare_queue(
278
- name=self.queue_name,
279
- passive=True
280
- ),
281
- timeout=self.rpc_timeout
282
- )
283
- return True
284
- except Exception:
285
- return False
286
-
287
- async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
288
- """将队列绑定到交换机(带重试)"""
289
- bind_routing_key = self.routing_key if self.routing_key else '#'
290
-
291
- for attempt in range(MAX_RETRY_COUNT + 1):
292
- try:
293
- await asyncio.wait_for(
294
- queue.bind(
295
- exchange,
296
- routing_key=bind_routing_key
297
- ),
298
- timeout=self.rpc_timeout
299
- )
300
- logger.info(
301
- f"队列 '{queue.name}' 已绑定到交换机 '{exchange.name}',路由键: {bind_routing_key}")
83
+ # 检查通道是否有效
84
+ if self._channel and not self._channel.is_closed:
85
+ # 检查交换机和队列(如果需要)
86
+ if self.create_if_not_exists and self.queue_name:
87
+ return bool(self._exchange and self._queue)
302
88
  return True
303
- except Exception as e:
304
- logger.warning(
305
- f"队列绑定失败(第{attempt+1}次尝试): {str(e)}")
306
- if attempt < MAX_RETRY_COUNT:
307
- await asyncio.sleep(1)
308
- return False
309
-
310
- # ------------------------------
311
- # 核心业务方法
312
- # ------------------------------
313
- async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
314
- """从连接池获取资源并初始化(交换机、队列)"""
315
- logger.info(
316
- f"连接参数 - force_reconnect={force_reconnect}, "
317
- f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
318
- )
89
+ return False
90
+ except Exception as e:
91
+ logger.warning(f"检查连接状态失败: {str(e)}")
92
+ return False
319
93
 
320
- # 检查是否已关闭
321
- if await self._is_closed():
322
- raise Exception("客户端已关闭,无法连接")
94
+ async def _get_connection_resources(self) -> tuple[Optional[Channel], Optional[AbstractExchange], Optional[AbstractQueue]]:
95
+ """原子获取连接资源(通道、交换机、队列)"""
96
+ async with self._connect_lock:
97
+ return self._channel, self._exchange, self._queue
323
98
 
324
- # 检查是否已连接(非强制重连则直接返回)
325
- if await self.is_connected and not force_reconnect:
326
- logger.info("已处于连接状态,无需重复连接")
327
- return
99
+ async def connect(self) -> None:
100
+ """建立连接并初始化交换机/队列(使用新的通道获取方式)"""
101
+ if self._closed:
102
+ raise RuntimeError("客户端已关闭,无法重新连接")
328
103
 
329
- # 取消现有重连任务
330
- if self._reconnect_task and not self._reconnect_task.done():
331
- self._reconnect_task.cancel()
104
+ async with self._connect_lock:
332
105
  try:
333
- await self._reconnect_task
334
- except asyncio.CancelledError:
335
- logger.info("旧重连任务已取消")
106
+ # 关键修复:使用连接池的 acquire_channel 方法获取通道
107
+ self._channel = await self.connection_pool.acquire_channel()
108
+ # 设置预取计数
109
+ await self._channel.set_qos(prefetch_count=self.prefetch_count)
336
110
 
337
- # 重置连接状态和跟踪记录
338
- await self._reset_connection_state()
339
- await self._clear_tracking_messages()
340
- await self._set_consumer_tag(None) # 重置消费者标签(停止消费)
341
-
342
- retries = 0
343
- last_exception = None
344
-
345
- while retries < self.max_reconnection_attempts:
346
- try:
347
- # 获取新通道
348
- channel = await self._get_channel()
349
- await channel.set_qos(prefetch_count=self.prefetch_count)
350
-
351
- # 处理交换机
352
- exchange_exists = await self._check_exchange_exists(channel)
353
- if not exchange_exists:
354
- if self.create_if_not_exists:
355
- exchange = await asyncio.wait_for(
356
- channel.declare_exchange(
357
- name=self.exchange_name,
358
- type=self.exchange_type,
359
- durable=self.durable,
360
- auto_delete=self.auto_delete
361
- ),
362
- timeout=self.rpc_timeout
363
- )
364
- logger.info(f"已创建交换机 '{self.exchange_name}'")
365
- else:
366
- raise Exception(
367
- f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
368
- else:
369
- exchange = await channel.get_exchange(self.exchange_name)
370
- logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
371
-
372
- # 处理队列
373
- queue = None
374
- if declare_queue and self.queue_name:
375
- queue_exists = await self._check_queue_exists(channel)
376
-
377
- if not queue_exists:
378
- if not self.create_if_not_exists:
379
- raise Exception(
380
- f"队列 '{self.queue_name}' 不存在且不允许自动创建")
381
-
382
- queue = await asyncio.wait_for(
383
- channel.declare_queue(
384
- name=self.queue_name,
385
- durable=self.durable,
386
- auto_delete=self.auto_delete,
387
- exclusive=False
388
- ),
389
- timeout=self.rpc_timeout
390
- )
391
- logger.info(f"已创建队列 '{self.queue_name}'")
392
- else:
393
- queue = await channel.get_queue(self.queue_name)
394
- logger.info(f"使用已存在的队列 '{self.queue_name}'")
111
+ # 声明交换机
112
+ self._exchange = await self._channel.declare_exchange(
113
+ name=self.exchange_name,
114
+ type=self.exchange_type,
115
+ durable=self.durable,
116
+ auto_delete=self.auto_delete,
117
+ passive=not self.create_if_not_exists # 不创建时使用passive模式检查
118
+ )
395
119
 
120
+ # 声明队列(如果配置了队列名)
121
+ if self.queue_name:
122
+ self._queue = await self._channel.declare_queue(
123
+ name=self.queue_name,
124
+ durable=self.durable,
125
+ auto_delete=self.auto_delete,
126
+ passive=not self.create_if_not_exists
127
+ )
396
128
  # 绑定队列到交换机
397
- if queue and exchange:
398
- bound = await self._bind_queue(channel, queue, exchange)
399
- if not bound:
400
- raise Exception(f"队列 '{queue.name}' 绑定到交换机失败")
401
-
402
- # 更新连接资源
403
- await self._update_connection_resources(channel, exchange, queue)
404
-
405
- # 验证连接状态
406
- if not await self.is_connected:
407
- raise Exception("连接验证失败,状态异常")
408
-
409
- # 重新开始消费(如果已设置处理器且之前在消费)
410
- _, handler, consumer_tag = await self._get_consume_state()
411
- if handler and consumer_tag: # 有处理器且之前有消费标签,说明需要恢复消费
412
- await self.start_consuming()
413
-
414
- # 启动监控和保活任务
415
- self._start_monitoring()
416
- self._start_keepalive()
417
-
418
- self._update_activity_timestamp()
419
-
420
- # 首次启动时延迟1秒(避免初始化未完成就接收消息)
421
- if not force_reconnect:
422
- logger.info("客户端初始化成功,延迟1秒接收消息(解决启动时序问题)")
423
- await asyncio.sleep(1)
424
-
425
- logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
426
- return
129
+ await self._queue.bind(
130
+ exchange=self._exchange,
131
+ routing_key=self.routing_key or self.queue_name
132
+ )
133
+ logger.info(
134
+ f"队列 '{self.queue_name}' 已声明并绑定到交换机 '{self.exchange_name}' "
135
+ f"(exchange_type: {self.exchange_type}, routing_key: {self.routing_key})"
136
+ )
137
+ else:
138
+ logger.info(
139
+ f"未配置队列名,仅初始化交换机 '{self.exchange_name}' (exchange_type: {self.exchange_type})")
427
140
 
141
+ logger.info(f"RabbitMQ客户端连接成功(exchange: {self.exchange_name})")
428
142
  except Exception as e:
429
- last_exception = e
430
- logger.warning(f"资源初始化失败: {str(e)},重试中...")
431
- retries += 1
432
- if retries < self.max_reconnection_attempts:
433
- await asyncio.sleep(self.reconnection_delay)
434
-
435
- logger.error(f"最终初始化失败: {str(last_exception)}")
436
- raise Exception(
437
- f"经过{self.max_reconnection_attempts}次重试后仍无法初始化客户端。最后错误: {str(last_exception)}")
438
-
439
- def _start_monitoring(self) -> None:
440
- """启动连接和消费监控任务(无锁,仅通过原子方法访问状态)"""
441
- if self._monitor_task and not self._monitor_task.done():
442
- return
443
-
444
- async def monitor():
445
- while not await self._is_closed():
446
- try:
447
- # 检查通道状态
448
- channel, _, _ = await self._get_connection_resources()
449
- if channel and channel.is_closed:
450
- logger.warning("检测到通道已关闭,尝试重建")
451
- await self._recreate_channel()
452
- continue
453
-
454
- current_time = asyncio.get_event_loop().time()
455
-
456
- # 清理已确认的跟踪记录
457
- cleaned_count = await self._cleanup_acked_tracking_messages()
458
- if cleaned_count > 0:
459
- logger.info(f"清理了 {cleaned_count} 条已确认消息记录")
460
-
461
- # 检查消费停滞(仅当有消费者标签时)
462
- _, handler, consumer_tag = await self._get_consume_state()
463
- if consumer_tag: # 有消费标签说明正在消费
464
- tracking_count = await self._get_tracking_count()
465
- if current_time - self._last_message_processed > self.consumption_stall_threshold:
466
- if tracking_count > 0:
467
- logger.warning(
468
- f"消费停滞,但有 {tracking_count} 个消息正在处理,暂不重启")
469
- else:
470
- logger.info("消费停滞且无消息处理,重启消费")
471
- try:
472
- await self.stop_consuming()
473
- await asyncio.sleep(1)
474
- # 检查处理器是否存在
475
- _, handler, _ = await self._get_consume_state()
476
- if handler:
477
- await self.start_consuming()
478
- else:
479
- logger.error("消费处理器已丢失,无法重启消费")
480
- except Exception as e:
481
- logger.error(
482
- f"重启消费失败: {str(e)}", exc_info=True)
483
- await self._set_consumer_tag(None)
143
+ logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
144
+ # 清理异常状态
145
+ if self._channel:
146
+ await self.connection_pool.release_channel(self._channel)
147
+ self._channel = None
148
+ self._exchange = None
149
+ self._queue = None
150
+ raise
151
+
152
+ async def set_message_handler(self, handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]):
153
+ """设置消息处理器(消费消息时使用)"""
154
+ async with self._consume_lock:
155
+ self._message_handler = handler
156
+ logger.info("消息处理器已设置")
484
157
 
485
- except Exception as e:
486
- logger.error(f"监控任务出错: {str(e)}", exc_info=True)
158
+ async def start_consuming(self) -> Optional[ConsumerTag]:
159
+ """启动消费(返回消费者标签)"""
160
+ if self._closed:
161
+ logger.warning("客户端已关闭,无法启动消费")
162
+ return None
487
163
 
488
- await asyncio.sleep(60) # 监控间隔60秒
164
+ async with self._consume_lock:
165
+ # 检查前置条件
166
+ if not self._message_handler:
167
+ raise RuntimeError("未设置消息处理器,请先调用 set_message_handler")
489
168
 
490
- self._monitor_task = asyncio.create_task(monitor())
491
- logger.info("监控任务已启动")
169
+ # 确保连接已建立
170
+ if not await self.is_connected:
171
+ await self.connect()
492
172
 
493
- async def _recreate_channel(self) -> None:
494
- """重建通道并恢复资源(无锁嵌套)"""
495
- # 先停止消费
496
- await self._set_consumer_tag(None)
497
- logger.info("开始重建通道...")
173
+ # 确保队列已初始化
174
+ _, _, queue = await self._get_connection_resources()
175
+ if not queue:
176
+ raise RuntimeError("队列未初始化,无法启动消费")
498
177
 
499
- try:
500
- # 获取新通道
501
- channel = await self._get_channel()
502
- await channel.set_qos(prefetch_count=self.prefetch_count)
503
-
504
- # 重新获取交换机
505
- exchange = await channel.get_exchange(self.exchange_name)
506
-
507
- # 重新获取队列并绑定
508
- queue = None
509
- if self.queue_name:
510
- queue = await channel.get_queue(self.queue_name)
511
- if queue and exchange:
512
- bound = await self._bind_queue(channel, queue, exchange)
513
- if not bound:
514
- raise Exception("队列绑定失败,通道重建不完整")
515
-
516
- # 更新连接资源
517
- await self._update_connection_resources(channel, exchange, queue)
518
-
519
- # 重新开始消费(如果有处理器)
520
- _, handler, _ = await self._get_consume_state()
521
- if handler:
522
- await self.start_consuming()
523
-
524
- # 清空跟踪记录
525
- await self._clear_tracking_messages()
526
- logger.info("通道已重建并恢复服务")
527
- self._update_activity_timestamp()
528
- except Exception as e:
529
- logger.error(f"通道重建失败: {str(e)},触发重连", exc_info=True)
530
- await self._set_consumer_tag(None)
531
- await self.connect(force_reconnect=True)
532
-
533
- def _start_keepalive(self) -> None:
534
- """启动连接保活任务(无锁,仅通过原子方法访问状态)"""
535
- if self._keepalive_task and not self._keepalive_task.done():
536
- return
537
-
538
- async def keepalive():
539
- while not await self._is_closed():
178
+ # 定义消费回调
179
+ async def consume_callback(message: AbstractIncomingMessage):
540
180
  try:
541
- # 检查连接状态
542
- if not await self.is_connected:
543
- logger.warning("保活任务检测到连接断开,触发重连")
544
- await self.connect(force_reconnect=True)
545
- await asyncio.sleep(5)
546
- continue
547
-
548
- current_time = asyncio.get_event_loop().time()
549
- # 检查活动时间
550
- if current_time - self._last_activity_timestamp > self.connection_pool.heartbeat * 2:
551
- logger.info(
552
- f"连接 {self.connection_pool.heartbeat*2}s 无活动,执行保活检查")
553
- channel, exchange, _ = await self._get_connection_resources()
554
- if channel and not channel.is_closed and exchange:
555
- # 轻量级操作:检查交换机是否存在
556
- await asyncio.wait_for(
557
- channel.declare_exchange(
558
- name=self.exchange_name,
559
- type=self.exchange_type,
560
- passive=True
561
- ),
562
- timeout=5
563
- )
564
- self._update_activity_timestamp()
565
- logger.info("保活检查成功")
566
- else:
567
- raise Exception("连接资源无效")
181
+ # 解析消息
182
+ if self.auto_parse_json:
183
+ body = json.loads(message.body.decode('utf-8'))
184
+ parsed_data = MQMsgModel(**body)
185
+ else:
186
+ parsed_data = MQMsgModel(
187
+ msg=message.body.decode('utf-8'))
568
188
 
189
+ # 调用处理器
190
+ await self._message_handler(parsed_data, message)
191
+ # 手动确认消息
192
+ await message.ack()
569
193
  except Exception as e:
570
- logger.warning(f"保活检查失败: {str(e)},触发重连")
571
- await self.connect(force_reconnect=True)
572
-
573
- await asyncio.sleep(self.connection_pool.heartbeat)
574
-
575
- self._keepalive_task = asyncio.create_task(keepalive())
576
- logger.info("保活任务已启动")
577
-
578
- async def _schedule_reconnect(self) -> None:
579
- """安排重新连接(无锁)"""
580
- if self._reconnect_task and not self._reconnect_task.done():
581
- return
582
-
583
- logger.info(f"将在 {self.reconnection_delay} 秒后尝试重新连接...")
584
-
585
- async def reconnect():
586
- try:
587
- await asyncio.sleep(self.reconnection_delay)
588
- if not await self._is_closed():
589
- await self.connect(force_reconnect=True)
590
- except Exception as e:
591
- logger.error(f"重连任务失败: {str(e)}")
592
- if not await self._is_closed():
593
- await self._schedule_reconnect()
594
-
595
- self._reconnect_task = asyncio.create_task(reconnect())
596
-
597
- async def close(self) -> None:
598
- """关闭客户端并释放资源(原子操作,无锁嵌套)"""
599
- if await self._is_closed():
600
- logger.info("客户端已关闭,无需重复操作")
601
- return
602
-
603
- logger.info("开始关闭RabbitMQ客户端...")
604
-
605
- # 标记为已关闭
606
- await self._mark_closed()
607
-
608
- # 停止消费
609
- await self.stop_consuming()
610
-
611
- # 取消所有后台任务
612
- tasks = [self._keepalive_task,
613
- self._reconnect_task, self._monitor_task]
614
- for task in tasks:
615
- if task and not task.done():
616
- task.cancel()
617
- try:
618
- await task
619
- except asyncio.CancelledError:
620
- logger.info(f"任务 {task.get_name()} 已取消")
194
+ logger.error(
195
+ f"处理消息失败 (delivery_tag: {message.delivery_tag}): {str(e)}", exc_info=True)
196
+ # 消费失败时重新入队(最多重试3次)
197
+ if message.redelivered:
198
+ logger.warning(
199
+ f"消息已重试过,拒绝入队 (delivery_tag: {message.delivery_tag})")
200
+ await message.reject(requeue=False)
201
+ else:
202
+ await message.nack(requeue=True)
621
203
 
622
- # 重置所有状态和资源
623
- await self._reset_connection_state()
624
- await self._clear_tracking_messages()
625
- async with self._consume_state_lock:
626
- self.message_handler = None
627
- self._consumer_tag = None
204
+ # 启动消费
205
+ self._consumer_tag = await queue.consume(consume_callback)
206
+ logger.info(
207
+ f"开始消费队列 '{queue.name}',consumer_tag: {self._consumer_tag}")
208
+ return self._consumer_tag
628
209
 
629
- logger.info("RabbitMQ客户端已完全关闭")
210
+ async def stop_consuming(self) -> None:
211
+ """停止消费"""
212
+ async with self._consume_lock:
213
+ if self._consumer_tag and not self._closed:
214
+ _, _, queue = await self._get_connection_resources()
215
+ if queue and not queue.is_closed:
216
+ try:
217
+ await queue.cancel(self._consumer_tag)
218
+ logger.info(f"停止消费,consumer_tag: {self._consumer_tag}")
219
+ except Exception as e:
220
+ logger.error(f"停止消费失败: {str(e)}")
221
+ self._consumer_tag = None
630
222
 
631
223
  async def publish(
632
224
  self,
633
- message_body: Union[str, Dict[str, Any]],
634
- routing_key: Optional[str] = None,
635
- content_type: str = "application/json",
225
+ message_body: Union[str, Dict[str, Any], MQMsgModel],
636
226
  headers: Optional[Dict[str, Any]] = None,
227
+ content_type: str = "application/json",
637
228
  delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
638
229
  ) -> None:
639
- """发布消息(从池获取通道,自动重试,无锁冲突)"""
640
- if await self._is_closed():
641
- raise Exception("客户端已关闭,无法发布消息")
230
+ """发布消息(修复 is_closed 问题)"""
231
+ if self._closed:
232
+ raise RuntimeError("客户端已关闭,无法发布消息")
642
233
 
643
- # 检查连接状态
234
+ # 确保连接已建立
644
235
  if not await self.is_connected:
645
- logger.warning("连接已断开,尝试重连后发布消息")
646
- await self.connect(force_reconnect=True)
236
+ await self.connect()
647
237
 
648
238
  # 处理消息体
649
- if isinstance(message_body, dict):
650
- message_body_str = json.dumps(message_body, ensure_ascii=False)
651
- if content_type == "text/plain":
652
- content_type = "application/json"
239
+ if isinstance(message_body, MQMsgModel):
240
+ body = json.dumps(message_body.__dict__,
241
+ ensure_ascii=False).encode('utf-8')
242
+ elif isinstance(message_body, dict):
243
+ body = json.dumps(message_body, ensure_ascii=False).encode('utf-8')
653
244
  else:
654
- message_body_str = str(message_body)
245
+ body = str(message_body).encode('utf-8')
655
246
 
656
- # 创建消息对象
247
+ # 创建消息
657
248
  message = Message(
658
- body=message_body_str.encode(),
659
- content_type=content_type,
249
+ body=body,
660
250
  headers=headers or {},
251
+ content_type=content_type,
661
252
  delivery_mode=delivery_mode
662
253
  )
663
254
 
664
- # 发布消息(带重试机制)
665
- retry_count = 0
666
- max_retries = 2
667
- while retry_count < max_retries:
668
- try:
669
- async with self.connection_pool.channel_pool.acquire() as publish_channel:
670
- exchange = await publish_channel.get_exchange(self.exchange_name)
671
- confirmed = await exchange.publish(
672
- message,
673
- routing_key=routing_key or self.routing_key or '#',
674
- mandatory=True,
675
- timeout=5
676
- )
677
- if not confirmed:
678
- raise Exception("消息未被服务器确认接收")
679
-
680
- self._update_activity_timestamp()
681
- logger.info(
682
- f"消息已发布到交换机 '{self.exchange_name}'(路由键: {routing_key or self.routing_key or '#'})")
683
- return
684
- except (ConnectionClosed, ChannelInvalidStateError, asyncio.TimeoutError):
685
- retry_count += 1
686
- logger.warning(f"连接异常,尝试重连后重新发布 (重试次数: {retry_count})")
687
- await self.connect(force_reconnect=True)
688
- except Exception as e:
689
- retry_count += 1
690
- logger.error(f"消息发布失败 (重试次数: {retry_count}): {str(e)}")
691
- if retry_count < max_retries:
692
- await asyncio.sleep(1)
693
-
694
- raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
695
-
696
- async def _safe_cancel_consumer(self, consumer_tag: ConsumerTag, queue: AbstractQueue) -> bool:
697
- """安全取消消费者(无锁,仅操作传入的局部变量)"""
255
+ # 发布消息
698
256
  try:
699
- await asyncio.wait_for(
700
- queue.cancel(consumer_tag),
701
- timeout=self.rpc_timeout
702
- )
703
- logger.info(f"消费者 {consumer_tag} 已取消")
704
- return True
705
- except Exception as e:
706
- logger.error(f"取消消费者 {consumer_tag} 异常: {str(e)}")
707
- return False
708
-
709
- async def start_consuming(self) -> ConsumerTag:
710
- """启动消费(无锁嵌套,通过原子方法获取/更新状态)"""
711
- # 检查客户端状态
712
- if await self._is_closed():
713
- raise Exception("客户端已关闭,无法启动消费")
714
-
715
- # 检查连接状态(确保通道完全就绪)
716
- if not await self.is_connected:
717
- await self.connect()
718
-
719
- # 确保通道未关闭(解决启动时通道初始化滞后问题)
720
- channel, _, _ = await self._get_connection_resources()
721
- max_wait_attempts = 5
722
- wait_interval = 0.5
723
- for attempt in range(max_wait_attempts):
724
- if channel and not channel.is_closed:
725
- break
726
- logger.debug(f"等待通道就绪(第{attempt+1}/{max_wait_attempts}次)")
727
- await asyncio.sleep(wait_interval)
728
- channel, _, _ = await self._get_connection_resources()
729
- if not channel or channel.is_closed:
730
- raise Exception("通道初始化失败,无法启动消费")
731
-
732
- # 获取消费状态和资源
733
- _, handler, consumer_tag = await self._get_consume_state()
734
- _, exchange, queue = await self._get_connection_resources()
735
-
736
- # 检查是否已在消费(通过 consumer_tag 是否存在判断)
737
- if consumer_tag:
738
- logger.info(f"已经在消费中,返回现有consumer_tag: {consumer_tag}")
739
- return consumer_tag
740
-
741
- # 检查必要条件
742
- if not handler:
743
- raise Exception("未设置消息处理函数,请先调用set_message_handler")
744
- if not queue:
745
- raise Exception("队列未初始化,无法开始消费")
746
- if not channel or channel.is_closed:
747
- raise Exception("通道无效,无法开始消费")
748
-
749
- try:
750
- # 启动消费
751
- new_consumer_tag = await queue.consume(
752
- self._message_wrapper,
753
- no_ack=False # 手动确认消息
754
- )
755
-
756
- if not new_consumer_tag:
757
- raise Exception("未能获取到有效的consumer_tag")
758
-
759
- # 更新消费状态(设置消费者标签)
760
- await self._set_consumer_tag(new_consumer_tag)
761
- logger.info(
762
- f"消费者已启动,队列: {queue.name}, tag: {new_consumer_tag}")
763
- return new_consumer_tag
257
+ async with self._connect_lock:
258
+ if not self._exchange:
259
+ # 交换机未初始化,重新声明
260
+ logger.warning("交换机未初始化,重新声明")
261
+ self._exchange = await self._channel.declare_exchange(
262
+ name=self.exchange_name,
263
+ type=self.exchange_type,
264
+ durable=self.durable,
265
+ auto_delete=self.auto_delete
266
+ )
267
+ await self._exchange.publish(
268
+ message=message,
269
+ routing_key=self.routing_key or self.queue_name or "#"
270
+ )
271
+ logger.debug(f"消息发布成功(routing_key: {self.routing_key})")
764
272
  except Exception as e:
765
- # 异常时回滚状态
766
- await self._set_consumer_tag(None)
767
- logger.error(f"启动消费失败: {str(e)}", exc_info=True)
273
+ logger.error(f"发布消息失败: {str(e)}", exc_info=True)
274
+ # 发布失败时清理状态,下次自动重连
275
+ self._exchange = None
768
276
  raise
769
277
 
770
- async def stop_consuming(self) -> None:
771
- """停止消费(无锁嵌套,通过原子方法获取/更新状态)"""
772
- # 获取消费状态和资源
773
- _, _, consumer_tag = await self._get_consume_state()
774
- _, _, queue = await self._get_connection_resources()
775
-
776
- if not consumer_tag: # 无消费标签说明未在消费
777
- logger.info("未处于消费状态,无需停止")
778
- return
779
-
780
- logger.info(f"开始停止消费(consumer_tag: {consumer_tag})")
781
-
782
- # 先清除消费标签
783
- await self._set_consumer_tag(None)
784
-
785
- # 取消消费者
786
- if consumer_tag and queue and not await self._is_closed():
787
- await self._safe_cancel_consumer(consumer_tag, queue)
788
-
789
- # 等待所有正在处理的消息完成
790
- tracking_count = await self._get_tracking_count()
791
- if tracking_count > 0:
792
- logger.info(f"等待 {tracking_count} 个正在处理的消息完成...")
793
- wait_start = asyncio.get_event_loop().time()
794
- while True:
795
- # 检查是否超时或已关闭
796
- if await self._is_closed() or asyncio.get_event_loop().time() - wait_start > 30:
797
- timeout = asyncio.get_event_loop().time() - wait_start > 30
798
- if timeout:
799
- logger.warning("等待消息处理超时,强制清理跟踪记录")
800
- await self._clear_tracking_messages()
801
- break
802
- # 检查跟踪记录是否为空
803
- current_count = await self._get_tracking_count()
804
- if current_count == 0:
805
- break
806
- await asyncio.sleep(1)
807
-
808
- logger.info(f"已停止消费队列: {queue.name if queue else '未知'}")
809
-
810
- async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
811
- """解析消息体(无锁,仅处理局部变量)"""
812
- try:
813
- body_str = message.body.decode('utf-8')
814
- self._update_activity_timestamp()
815
-
816
- if self.auto_parse_json:
817
- return json.loads(body_str)
818
- return body_str
819
- except json.JSONDecodeError:
820
- logger.warning(
821
- f"消息 {message.message_id or id(message)} 解析JSON失败,返回原始字符串")
822
- return body_str
823
- except Exception as e:
824
- logger.error(
825
- f"消息 {message.message_id or id(message)} 解析出错: {str(e)}")
826
- return message.body.decode('utf-8')
827
-
828
- async def _handle_business_retry(
829
- self,
830
- message: AbstractIncomingMessage,
831
- error: Exception,
832
- drop: bool = True
833
- ) -> None:
834
- """
835
- 封装业务失败重试逻辑:更新重试计数Header,延迟3秒重新发布
836
- 达到最大次数则标记失败(无锁,仅通过原子方法操作跟踪记录)
837
- """
838
- # 获取当前重试次数
839
- current_headers = message.headers or {}
840
- retry_count = current_headers.get('x-retry-count', 0)
841
- retry_count += 1
842
- message_id = message.message_id or str(id(message))
843
-
844
- error_msg = f"[{type(error).__name__}] {str(error)}"[:200]
845
-
846
- # 打印错误日志
847
- logger.error(
848
- f"消息 {message_id} 处理出错(第{retry_count}次重试): {error_msg}",
849
- exc_info=True
850
- )
851
-
852
- # 达到最大重试次数:ack标记失败
853
- if drop and retry_count >= MAX_RETRY_COUNT:
854
- logger.error(
855
- f"消息 {message_id} 已达到最大重试次数{MAX_RETRY_COUNT},标记为失败")
856
- # 标记跟踪记录为已确认
857
- await self._mark_tracking_acked(message_id)
858
- await message.ack()
859
- self._update_activity_timestamp()
860
- return
861
-
862
- # 构造新消息Header
863
- new_headers = current_headers.copy()
864
- new_headers['x-retry-count'] = retry_count
865
- new_headers['x-retry-error'] = error_msg
866
-
867
- # 提交异步任务,延迟3秒后重新发布
868
- asyncio.create_task(
869
- self._delayed_republish(
870
- message, new_headers, retry_count, message_id)
871
- )
872
-
873
- async def _delayed_republish(
874
- self,
875
- message: AbstractIncomingMessage,
876
- new_headers: Dict[str, Any],
877
- retry_count: int,
878
- message_id: str
879
- ) -> None:
880
- """延迟发布重试消息(无锁,仅通过原子方法操作资源)"""
881
- try:
882
- # 延迟3秒重试
883
- await asyncio.sleep(3)
884
-
885
- # 检查客户端状态
886
- if await self._is_closed():
887
- logger.warning(f"客户端已关闭,放弃消息 {message_id} 的重试发布")
888
- return
889
-
890
- # 获取交换机
891
- _, exchange, _ = await self._get_connection_resources()
892
- if not exchange:
893
- raise Exception("交换机未初始化,无法发布重试消息")
894
-
895
- # 构造新消息
896
- new_message = Message(
897
- body=message.body,
898
- content_type=message.content_type,
899
- headers=new_headers,
900
- delivery_mode=message.delivery_mode
901
- )
902
-
903
- # 重新发布消息
904
- await exchange.publish(
905
- new_message,
906
- routing_key=self.routing_key or '#',
907
- mandatory=True,
908
- timeout=5.0
909
- )
910
- self._update_activity_timestamp()
911
- logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
912
-
913
- # 拒绝原始消息(不重新入队)
914
- await message.reject(requeue=False)
915
- # 标记跟踪记录为已确认
916
- await self._mark_tracking_acked(message_id)
917
-
918
- except Exception as e:
919
- logger.error(
920
- f"消息 {message_id} 延迟发布失败(错误:{str(e)}),触发requeue兜底",
921
- exc_info=True
922
- )
923
- # 发布失败兜底:requeue原始消息
924
- await message.reject(requeue=True)
925
-
926
- async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
927
- """消息处理包装器(无锁嵌套,仅通过原子方法操作状态)"""
928
- message_id = message.message_id or str(id(message))
929
- max_check_attempts = 3
930
- check_interval = 1
931
-
932
- # 重试检查消费状态(处理极端并发场景)
933
- for attempt in range(max_check_attempts):
934
- _, handler, consumer_tag = await self._get_consume_state()
935
- if consumer_tag and handler: # 有消费标签且有处理器才继续
936
- break
937
- if attempt < max_check_attempts - 1:
938
- logger.debug(
939
- f"消息 {message_id} 处理状态检查重试(第{attempt+1}次): "
940
- f"handler={'存在' if handler else '不存在'}, "
941
- f"consumer_tag={'存在' if consumer_tag else '不存在'}"
942
- )
943
- await asyncio.sleep(check_interval)
944
-
945
- # 最终状态判断:状态异常则拒绝消息
946
- _, handler, consumer_tag = await self._get_consume_state()
947
- if not consumer_tag or not handler:
948
- err_msg = f"消息 {message_id} 拒绝处理:handler={'存在' if handler else '不存在'}, consumer_tag={'存在' if consumer_tag else '不存在'}"
949
- logger.warning(err_msg)
950
- try:
951
- await self._handle_business_retry(message, Exception(err_msg), drop=False)
952
- except Exception as e:
953
- logger.error(f"消息 {message_id} 拒绝处理失败: {e}")
278
+ async def close(self) -> None:
279
+ """关闭客户端(释放通道)"""
280
+ if self._closed:
954
281
  return
955
282
 
956
- # 检查重复处理
957
- if await self._check_duplicate_message(message_id):
958
- logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
959
- await message.ack()
960
- return
283
+ self._closed = True
284
+ logger.info("开始关闭RabbitMQ客户端...")
961
285
 
962
- # 添加跟踪记录
963
- channel, _, _ = await self._get_connection_resources()
964
- channel_number = channel.number if channel else None
965
- await self._add_tracking_message(message_id, message.delivery_tag, channel_number)
286
+ # 先停止消费
287
+ await self.stop_consuming()
966
288
 
967
- try:
968
- logger.info(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
969
-
970
- # 解析消息
971
- parsed_data = await self._parse_message(message)
972
- # 转换为MQMsgModel
973
- if isinstance(parsed_data, dict):
974
- msg_model = MQMsgModel(**parsed_data)
975
- else:
976
- msg_model = MQMsgModel(data=parsed_data)
977
-
978
- # 调用业务处理器
979
- await handler(msg_model, message)
980
-
981
- # 处理成功:标记跟踪记录并确认消息
982
- await self._mark_tracking_acked(message_id)
983
- await message.ack()
984
- self._update_activity_timestamp()
985
- self._update_message_processed_timestamp()
986
- logger.info(f"消息 {message_id} 处理完成并确认")
289
+ # 释放通道到连接池
290
+ async with self._connect_lock:
291
+ if self._channel and not self._channel.is_closed:
292
+ try:
293
+ await self.connection_pool.release_channel(self._channel)
294
+ except Exception as e:
295
+ logger.warning(f"释放通道失败: {str(e)}")
296
+ self._channel = None
297
+ self._exchange = None
298
+ self._queue = None
987
299
 
988
- except Exception as e:
989
- # 业务处理失败:触发重试逻辑
990
- await self._handle_business_retry(message, e)
991
- finally:
992
- # 清理跟踪记录
993
- await self._remove_tracking_message(message_id)
994
-
995
- async def __aenter__(self):
996
- await self.connect()
997
- return self
998
-
999
- async def __aexit__(self, exc_type, exc, tb):
1000
- await self.close()
300
+ logger.info("RabbitMQ客户端已关闭")