sycommon-python-lib 0.1.32__py3-none-any.whl → 0.1.56b5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. sycommon/config/Config.py +6 -2
  2. sycommon/config/RerankerConfig.py +1 -0
  3. sycommon/database/async_base_db_service.py +36 -0
  4. sycommon/database/async_database_service.py +96 -0
  5. sycommon/llm/__init__.py +0 -0
  6. sycommon/llm/embedding.py +149 -0
  7. sycommon/llm/get_llm.py +246 -0
  8. sycommon/llm/llm_logger.py +126 -0
  9. sycommon/llm/llm_tokens.py +119 -0
  10. sycommon/logging/async_sql_logger.py +65 -0
  11. sycommon/logging/kafka_log.py +21 -9
  12. sycommon/logging/logger_levels.py +23 -0
  13. sycommon/middleware/context.py +2 -0
  14. sycommon/middleware/traceid.py +155 -32
  15. sycommon/notice/__init__.py +0 -0
  16. sycommon/notice/uvicorn_monitor.py +195 -0
  17. sycommon/rabbitmq/rabbitmq_client.py +385 -626
  18. sycommon/rabbitmq/rabbitmq_pool.py +287 -71
  19. sycommon/rabbitmq/rabbitmq_service.py +345 -191
  20. sycommon/services.py +104 -64
  21. sycommon/synacos/feign.py +71 -18
  22. sycommon/synacos/feign_client.py +26 -8
  23. sycommon/synacos/nacos_service.py +91 -54
  24. sycommon/tools/merge_headers.py +97 -0
  25. sycommon/tools/snowflake.py +290 -23
  26. {sycommon_python_lib-0.1.32.dist-info → sycommon_python_lib-0.1.56b5.dist-info}/METADATA +17 -12
  27. {sycommon_python_lib-0.1.32.dist-info → sycommon_python_lib-0.1.56b5.dist-info}/RECORD +30 -18
  28. {sycommon_python_lib-0.1.32.dist-info → sycommon_python_lib-0.1.56b5.dist-info}/WHEEL +0 -0
  29. {sycommon_python_lib-0.1.32.dist-info → sycommon_python_lib-0.1.56b5.dist-info}/entry_points.txt +0 -0
  30. {sycommon_python_lib-0.1.32.dist-info → sycommon_python_lib-0.1.56b5.dist-info}/top_level.txt +0 -0
@@ -1,30 +1,32 @@
1
+ from aio_pika import Channel
2
+ from typing import Optional
1
3
  import asyncio
2
4
  import json
3
- from typing import Callable, Coroutine, Optional, Dict, Any, Union, Set
5
+ from typing import Callable, Coroutine, Dict, Any, Union
4
6
  from aio_pika import Message, DeliveryMode, ExchangeType
5
7
  from aio_pika.abc import (
6
- AbstractChannel,
7
8
  AbstractExchange,
8
9
  AbstractQueue,
9
10
  AbstractIncomingMessage,
10
11
  ConsumerTag,
12
+ AbstractRobustConnection,
11
13
  )
12
- from aiormq.exceptions import ChannelInvalidStateError, ConnectionClosed
13
-
14
+ from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
14
15
  from sycommon.logging.kafka_log import SYLogger
15
16
  from sycommon.models.mqmsg_model import MQMsgModel
16
- from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
17
-
18
- # 最大重试次数限制
19
- MAX_RETRY_COUNT = 3
20
17
 
21
18
  logger = SYLogger
22
19
 
23
20
 
24
21
  class RabbitMQClient:
25
22
  """
26
- RabbitMQ客户端(基于连接池),支持集群多节点配置
27
- 提供自动故障转移、连接恢复和消息可靠性保障
23
+ RabbitMQ 客户端(支持消息发布、消费、自动重连、异常重试)
24
+ 核心特性:
25
+ 1. 基于单通道连接池复用资源,性能优化
26
+ 2. 依赖连接池原生自动重连,客户端仅重建自身资源
27
+ 3. 消息发布支持重试+mandatory机制+超时控制,确保路由有效
28
+ 4. 消费支持手动ACK/NACK
29
+ 5. 兼容JSON/字符串/字典消息格式
28
30
  """
29
31
 
30
32
  def __init__(
@@ -38,668 +40,425 @@ class RabbitMQClient:
38
40
  auto_delete: bool = False,
39
41
  auto_parse_json: bool = True,
40
42
  create_if_not_exists: bool = True,
41
- connection_timeout: int = 10,
42
- rpc_timeout: int = 10,
43
- reconnection_delay: int = 1,
44
- max_reconnection_attempts: int = 5,
45
- prefetch_count: int = 2,
46
- consumption_stall_threshold: int = 10
43
+ **kwargs,
47
44
  ):
48
- """
49
- 初始化RabbitMQ客户端(依赖连接池)
50
-
51
- :param connection_pool: 连接池实例
52
- """
53
- # 连接池依赖
45
+ # 依赖注入:连接池(必须已初始化)
54
46
  self.connection_pool = connection_pool
47
+ if not self.connection_pool._initialized:
48
+ raise RuntimeError("连接池未初始化,请先调用 connection_pool.init_pools()")
55
49
 
56
- # 交换器和队列参数
57
- self.exchange_name = exchange_name
58
- self.exchange_type = ExchangeType(exchange_type)
59
- self.queue_name = queue_name
60
- self.routing_key = routing_key
61
- self.durable = durable
62
- self.auto_delete = auto_delete
63
-
64
- # 行为控制参数
65
- self.auto_parse_json = auto_parse_json
66
- self.create_if_not_exists = create_if_not_exists
67
- self.connection_timeout = connection_timeout
68
- self.rpc_timeout = rpc_timeout
69
- self.prefetch_count = prefetch_count
70
-
71
- # 重连参数
72
- self.reconnection_delay = reconnection_delay
73
- self.max_reconnection_attempts = max_reconnection_attempts
74
-
75
- # 消息处理参数
76
- self.consumption_stall_threshold = consumption_stall_threshold
77
-
78
- # 通道和资源对象(从池获取)
79
- self.channel: Optional[AbstractChannel] = None
80
- self.exchange: Optional[AbstractExchange] = None
81
- self.queue: Optional[AbstractQueue] = None
82
-
83
- # 状态跟踪
84
- self.actual_queue_name: Optional[str] = None
85
- self._exchange_exists = False
86
- self._queue_exists = False
87
- self._queue_bound = False
88
- self._is_consuming = False
89
- self._closed = False
50
+ # 交换机配置
51
+ self.exchange_name = exchange_name.strip()
52
+ try:
53
+ self.exchange_type = ExchangeType(exchange_type.lower())
54
+ except ValueError:
55
+ logger.warning(f"无效的exchange_type: {exchange_type},默认使用'topic'")
56
+ self.exchange_type = ExchangeType.TOPIC
57
+
58
+ # 队列配置
59
+ self.queue_name = queue_name.strip() if queue_name else None
60
+ self.routing_key = routing_key.strip() if routing_key else "#"
61
+ self.durable = durable # 消息/队列持久化
62
+ self.auto_delete = auto_delete # 无消费者时自动删除队列/交换机
63
+ self.auto_parse_json = auto_parse_json # 自动解析JSON消息体
64
+ self.create_if_not_exists = create_if_not_exists # 不存在则创建交换机/队列
65
+
66
+ # 内部状态(资源+连接)
67
+ self._channel: Optional[Channel] = None
68
+ self._channel_conn: Optional[AbstractRobustConnection] = None # 通道所属连接
69
+ self._exchange: Optional[AbstractExchange] = None
70
+ self._queue: Optional[AbstractQueue] = None
90
71
  self._consumer_tag: Optional[ConsumerTag] = None
91
- self._last_activity_timestamp = asyncio.get_event_loop().time()
92
- self._last_message_processed = asyncio.get_event_loop().time()
93
-
94
- # 任务和处理器
95
- self.message_handler: Optional[Callable[
96
- [Union[Dict[str, Any], str], AbstractIncomingMessage],
97
- Coroutine[Any, Any, None]
98
- ]] = None
99
- self._consuming_task: Optional[asyncio.Task] = None
100
- self._reconnect_task: Optional[asyncio.Task] = None
101
- self._keepalive_task: Optional[asyncio.Task] = None
102
- self._monitor_task: Optional[asyncio.Task] = None
103
-
104
- # 消息处理跟踪
105
- self._tracking_messages: Dict[str, Dict[str, Any]] = {}
72
+ self._message_handler: Optional[Callable[[
73
+ MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]] = None
74
+ self._closed = False
106
75
 
107
- @property
108
- def is_connected(self) -> bool:
109
- """检查当前通道是否有效"""
110
- return (not self._closed and
111
- self.channel is not None and
112
- not self.channel.is_closed and
113
- self.exchange is not None)
114
-
115
- def _update_activity_timestamp(self) -> None:
116
- """更新最后活动时间戳"""
117
- self._last_activity_timestamp = asyncio.get_event_loop().time()
118
-
119
- def _update_message_processed_timestamp(self) -> None:
120
- """更新最后消息处理时间戳"""
121
- self._last_message_processed = asyncio.get_event_loop().time()
122
-
123
- async def _get_channel(self) -> AbstractChannel:
124
- """从通道池获取通道(使用上下文管理器)"""
125
- if not self.connection_pool.channel_pool:
126
- raise Exception("连接池未初始化,请先调用init_pools")
127
-
128
- # 使用async with获取通道,并通过变量返回
129
- async with self.connection_pool.channel_pool.acquire() as channel:
130
- return channel
131
-
132
- async def _check_exchange_exists(self, channel: AbstractChannel) -> bool:
133
- """检查交换机是否存在"""
134
- try:
135
- # 使用被动模式检查交换机
136
- await asyncio.wait_for(
137
- channel.declare_exchange(
138
- name=self.exchange_name,
139
- type=self.exchange_type,
140
- passive=True
141
- ),
142
- timeout=self.rpc_timeout
143
- )
144
- return True
145
- except Exception:
146
- return False
76
+ # 线程安全锁
77
+ self._consume_lock = asyncio.Lock()
78
+ self._connect_lock = asyncio.Lock()
79
+ # 跟踪连接关闭回调(用于后续移除)
80
+ self._conn_close_callback: Optional[Callable] = None
81
+ # 控制重连频率的信号量(限制并发重连数)
82
+ self._reconnect_semaphore = asyncio.Semaphore(1)
83
+ # 固定重连间隔15秒(全局统一)
84
+ self._RECONNECT_INTERVAL = 15
85
+ # 跟踪当前重连任务(避免重复创建)
86
+ self._current_reconnect_task: Optional[asyncio.Task] = None
87
+ # 连接失败计数器(用于告警)
88
+ self._reconnect_fail_count = 0
89
+ # 连接失败告警阈值
90
+ self._reconnect_alert_threshold = 5
147
91
 
148
- async def _check_queue_exists(self, channel: AbstractChannel) -> bool:
149
- """检查队列是否存在"""
150
- if not self.queue_name:
92
+ @property
93
+ async def is_connected(self) -> bool:
94
+ """异步检查客户端连接状态(属性,不可调用)"""
95
+ if self._closed:
151
96
  return False
152
97
  try:
153
- # 使用被动模式检查队列
154
- await asyncio.wait_for(
155
- channel.declare_queue(
156
- name=self.queue_name,
157
- passive=True
158
- ),
159
- timeout=self.rpc_timeout
98
+ # 单通道场景:校验通道+连接+核心资源都有效
99
+ return (
100
+ self._channel and not self._channel.is_closed
101
+ and self._channel_conn and not self._channel_conn.is_closed
102
+ and self._exchange is not None
103
+ and (not self.queue_name or self._queue is not None)
160
104
  )
161
- return True
162
- except Exception:
105
+ except Exception as e:
106
+ logger.warning(f"检查连接状态失败: {str(e)}")
163
107
  return False
164
108
 
165
- async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
166
- """将队列绑定到交换机"""
167
- bind_routing_key = self.routing_key if self.routing_key else '#'
168
-
169
- for attempt in range(MAX_RETRY_COUNT + 1):
170
- try:
171
- await asyncio.wait_for(
172
- queue.bind(
173
- exchange,
174
- routing_key=bind_routing_key
175
- ),
176
- timeout=self.rpc_timeout
177
- )
178
- logger.info(
179
- f"队列 '{self.queue_name}' 已绑定到交换机 '{self.exchange_name}',路由键: {bind_routing_key}")
180
- return True
181
- except Exception as e:
182
- logger.warning(
183
- f"队列绑定失败(第{attempt+1}次尝试): {str(e)}")
184
- if attempt < MAX_RETRY_COUNT:
185
- await asyncio.sleep(1)
186
- return False
187
-
188
- async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
189
- """
190
- 从连接池获取资源并初始化(交换机、队列)
191
- """
192
- logger.info(
193
- f"连接参数 - force_reconnect={force_reconnect}, "
194
- f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
109
+ async def _rebuild_resources(self) -> None:
110
+ """重建交换机/队列等资源(依赖已有的通道)"""
111
+ if not self._channel or self._channel.is_closed:
112
+ raise RuntimeError("无有效通道,无法重建资源")
113
+
114
+ # 1. 声明交换机
115
+ self._exchange = await self._channel.declare_exchange(
116
+ name=self.exchange_name,
117
+ type=self.exchange_type,
118
+ durable=self.durable,
119
+ auto_delete=self.auto_delete,
120
+ passive=not self.create_if_not_exists,
195
121
  )
122
+ logger.info(
123
+ f"交换机重建成功: {self.exchange_name}(类型: {self.exchange_type.value})")
124
+
125
+ # 2. 声明队列(如果配置了队列名)
126
+ if self.queue_name:
127
+ self._queue = await self._channel.declare_queue(
128
+ name=self.queue_name,
129
+ durable=self.durable,
130
+ auto_delete=self.auto_delete,
131
+ passive=not self.create_if_not_exists,
132
+ )
133
+ # 绑定队列到交换机
134
+ await self._queue.bind(
135
+ exchange=self._exchange,
136
+ routing_key=self.routing_key,
137
+ )
138
+ logger.info(
139
+ f"队列重建成功: {self.queue_name} "
140
+ f"(绑定交换机: {self.exchange_name}, routing_key: {self.routing_key})"
141
+ )
196
142
 
197
- # 如果已连接且不强制重连,则直接返回
198
- if self.is_connected and not force_reconnect:
199
- return
200
-
201
- # 取消正在进行的重连任务
202
- if self._reconnect_task and not self._reconnect_task.done():
203
- self._reconnect_task.cancel()
143
+ async def connect(self) -> None:
144
+ if self._closed:
145
+ raise RuntimeError("客户端已关闭,无法重新连接")
204
146
 
205
- # 重置状态
206
- self._exchange_exists = False
207
- self._queue_exists = False
208
- self._queue_bound = False
147
+ async with self._connect_lock:
148
+ # 1. 清理旧连接回调(防止内存泄漏)
149
+ if self._channel_conn and self._conn_close_callback:
150
+ try:
151
+ self._channel_conn.close_callbacks.discard(
152
+ self._conn_close_callback)
153
+ except Exception:
154
+ pass
209
155
 
210
- retries = 0
211
- last_exception = None
156
+ # 2. 清理状态
157
+ self._channel = None
158
+ self._channel_conn = None
159
+ self._exchange = None
160
+ self._queue = None
161
+ self._conn_close_callback = None
212
162
 
213
- while retries < self.max_reconnection_attempts:
214
163
  try:
215
- # 从池获取通道
216
- self.channel = await self._get_channel()
217
- await self.channel.set_qos(prefetch_count=self.prefetch_count)
218
-
219
- # 处理交换机
220
- exchange_exists = await self._check_exchange_exists(self.channel)
221
- if not exchange_exists:
222
- if self.create_if_not_exists:
223
- # 创建交换机
224
- self.exchange = await asyncio.wait_for(
225
- self.channel.declare_exchange(
226
- name=self.exchange_name,
227
- type=self.exchange_type,
228
- durable=self.durable,
229
- auto_delete=self.auto_delete
230
- ),
231
- timeout=self.rpc_timeout
232
- )
233
- logger.info(f"已创建交换机 '{self.exchange_name}'")
234
- else:
235
- raise Exception(
236
- f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
237
- else:
238
- # 获取已有交换机
239
- self.exchange = await self.channel.get_exchange(self.exchange_name)
240
- logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
241
-
242
- # 处理队列
243
- if declare_queue and self.queue_name:
244
- queue_exists = await self._check_queue_exists(self.channel)
245
-
246
- if not queue_exists:
247
- if not self.create_if_not_exists:
248
- raise Exception(
249
- f"队列 '{self.queue_name}' 不存在且不允许自动创建")
250
-
251
- # 创建队列
252
- self.queue = await asyncio.wait_for(
253
- self.channel.declare_queue(
254
- name=self.queue_name,
255
- durable=self.durable,
256
- auto_delete=self.auto_delete,
257
- exclusive=False
258
- ),
259
- timeout=self.rpc_timeout
260
- )
261
- self.actual_queue_name = self.queue_name
262
- logger.info(f"已创建队列 '{self.queue_name}'")
263
- else:
264
- # 获取已有队列
265
- self.queue = await self.channel.get_queue(self.queue_name)
266
- self.actual_queue_name = self.queue_name
267
- logger.info(f"使用已存在的队列 '{self.queue_name}'")
268
-
269
- # 绑定队列到交换机
270
- if self.queue and self.exchange:
271
- bound = await self._bind_queue(self.channel, self.queue, self.exchange)
272
- if not bound:
273
- raise Exception(f"队列 '{self.queue_name}' 绑定到交换机失败")
274
- else:
275
- # 不声明队列时的状态处理
276
- self.queue = None
277
- self.actual_queue_name = None
278
- logger.info(f"跳过队列 '{self.queue_name}' 的声明和绑定")
279
-
280
- # 验证连接状态
281
- if not self.is_connected:
282
- raise Exception("连接验证失败,状态异常")
283
-
284
- # 如果之前在消费,重新开始消费
285
- if self._is_consuming and self.message_handler:
286
- await self.start_consuming()
287
-
288
- # 启动连接监控和保活任务
289
- self._start_monitoring()
290
- self._start_keepalive()
291
-
292
- self._update_activity_timestamp()
293
- logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
294
- return
295
-
164
+ # 3. 获取新通道
165
+ self._channel, self._channel_conn = await self.connection_pool.acquire_channel()
166
+
167
+ # 4. 设置新连接回调(使用 weakref)
168
+ def on_conn_closed(conn: AbstractRobustConnection, exc: Optional[BaseException]):
169
+ # 注意:这里需要访问外部的 self,使用闭包或 weakref
170
+ # 简单起见,这里用闭包,但务必在 self.close 或 self.connect 时清理回调
171
+ logger.warning(f"检测到连接关闭: {exc}")
172
+ if not self._closed:
173
+ asyncio.create_task(self._safe_reconnect())
174
+
175
+ self._conn_close_callback = on_conn_closed
176
+ if self._channel_conn:
177
+ self._channel_conn.close_callbacks.add(
178
+ self._conn_close_callback)
179
+
180
+ # 5. 重建资源
181
+ await self._rebuild_resources()
182
+
183
+ # 重置计数
184
+ self._reconnect_fail_count = 0
185
+ logger.info("客户端连接初始化完成")
296
186
  except Exception as e:
297
- last_exception = e
298
- logger.warning(f"资源初始化失败: {str(e)},重试中...")
299
- # 释放当前通道(放回池并重新获取)
300
- self.channel = None
301
- retries += 1
302
- if retries < self.max_reconnection_attempts:
303
- await asyncio.sleep(self.reconnection_delay)
304
-
305
- logger.error(f"最终初始化失败: {str(last_exception)}")
306
- raise Exception(
307
- f"经过{self.max_reconnection_attempts}次重试后仍无法初始化客户端。最后错误: {str(last_exception)}")
308
-
309
- def _start_monitoring(self) -> None:
310
- """启动连接和消费监控任务"""
311
- if self._closed or (self._monitor_task and not self._monitor_task.done()):
312
- return
313
-
314
- async def monitor():
315
- while not self._closed and self.channel:
316
- try:
317
- # 检查通道状态
318
- if self.channel.is_closed:
319
- logger.warning("检测到通道已关闭,尝试重建")
320
- await self._recreate_channel()
321
- continue
322
-
323
- # 清理已确认的消息记录
324
- if self._tracking_messages:
325
- # 收集所有已确认的消息ID
326
- acked_message_ids = [
327
- msg_id for msg_id, msg_info in self._tracking_messages.items()
328
- if msg_info.get('acked', False)
329
- ]
330
- # 移除已确认的消息
331
- for msg_id in acked_message_ids:
332
- del self._tracking_messages[msg_id]
333
- if acked_message_ids:
334
- logger.info(
335
- f"清理了 {len(acked_message_ids)} 条已确认的消息跟踪记录")
336
-
337
- # 检查消费停滞
338
- if self._is_consuming:
339
- current_time = asyncio.get_event_loop().time()
340
- if current_time - self._last_message_processed > self.consumption_stall_threshold:
341
- if self._tracking_messages:
342
- logger.info(
343
- f"消费停滞,但有 {len(self._tracking_messages)} 个消息正在处理,暂不重启")
344
- else:
345
- # 确实无消息处理,再重启
346
- await self.stop_consuming()
347
- await asyncio.sleep(1)
348
- await self.start_consuming()
349
- except Exception as e:
350
- logger.error(f"监控任务出错: {str(e)}")
351
- await asyncio.sleep(1)
352
-
353
- await asyncio.sleep(60)
354
-
355
- self._monitor_task = asyncio.create_task(monitor())
187
+ logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
188
+ # 失败时也要清理可能产生的残留引用
189
+ if self._channel_conn and self._conn_close_callback:
190
+ try:
191
+ self._channel_conn.close_callbacks.discard(
192
+ self._conn_close_callback)
193
+ except Exception:
194
+ pass
195
+ # 清空状态
196
+ self._channel = None
197
+ self._channel_conn = None
198
+ self._conn_close_callback = None
199
+
200
+ # 触发重连
201
+ if not self._closed:
202
+ asyncio.create_task(self._safe_reconnect())
203
+ raise
204
+
205
+ async def _safe_reconnect(self):
206
+ """安全重连:信号量控制并发+固定15秒间隔"""
207
+ async with self._reconnect_semaphore:
208
+ # 检查是否已有重连任务在运行
209
+ if self._current_reconnect_task and not self._current_reconnect_task.done():
210
+ logger.debug("已有重连任务在运行,跳过重复触发")
211
+ return
356
212
 
357
- async def _recreate_channel(self) -> None:
358
- try:
359
- # 无需手动释放,上下文管理器会自动处理
360
- self.channel = await self._get_channel()
361
- await self.channel.set_qos(prefetch_count=self.prefetch_count)
213
+ if self._closed or await self.is_connected:
214
+ logger.debug("客户端已关闭或已连接,取消重连")
215
+ return
362
216
 
363
- # 重新获取交换机
364
- self.exchange = await self.channel.get_exchange(self.exchange_name)
217
+ # 固定15秒重连间隔
218
+ logger.info(f"将在{self._RECONNECT_INTERVAL}秒后尝试重连...")
219
+ await asyncio.sleep(self._RECONNECT_INTERVAL)
365
220
 
366
- # 重新绑定队列
367
- if self.queue_name:
368
- self.queue = await self.channel.get_queue(self.queue_name)
369
- if self.queue and self.exchange:
370
- await self._bind_queue(self.channel, self.queue, self.exchange)
221
+ if self._closed or await self.is_connected:
222
+ logger.debug("重连等待期间客户端状态变化,取消重连")
223
+ return
371
224
 
372
- # 重新开始消费
373
- if self._is_consuming and self.message_handler:
374
- await self.start_consuming()
225
+ try:
226
+ logger.info("开始重连RabbitMQ客户端...")
227
+ self._current_reconnect_task = asyncio.create_task(
228
+ self.connect())
229
+ await self._current_reconnect_task
230
+ except Exception as e:
231
+ logger.warning(f"重连失败: {str(e)}")
232
+ finally:
233
+ self._current_reconnect_task = None
375
234
 
376
- logger.info("通道已重建并恢复服务")
377
- self._update_activity_timestamp()
378
- except Exception as e:
379
- logger.error(f"通道重建失败: {str(e)},触发重连")
380
- await self.connect(force_reconnect=True)
381
-
382
- def _start_keepalive(self) -> None:
383
- """启动连接保活任务"""
384
- if self._closed or (self._keepalive_task and not self._keepalive_task.done()):
385
- return
386
-
387
- async def keepalive():
388
- while not self._closed and self.is_connected:
389
- current_time = asyncio.get_event_loop().time()
390
- # 检查是否超过指定时间无活动
391
- if current_time - self._last_activity_timestamp > self.connection_pool.heartbeat * 1.5:
392
- logger.info(
393
- f"连接 {self.connection_pool.heartbeat*1.5}s 无活动,执行保活检查")
394
- try:
395
- if self.channel.is_closed:
396
- logger.warning("连接已关闭,触发重连")
397
- await self.connect(force_reconnect=True)
235
+ async def set_message_handler(
236
+ self,
237
+ handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]],
238
+ ) -> None:
239
+ """设置消息处理器(必须是协程函数)"""
240
+ if not asyncio.iscoroutinefunction(handler):
241
+ raise TypeError("消息处理器必须是协程函数(使用 async def 定义)")
242
+
243
+ async with self._consume_lock:
244
+ self._message_handler = handler
245
+ logger.info("消息处理器设置成功")
246
+
247
+ async def start_consuming(self) -> Optional[ConsumerTag]:
248
+ """启动消息消费(支持自动重连)"""
249
+ if self._closed:
250
+ raise RuntimeError("客户端已关闭,无法启动消费")
251
+
252
+ async with self._consume_lock:
253
+ # 1. 校验前置条件
254
+ if not self._message_handler:
255
+ raise RuntimeError("未设置消息处理器,请先调用 set_message_handler()")
256
+ if not await self.is_connected:
257
+ await self.connect()
258
+ if not self._queue:
259
+ raise RuntimeError("未配置队列名或队列未创建,无法启动消费")
260
+
261
+ # 2. 定义消费回调(包含异常处理和重连逻辑)
262
+ async def consume_callback(message: AbstractIncomingMessage):
263
+ try:
264
+ # 解析消息体
265
+ if self.auto_parse_json:
266
+ try:
267
+ body_dict = json.loads(
268
+ message.body.decode("utf-8"))
269
+ msg_obj = MQMsgModel(**body_dict)
270
+ except json.JSONDecodeError as e:
271
+ logger.error(
272
+ f"JSON消息解析失败: {str(e)},消息体: {message.body[:100]}...")
273
+ await message.nack(requeue=False) # 解析失败,不重入队
398
274
  return
399
-
400
- # 轻量级操作保持连接活跃
401
- await asyncio.wait_for(
402
- self.channel.declare_exchange(
403
- name=self.exchange_name,
404
- type=self.exchange_type,
405
- passive=True
406
- ),
407
- timeout=5
275
+ else:
276
+ msg_obj = MQMsgModel(
277
+ body=message.body.decode("utf-8"),
278
+ routing_key=message.routing_key,
279
+ delivery_tag=message.delivery_tag,
408
280
  )
409
- self._update_activity_timestamp()
410
- except Exception as e:
411
- logger.warning(f"保活检查失败: {str(e)},触发重连")
412
- await self.connect(force_reconnect=True)
413
281
 
414
- await asyncio.sleep(self.connection_pool.heartbeat / 2)
282
+ # 调用消息处理器
283
+ await self._message_handler(msg_obj, message)
415
284
 
416
- self._keepalive_task = asyncio.create_task(keepalive())
285
+ # 手动ACK
286
+ await message.ack()
287
+ logger.debug(
288
+ f"消息处理成功,delivery_tag: {message.delivery_tag}")
289
+
290
+ except Exception as e:
291
+ logger.error(
292
+ f"消息处理失败,delivery_tag: {message.delivery_tag}",
293
+ exc_info=True
294
+ )
295
+ # 处理失败逻辑:首次失败重入队,再次失败丢弃
296
+ if message.redelivered:
297
+ logger.warning(
298
+ f"消息已重入队过,本次拒绝入队: {message.delivery_tag}")
299
+ await message.reject(requeue=False)
300
+ else:
301
+ logger.warning(f"消息重入队: {message.delivery_tag}")
302
+ await message.nack(requeue=True)
417
303
 
418
- async def _schedule_reconnect(self) -> None:
419
- """安排重新连接"""
420
- if self._reconnect_task and not self._reconnect_task.done():
421
- return
304
+ # 连接失效则触发重连
305
+ if not await self.is_connected:
306
+ logger.warning("连接已失效,触发客户端重连")
307
+ asyncio.create_task(self._safe_reconnect())
422
308
 
423
- logger.info(f"将在 {self.reconnection_delay} 秒后尝试重新连接...")
309
+ # 3. 启动消费(单通道消费,避免阻塞发布需确保业务回调非阻塞)
310
+ self._consumer_tag = await self._queue.consume(consume_callback)
311
+ logger.info(
312
+ f"开始消费队列: {self._queue.name},consumer_tag: {self._consumer_tag}"
313
+ )
314
+ return self._consumer_tag
424
315
 
425
- async def reconnect():
316
+ async def stop_consuming(self) -> None:
317
+ """停止消息消费(适配 RobustChannel)"""
318
+ async with self._consume_lock:
426
319
  try:
427
- await asyncio.sleep(self.reconnection_delay)
428
- if not self._closed:
429
- await self.connect(force_reconnect=True)
320
+ # 校验核心条件:消费标签、队列、通道均有效
321
+ if self._consumer_tag and self._queue and self._channel and not self._channel.is_closed:
322
+ # 使用队列的 cancel 方法(适配 RobustChannel)
323
+ await self._queue.cancel(self._consumer_tag)
324
+ logger.info(
325
+ f"停止消费成功,consumer_tag: {self._consumer_tag},队列: {self._queue.name}"
326
+ )
327
+ elif self._consumer_tag:
328
+ # 部分资源无效时的日志提示
329
+ if not self._queue:
330
+ logger.warning(
331
+ f"消费标签存在但队列为空,无法取消消费(consumer_tag: {self._consumer_tag})")
332
+ elif not self._channel or self._channel.is_closed:
333
+ logger.warning(
334
+ f"通道已关闭,消费已自动停止(consumer_tag: {self._consumer_tag},队列: {self._queue.name if self._queue else '未知'})"
335
+ )
430
336
  except Exception as e:
431
- logger.error(f"重连任务失败: {str(e)}")
432
- if not self._closed:
433
- await self._schedule_reconnect()
434
-
435
- self._reconnect_task = asyncio.create_task(reconnect())
436
-
437
- async def close(self) -> None:
438
- """关闭客户端并释放资源"""
439
- self._closed = True
440
- self._is_consuming = False
441
-
442
- # 取消所有任务
443
- for task in [self._keepalive_task, self._reconnect_task,
444
- self._consuming_task, self._monitor_task]:
445
- if task and not task.done():
446
- task.cancel()
447
- try:
448
- await task
449
- except asyncio.CancelledError:
450
- pass
451
-
452
- # 重置状态
453
- self.channel = None
454
- self.exchange = None
455
- self.queue = None
456
- self._consumer_tag = None
457
- self._tracking_messages.clear()
458
-
459
- logger.info("RabbitMQ客户端已关闭")
337
+ logger.error(
338
+ f"停止消费者 '{self._queue.name if self._queue else '未知队列'}' 时出错: {str(e)}", exc_info=True
339
+ )
340
+ finally:
341
+ self._consumer_tag = None # 无论成败,清理消费标签
460
342
 
461
343
  async def publish(
462
344
  self,
463
- message_body: Union[str, Dict[str, Any]],
464
- routing_key: Optional[str] = None,
465
- content_type: str = "application/json",
345
+ message_body: Union[str, Dict[str, Any], MQMsgModel],
466
346
  headers: Optional[Dict[str, Any]] = None,
467
- delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
347
+ content_type: str = "application/json",
348
+ delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
349
+ retry_count: int = 3,
468
350
  ) -> None:
469
- """发布消息(从池获取通道,自动重试)"""
470
- if not self.is_connected:
471
- logger.warning("连接已关闭,尝试重连后发布消息")
472
- await self.connect(force_reconnect=True)
473
-
474
- if not self.channel or not self.exchange:
475
- raise Exception("RabbitMQ连接未初始化")
476
-
477
- # 处理消息体
478
- if isinstance(message_body, dict):
479
- message_body_str = json.dumps(message_body, ensure_ascii=False)
480
- if content_type == "text/plain":
481
- content_type = "application/json"
482
- else:
483
- message_body_str = str(message_body)
484
-
485
- # 创建消息对象
486
- message = Message(
487
- body=message_body_str.encode(),
488
- content_type=content_type,
489
- headers=headers or {},
490
- delivery_mode=delivery_mode
491
- )
492
-
493
- # 发布消息(带重试机制)
494
- retry_count = 0
495
- max_retries = 2
496
- while retry_count < max_retries:
497
- try:
498
- # 从池获取新通道用于发布(避免长时间占用消费通道)
499
- async with self.connection_pool.channel_pool.acquire() as publish_channel:
500
- exchange = await publish_channel.get_exchange(self.exchange_name)
501
- confirmed = await exchange.publish(
502
- message,
503
- routing_key=routing_key or self.routing_key or '#',
504
- mandatory=True,
505
- timeout=5.0
506
- )
507
- if not confirmed:
508
- raise Exception("消息未被服务器确认接收")
509
-
510
- self._update_activity_timestamp()
511
- logger.info(f"消息已发布到交换机 '{self.exchange_name}'")
512
- return
513
- except (ConnectionClosed, ChannelInvalidStateError):
514
- retry_count += 1
515
- logger.warning(f"连接已关闭,尝试重连后重新发布 (重试次数: {retry_count})")
516
- await self.connect(force_reconnect=True)
517
- except Exception as e:
518
- retry_count += 1
519
- logger.error(f"消息发布失败 (重试次数: {retry_count}): {str(e)}")
520
- if retry_count < max_retries:
521
- await asyncio.sleep(1)
522
-
523
- raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
524
-
525
- # 以下方法(消息消费相关)逻辑与原有保持一致,仅适配连接池
526
- def set_message_handler(self, handler):
527
- self.message_handler = handler
528
-
529
- async def start_consuming(self) -> ConsumerTag:
530
- if self._is_consuming:
531
- logger.info("已经在消费中,返回现有consumer_tag")
532
- if self._consumer_tag:
533
- return self._consumer_tag
534
- # 如果_is_consuming为True但无consumer_tag,重置状态并重新启动
535
- logger.warning("检测到消费状态异常(无consumer_tag),重置状态后重试")
536
- self._is_consuming = False # 重置状态
537
-
538
- if not self.is_connected:
539
- await self.connect()
540
-
541
- if not self.queue:
542
- raise Exception("队列未初始化,无法开始消费")
543
-
544
- if not self.message_handler:
545
- raise Exception("未设置消息处理函数")
546
-
547
- self._is_consuming = True
548
- logger.info(f"开始消费队列: {self.actual_queue_name}")
351
+ """
352
+ 发布消息(支持自动重试、mandatory路由校验、5秒超时控制)
353
+ :param message_body: 消息体(字符串/字典/MQMsgModel)
354
+ :param headers: 消息头(可选)
355
+ :param content_type: 内容类型(默认application/json)
356
+ :param delivery_mode: 投递模式(PERSISTENT=持久化,TRANSIENT=非持久化)
357
+ :param retry_count: 重试次数(默认3次)
358
+ """
359
+ if self._closed:
360
+ raise RuntimeError("客户端已关闭,无法发布消息")
549
361
 
362
+ # 处理消息体序列化
550
363
  try:
551
- # 调用队列的consume方法,确保获取到consumer_tag
552
- self._consumer_tag = await self.queue.consume(
553
- self._message_wrapper,
554
- no_ack=False # 手动确认消息
555
- )
556
-
557
- # 确保consumer_tag有效
558
- if not self._consumer_tag:
559
- raise Exception("未能获取到有效的consumer_tag")
560
-
561
- logger.info(
562
- f"消费者已启动,队列: {self.actual_queue_name}, tag: {self._consumer_tag}")
563
- return self._consumer_tag
364
+ if isinstance(message_body, MQMsgModel):
365
+ body = json.dumps(message_body.to_dict(),
366
+ ensure_ascii=False).encode("utf-8")
367
+ elif isinstance(message_body, dict):
368
+ body = json.dumps(
369
+ message_body, ensure_ascii=False).encode("utf-8")
370
+ elif isinstance(message_body, str):
371
+ body = message_body.encode("utf-8")
372
+ else:
373
+ raise TypeError(f"不支持的消息体类型: {type(message_body)}")
564
374
  except Exception as e:
565
- self._is_consuming = False # 异常时重置状态
566
- logger.error(f"启动消费失败: {str(e)}", exc_info=True)
375
+ logger.error(f"消息体序列化失败: {str(e)}", exc_info=True)
567
376
  raise
568
377
 
569
- async def _safe_cancel_consumer(self) -> bool:
570
- if not self._consumer_tag or not self.queue or not self.channel:
571
- return True
572
-
573
- try:
574
- await asyncio.wait_for(
575
- self.queue.cancel(self._consumer_tag),
576
- timeout=self.rpc_timeout
577
- )
578
- logger.info(f"消费者 {self._consumer_tag} 已取消")
579
- return True
580
- except Exception as e:
581
- logger.error(f"取消消费者异常: {str(e)}")
582
- return False
583
-
584
- async def stop_consuming(self) -> None:
585
- if not self._is_consuming:
586
- return
587
-
588
- self._is_consuming = False
589
-
590
- if self._consumer_tag and self.queue:
591
- await self._safe_cancel_consumer()
592
-
593
- # 等待所有正在处理的消息完成
594
- if self._tracking_messages:
595
- logger.info(
596
- f"等待 {len(self._tracking_messages)} 个正在处理的消息完成...")
597
- while self._tracking_messages and not self._closed:
598
- await asyncio.sleep(1)
599
-
600
- # 清理状态
601
- self._consumer_tag = None
602
- self._tracking_messages.clear()
603
-
604
- logger.info(f"已停止消费队列: {self.actual_queue_name}")
605
-
606
- async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
607
- try:
608
- body_str = message.body.decode('utf-8')
609
- self._update_activity_timestamp()
610
-
611
- if self.auto_parse_json:
612
- return json.loads(body_str)
613
- return body_str
614
- except json.JSONDecodeError:
615
- logger.warning(f"消息解析JSON失败,返回原始字符串")
616
- return body_str
617
- except Exception as e:
618
- logger.error(f"消息解析出错: {str(e)}")
619
- return message.body.decode('utf-8')
378
+ # 构建消息对象
379
+ message = Message(
380
+ body=body,
381
+ headers=headers or {},
382
+ content_type=content_type,
383
+ delivery_mode=delivery_mode,
384
+ )
620
385
 
621
- async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
622
- if not self.message_handler or not self._is_consuming:
623
- logger.warning("未设置消息处理器或已停止消费")
624
- # await message.ack()
386
+ # 发布重试逻辑
387
+ for retry in range(retry_count):
625
388
  try:
626
- await message.reject(requeue=True)
627
- except Exception as e:
628
- logger.error(f"拒绝消息失败: {e}")
629
- return
630
-
631
- message_id = message.message_id or str(id(message))
632
- if message_id in self._tracking_messages:
633
- logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
634
- await message.ack()
635
- return
636
-
637
- # 记录消息跟踪信息
638
- self._tracking_messages[message_id] = {
639
- 'delivery_tag': message.delivery_tag,
640
- 'acked': False,
641
- 'channel_number': self.channel.number if self.channel else None
642
- }
643
-
644
- try:
645
- logger.info(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
646
-
647
- parsed_data = await self._parse_message(message)
648
- await self.message_handler(MQMsgModel(** parsed_data), message)
649
-
650
- await message.ack()
651
- self._tracking_messages[message_id]['acked'] = True # 更新ACK状态
652
- self._update_activity_timestamp()
653
- self._update_message_processed_timestamp()
654
- logger.info(f"消息 {message_id} 处理完成并确认")
655
-
656
- except Exception as e:
657
- current_headers = message.headers or {}
658
- retry_count = current_headers.get('x-retry-count', 0)
659
- retry_count += 1
389
+ # 确保连接有效
390
+ if not await self.is_connected:
391
+ logger.warning(f"发布消息前连接失效,触发重连(retry: {retry}")
392
+ await self.connect()
393
+
394
+ # 核心:发布消息(mandatory=True 确保路由有效,timeout=5s 避免阻塞)
395
+ publish_result = await self._exchange.publish(
396
+ message=message,
397
+ routing_key=self.routing_key or self.queue_name or "#",
398
+ mandatory=True,
399
+ timeout=5.0
400
+ )
660
401
 
661
- logger.error(
662
- f"消息 {message_id} 处理出错(第{retry_count}次重试): {str(e)}",
663
- exc_info=True
664
- )
402
+ # 处理 mandatory 未路由场景
403
+ if publish_result is None:
404
+ raise RuntimeError(
405
+ f"消息未找到匹配的队列(routing_key: {self.routing_key}),mandatory=True 触发失败"
406
+ )
665
407
 
666
- if retry_count >= MAX_RETRY_COUNT:
667
- logger.error(
668
- f"消息 {message_id} 已达到最大重试次数({MAX_RETRY_COUNT}次),标记为失败")
669
- await message.ack()
670
- self._tracking_messages[message_id]['acked'] = True # 更新ACK状态
671
- self._update_activity_timestamp()
408
+ logger.info(
409
+ f"消息发布成功(retry: {retry}),routing_key: {self.routing_key},"
410
+ f"delivery_mode: {delivery_mode.value},mandatory: True,timeout: 5.0s"
411
+ )
672
412
  return
413
+ except asyncio.TimeoutError:
414
+ logger.error(
415
+ f"消息发布超时(retry: {retry}/{retry_count-1}),超时时间: 5.0s"
416
+ )
417
+ except RuntimeError as e:
418
+ logger.error(
419
+ f"消息发布业务失败(retry: {retry}/{retry_count-1}): {str(e)}"
420
+ )
421
+ except Exception as e:
422
+ logger.error(
423
+ f"消息发布失败(retry: {retry}/{retry_count-1}): {str(e)}",
424
+ exc_info=True
425
+ )
426
+ # 清理失效状态,下次重试重连
427
+ self._exchange = None
428
+ # 指数退避重试间隔
429
+ await asyncio.sleep(0.5 * (2 ** retry))
430
+
431
+ # 所有重试失败
432
+ raise RuntimeError(
433
+ f"消息发布失败(已重试{retry_count}次),routing_key: {self.routing_key},"
434
+ f"mandatory: True,timeout: 5.0s"
435
+ )
673
436
 
674
- new_headers = current_headers.copy()
675
- new_headers['x-retry-count'] = retry_count
676
-
677
- new_message = Message(
678
- body=message.body,
679
- content_type=message.content_type,
680
- headers=new_headers,
681
- delivery_mode=message.delivery_mode
682
- )
683
-
684
- await message.reject(requeue=False)
685
- self._tracking_messages[message_id]['acked'] = True # 拒绝也算处理完成
437
+ async def close(self) -> None:
438
+ """关闭客户端(移除回调+释放资源)"""
439
+ self._closed = True
440
+ logger.info("开始关闭RabbitMQ客户端...")
686
441
 
687
- if self.exchange:
688
- await self.exchange.publish(
689
- new_message,
690
- routing_key=self.routing_key or '#',
691
- mandatory=True,
692
- timeout=5.0
693
- )
694
- self._update_activity_timestamp()
695
- logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
696
- finally:
697
- if message_id in self._tracking_messages:
698
- del self._tracking_messages[message_id]
699
-
700
- async def __aenter__(self):
701
- await self.connect()
702
- return self
703
-
704
- async def __aexit__(self, exc_type, exc, tb):
705
- await self.close()
442
+ # 停止重连任务
443
+ if self._current_reconnect_task and not self._current_reconnect_task.done():
444
+ self._current_reconnect_task.cancel()
445
+ try:
446
+ await self._current_reconnect_task
447
+ except asyncio.CancelledError:
448
+ logger.debug("重连任务已取消")
449
+
450
+ # 1. 停止消费
451
+ await self.stop_consuming()
452
+
453
+ # 2. 清理回调+状态(单通道无需归还,连接池统一管理)
454
+ async with self._connect_lock:
455
+ if self._conn_close_callback and self._channel_conn:
456
+ self._channel_conn.close_callbacks.discard(
457
+ self._conn_close_callback)
458
+ self._channel = None
459
+ self._channel_conn = None
460
+ self._exchange = None
461
+ self._queue = None
462
+ self._message_handler = None
463
+
464
+ logger.info("RabbitMQ客户端已完全关闭")