sycommon-python-lib 0.1.29__py3-none-any.whl → 0.1.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/models/mqlistener_config.py +1 -0
- sycommon/rabbitmq/rabbitmq_client.py +205 -588
- sycommon/rabbitmq/rabbitmq_pool.py +141 -65
- sycommon/rabbitmq/rabbitmq_service.py +269 -145
- sycommon/services.py +64 -22
- sycommon/synacos/example.py +153 -0
- sycommon/synacos/example2.py +129 -0
- sycommon/synacos/feign.py +51 -436
- sycommon/synacos/feign_client.py +317 -0
- sycommon/synacos/nacos_service.py +1 -1
- sycommon/synacos/param.py +75 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.40.dist-info}/METADATA +1 -1
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.40.dist-info}/RECORD +16 -12
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.40.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.40.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.29.dist-info → sycommon_python_lib-0.1.40.dist-info}/top_level.txt +0 -0
|
@@ -1,19 +1,19 @@
|
|
|
1
|
+
from aio_pika import Channel
|
|
2
|
+
from typing import Optional
|
|
1
3
|
import asyncio
|
|
2
4
|
import json
|
|
3
|
-
from typing import Callable, Coroutine,
|
|
5
|
+
from typing import Callable, Coroutine, Dict, Any, Union
|
|
4
6
|
from aio_pika import Message, DeliveryMode, ExchangeType
|
|
5
7
|
from aio_pika.abc import (
|
|
6
|
-
AbstractChannel,
|
|
7
8
|
AbstractExchange,
|
|
8
9
|
AbstractQueue,
|
|
9
10
|
AbstractIncomingMessage,
|
|
10
11
|
ConsumerTag,
|
|
11
12
|
)
|
|
12
|
-
from
|
|
13
|
-
|
|
13
|
+
from sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
|
|
14
14
|
from sycommon.logging.kafka_log import SYLogger
|
|
15
15
|
from sycommon.models.mqmsg_model import MQMsgModel
|
|
16
|
-
|
|
16
|
+
|
|
17
17
|
|
|
18
18
|
# 最大重试次数限制
|
|
19
19
|
MAX_RETRY_COUNT = 3
|
|
@@ -23,8 +23,8 @@ logger = SYLogger
|
|
|
23
23
|
|
|
24
24
|
class RabbitMQClient:
|
|
25
25
|
"""
|
|
26
|
-
RabbitMQ
|
|
27
|
-
|
|
26
|
+
RabbitMQ客户端(基于连接池),支持集群、自动重连、消息发布/消费
|
|
27
|
+
依赖 aio_pika 的内置重连机制,移除手动重连逻辑
|
|
28
28
|
"""
|
|
29
29
|
|
|
30
30
|
def __init__(
|
|
@@ -38,646 +38,263 @@ class RabbitMQClient:
|
|
|
38
38
|
auto_delete: bool = False,
|
|
39
39
|
auto_parse_json: bool = True,
|
|
40
40
|
create_if_not_exists: bool = True,
|
|
41
|
-
connection_timeout: int = 10,
|
|
42
41
|
rpc_timeout: int = 10,
|
|
43
|
-
reconnection_delay: int = 1,
|
|
44
|
-
max_reconnection_attempts: int = 5,
|
|
45
42
|
prefetch_count: int = 2,
|
|
46
|
-
consumption_stall_threshold: int =
|
|
43
|
+
consumption_stall_threshold: int = 60,
|
|
47
44
|
):
|
|
48
|
-
"""
|
|
49
|
-
初始化RabbitMQ客户端(依赖连接池)
|
|
50
|
-
|
|
51
|
-
:param connection_pool: 连接池实例
|
|
52
|
-
"""
|
|
53
|
-
# 连接池依赖
|
|
54
45
|
self.connection_pool = connection_pool
|
|
46
|
+
if not self.connection_pool._initialized:
|
|
47
|
+
raise RuntimeError("连接池未初始化,请先调用 connection_pool.init_pools()")
|
|
55
48
|
|
|
56
|
-
#
|
|
49
|
+
# 交换机配置
|
|
57
50
|
self.exchange_name = exchange_name
|
|
58
|
-
self.exchange_type = ExchangeType(exchange_type)
|
|
51
|
+
self.exchange_type = ExchangeType(exchange_type.lower())
|
|
52
|
+
# 队列配置
|
|
59
53
|
self.queue_name = queue_name
|
|
60
54
|
self.routing_key = routing_key
|
|
61
55
|
self.durable = durable
|
|
62
56
|
self.auto_delete = auto_delete
|
|
63
|
-
|
|
64
|
-
# 行为控制参数
|
|
65
57
|
self.auto_parse_json = auto_parse_json
|
|
66
58
|
self.create_if_not_exists = create_if_not_exists
|
|
67
|
-
|
|
59
|
+
|
|
60
|
+
# 运行时配置
|
|
68
61
|
self.rpc_timeout = rpc_timeout
|
|
69
62
|
self.prefetch_count = prefetch_count
|
|
70
|
-
|
|
71
|
-
# 重连参数
|
|
72
|
-
self.reconnection_delay = reconnection_delay
|
|
73
|
-
self.max_reconnection_attempts = max_reconnection_attempts
|
|
74
|
-
|
|
75
|
-
# 消息处理参数
|
|
76
63
|
self.consumption_stall_threshold = consumption_stall_threshold
|
|
77
64
|
|
|
78
|
-
#
|
|
79
|
-
self.
|
|
80
|
-
self.
|
|
81
|
-
self.
|
|
82
|
-
|
|
83
|
-
# 状态跟踪
|
|
84
|
-
self.actual_queue_name: Optional[str] = None
|
|
85
|
-
self._exchange_exists = False
|
|
86
|
-
self._queue_exists = False
|
|
87
|
-
self._queue_bound = False
|
|
88
|
-
self._is_consuming = False
|
|
89
|
-
self._closed = False
|
|
65
|
+
# 内部状态
|
|
66
|
+
self._channel: Optional[Channel] = None
|
|
67
|
+
self._exchange: Optional[AbstractExchange] = None
|
|
68
|
+
self._queue: Optional[AbstractQueue] = None
|
|
90
69
|
self._consumer_tag: Optional[ConsumerTag] = None
|
|
91
|
-
self.
|
|
92
|
-
self.
|
|
93
|
-
|
|
94
|
-
# 任务和处理器
|
|
95
|
-
self.message_handler: Optional[Callable[
|
|
96
|
-
[Union[Dict[str, Any], str], AbstractIncomingMessage],
|
|
97
|
-
Coroutine[Any, Any, None]
|
|
98
|
-
]] = None
|
|
99
|
-
self._consuming_task: Optional[asyncio.Task] = None
|
|
100
|
-
self._reconnect_task: Optional[asyncio.Task] = None
|
|
101
|
-
self._keepalive_task: Optional[asyncio.Task] = None
|
|
102
|
-
self._monitor_task: Optional[asyncio.Task] = None
|
|
103
|
-
|
|
104
|
-
# 消息处理跟踪
|
|
105
|
-
self._processing_message_ids: Set[str] = set()
|
|
70
|
+
self._message_handler: Optional[Callable] = None
|
|
71
|
+
self._closed = False
|
|
106
72
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
return (not self._closed and
|
|
111
|
-
self.channel is not None and
|
|
112
|
-
not self.channel.is_closed and
|
|
113
|
-
self.exchange is not None)
|
|
114
|
-
|
|
115
|
-
def _update_activity_timestamp(self) -> None:
|
|
116
|
-
"""更新最后活动时间戳"""
|
|
117
|
-
self._last_activity_timestamp = asyncio.get_event_loop().time()
|
|
118
|
-
|
|
119
|
-
def _update_message_processed_timestamp(self) -> None:
|
|
120
|
-
"""更新最后消息处理时间戳"""
|
|
121
|
-
self._last_message_processed = asyncio.get_event_loop().time()
|
|
122
|
-
|
|
123
|
-
async def _get_channel(self) -> AbstractChannel:
|
|
124
|
-
"""从通道池获取通道(使用上下文管理器)"""
|
|
125
|
-
if not self.connection_pool.channel_pool:
|
|
126
|
-
raise Exception("连接池未初始化,请先调用init_pools")
|
|
127
|
-
|
|
128
|
-
# 使用async with获取通道,并通过变量返回
|
|
129
|
-
async with self.connection_pool.channel_pool.acquire() as channel:
|
|
130
|
-
return channel
|
|
131
|
-
|
|
132
|
-
async def _check_exchange_exists(self, channel: AbstractChannel) -> bool:
|
|
133
|
-
"""检查交换机是否存在"""
|
|
134
|
-
try:
|
|
135
|
-
# 使用被动模式检查交换机
|
|
136
|
-
await asyncio.wait_for(
|
|
137
|
-
channel.declare_exchange(
|
|
138
|
-
name=self.exchange_name,
|
|
139
|
-
type=self.exchange_type,
|
|
140
|
-
passive=True
|
|
141
|
-
),
|
|
142
|
-
timeout=self.rpc_timeout
|
|
143
|
-
)
|
|
144
|
-
return True
|
|
145
|
-
except Exception:
|
|
146
|
-
return False
|
|
73
|
+
# 细粒度锁
|
|
74
|
+
self._consume_lock = asyncio.Lock()
|
|
75
|
+
self._connect_lock = asyncio.Lock()
|
|
147
76
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
77
|
+
@property
|
|
78
|
+
async def is_connected(self) -> bool:
|
|
79
|
+
"""异步检查客户端连接状态(属性,不是函数)"""
|
|
80
|
+
if self._closed:
|
|
151
81
|
return False
|
|
152
82
|
try:
|
|
153
|
-
#
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
),
|
|
159
|
-
timeout=self.rpc_timeout
|
|
160
|
-
)
|
|
161
|
-
return True
|
|
162
|
-
except Exception:
|
|
163
|
-
return False
|
|
164
|
-
|
|
165
|
-
async def _bind_queue(self, channel: AbstractChannel, queue: AbstractQueue, exchange: AbstractExchange) -> bool:
|
|
166
|
-
"""将队列绑定到交换机"""
|
|
167
|
-
bind_routing_key = self.routing_key if self.routing_key else '#'
|
|
168
|
-
|
|
169
|
-
for attempt in range(MAX_RETRY_COUNT + 1):
|
|
170
|
-
try:
|
|
171
|
-
await asyncio.wait_for(
|
|
172
|
-
queue.bind(
|
|
173
|
-
exchange,
|
|
174
|
-
routing_key=bind_routing_key
|
|
175
|
-
),
|
|
176
|
-
timeout=self.rpc_timeout
|
|
177
|
-
)
|
|
178
|
-
logger.info(
|
|
179
|
-
f"队列 '{self.queue_name}' 已绑定到交换机 '{self.exchange_name}',路由键: {bind_routing_key}")
|
|
83
|
+
# 检查通道是否有效
|
|
84
|
+
if self._channel and not self._channel.is_closed:
|
|
85
|
+
# 检查交换机和队列(如果需要)
|
|
86
|
+
if self.create_if_not_exists and self.queue_name:
|
|
87
|
+
return bool(self._exchange and self._queue)
|
|
180
88
|
return True
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
await asyncio.sleep(1)
|
|
186
|
-
return False
|
|
187
|
-
|
|
188
|
-
async def connect(self, force_reconnect: bool = False, declare_queue: bool = True) -> None:
|
|
189
|
-
"""
|
|
190
|
-
从连接池获取资源并初始化(交换机、队列)
|
|
191
|
-
"""
|
|
192
|
-
logger.debug(
|
|
193
|
-
f"连接参数 - force_reconnect={force_reconnect}, "
|
|
194
|
-
f"declare_queue={declare_queue}, create_if_not_exists={self.create_if_not_exists}"
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
# 如果已连接且不强制重连,则直接返回
|
|
198
|
-
if self.is_connected and not force_reconnect:
|
|
199
|
-
return
|
|
200
|
-
|
|
201
|
-
# 取消正在进行的重连任务
|
|
202
|
-
if self._reconnect_task and not self._reconnect_task.done():
|
|
203
|
-
self._reconnect_task.cancel()
|
|
89
|
+
return False
|
|
90
|
+
except Exception as e:
|
|
91
|
+
logger.warning(f"检查连接状态失败: {str(e)}")
|
|
92
|
+
return False
|
|
204
93
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
self.
|
|
208
|
-
|
|
94
|
+
async def _get_connection_resources(self) -> tuple[Optional[Channel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
95
|
+
"""原子获取连接资源(通道、交换机、队列)"""
|
|
96
|
+
async with self._connect_lock:
|
|
97
|
+
return self._channel, self._exchange, self._queue
|
|
209
98
|
|
|
210
|
-
|
|
211
|
-
|
|
99
|
+
async def connect(self) -> None:
|
|
100
|
+
"""建立连接并初始化交换机/队列(使用新的通道获取方式)"""
|
|
101
|
+
if self._closed:
|
|
102
|
+
raise RuntimeError("客户端已关闭,无法重新连接")
|
|
212
103
|
|
|
213
|
-
|
|
104
|
+
async with self._connect_lock:
|
|
214
105
|
try:
|
|
215
|
-
#
|
|
216
|
-
self.
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
durable=self.durable,
|
|
229
|
-
auto_delete=self.auto_delete
|
|
230
|
-
),
|
|
231
|
-
timeout=self.rpc_timeout
|
|
232
|
-
)
|
|
233
|
-
logger.info(f"已创建交换机 '{self.exchange_name}'")
|
|
234
|
-
else:
|
|
235
|
-
raise Exception(
|
|
236
|
-
f"交换机 '{self.exchange_name}' 不存在且不允许自动创建")
|
|
237
|
-
else:
|
|
238
|
-
# 获取已有交换机
|
|
239
|
-
self.exchange = await self.channel.get_exchange(self.exchange_name)
|
|
240
|
-
logger.info(f"使用已存在的交换机 '{self.exchange_name}'")
|
|
241
|
-
|
|
242
|
-
# 处理队列
|
|
243
|
-
if declare_queue and self.queue_name:
|
|
244
|
-
queue_exists = await self._check_queue_exists(self.channel)
|
|
245
|
-
|
|
246
|
-
if not queue_exists:
|
|
247
|
-
if not self.create_if_not_exists:
|
|
248
|
-
raise Exception(
|
|
249
|
-
f"队列 '{self.queue_name}' 不存在且不允许自动创建")
|
|
250
|
-
|
|
251
|
-
# 创建队列
|
|
252
|
-
self.queue = await asyncio.wait_for(
|
|
253
|
-
self.channel.declare_queue(
|
|
254
|
-
name=self.queue_name,
|
|
255
|
-
durable=self.durable,
|
|
256
|
-
auto_delete=self.auto_delete,
|
|
257
|
-
exclusive=False
|
|
258
|
-
),
|
|
259
|
-
timeout=self.rpc_timeout
|
|
260
|
-
)
|
|
261
|
-
self.actual_queue_name = self.queue_name
|
|
262
|
-
logger.info(f"已创建队列 '{self.queue_name}'")
|
|
263
|
-
else:
|
|
264
|
-
# 获取已有队列
|
|
265
|
-
self.queue = await self.channel.get_queue(self.queue_name)
|
|
266
|
-
self.actual_queue_name = self.queue_name
|
|
267
|
-
logger.info(f"使用已存在的队列 '{self.queue_name}'")
|
|
106
|
+
# 关键修复:使用连接池的 acquire_channel 方法获取通道
|
|
107
|
+
self._channel = await self.connection_pool.acquire_channel()
|
|
108
|
+
# 设置预取计数
|
|
109
|
+
await self._channel.set_qos(prefetch_count=self.prefetch_count)
|
|
110
|
+
|
|
111
|
+
# 声明交换机
|
|
112
|
+
self._exchange = await self._channel.declare_exchange(
|
|
113
|
+
name=self.exchange_name,
|
|
114
|
+
type=self.exchange_type,
|
|
115
|
+
durable=self.durable,
|
|
116
|
+
auto_delete=self.auto_delete,
|
|
117
|
+
passive=not self.create_if_not_exists # 不创建时使用passive模式检查
|
|
118
|
+
)
|
|
268
119
|
|
|
120
|
+
# 声明队列(如果配置了队列名)
|
|
121
|
+
if self.queue_name:
|
|
122
|
+
self._queue = await self._channel.declare_queue(
|
|
123
|
+
name=self.queue_name,
|
|
124
|
+
durable=self.durable,
|
|
125
|
+
auto_delete=self.auto_delete,
|
|
126
|
+
passive=not self.create_if_not_exists
|
|
127
|
+
)
|
|
269
128
|
# 绑定队列到交换机
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
129
|
+
await self._queue.bind(
|
|
130
|
+
exchange=self._exchange,
|
|
131
|
+
routing_key=self.routing_key or self.queue_name
|
|
132
|
+
)
|
|
133
|
+
logger.info(
|
|
134
|
+
f"队列 '{self.queue_name}' 已声明并绑定到交换机 '{self.exchange_name}' "
|
|
135
|
+
f"(exchange_type: {self.exchange_type}, routing_key: {self.routing_key})"
|
|
136
|
+
)
|
|
274
137
|
else:
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
self.actual_queue_name = None
|
|
278
|
-
logger.debug(f"跳过队列 '{self.queue_name}' 的声明和绑定")
|
|
279
|
-
|
|
280
|
-
# 验证连接状态
|
|
281
|
-
if not self.is_connected:
|
|
282
|
-
raise Exception("连接验证失败,状态异常")
|
|
283
|
-
|
|
284
|
-
# 如果之前在消费,重新开始消费
|
|
285
|
-
if self._is_consuming and self.message_handler:
|
|
286
|
-
await self.start_consuming()
|
|
287
|
-
|
|
288
|
-
# 启动连接监控和保活任务
|
|
289
|
-
self._start_monitoring()
|
|
290
|
-
self._start_keepalive()
|
|
291
|
-
|
|
292
|
-
self._update_activity_timestamp()
|
|
293
|
-
logger.info(f"RabbitMQ客户端初始化成功 (队列: {self.actual_queue_name})")
|
|
294
|
-
return
|
|
138
|
+
logger.info(
|
|
139
|
+
f"未配置队列名,仅初始化交换机 '{self.exchange_name}' (exchange_type: {self.exchange_type})")
|
|
295
140
|
|
|
141
|
+
logger.info(f"RabbitMQ客户端连接成功(exchange: {self.exchange_name})")
|
|
296
142
|
except Exception as e:
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
143
|
+
logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
|
|
144
|
+
# 清理异常状态
|
|
145
|
+
if self._channel:
|
|
146
|
+
await self.connection_pool.release_channel(self._channel)
|
|
147
|
+
self._channel = None
|
|
148
|
+
self._exchange = None
|
|
149
|
+
self._queue = None
|
|
150
|
+
raise
|
|
151
|
+
|
|
152
|
+
async def set_message_handler(self, handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]):
|
|
153
|
+
"""设置消息处理器(消费消息时使用)"""
|
|
154
|
+
async with self._consume_lock:
|
|
155
|
+
self._message_handler = handler
|
|
156
|
+
logger.info("消息处理器已设置")
|
|
157
|
+
|
|
158
|
+
async def start_consuming(self) -> Optional[ConsumerTag]:
|
|
159
|
+
"""启动消费(返回消费者标签)"""
|
|
160
|
+
if self._closed:
|
|
161
|
+
logger.warning("客户端已关闭,无法启动消费")
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
async with self._consume_lock:
|
|
165
|
+
# 检查前置条件
|
|
166
|
+
if not self._message_handler:
|
|
167
|
+
raise RuntimeError("未设置消息处理器,请先调用 set_message_handler")
|
|
168
|
+
|
|
169
|
+
# 确保连接已建立
|
|
170
|
+
if not await self.is_connected:
|
|
171
|
+
await self.connect()
|
|
172
|
+
|
|
173
|
+
# 确保队列已初始化
|
|
174
|
+
_, _, queue = await self._get_connection_resources()
|
|
175
|
+
if not queue:
|
|
176
|
+
raise RuntimeError("队列未初始化,无法启动消费")
|
|
177
|
+
|
|
178
|
+
# 定义消费回调
|
|
179
|
+
async def consume_callback(message: AbstractIncomingMessage):
|
|
316
180
|
try:
|
|
317
|
-
#
|
|
318
|
-
if self.
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
if self._is_consuming:
|
|
325
|
-
current_time = asyncio.get_event_loop().time()
|
|
326
|
-
if current_time - self._last_message_processed > self.consumption_stall_threshold:
|
|
327
|
-
if self._processing_message_ids:
|
|
328
|
-
logger.warning(
|
|
329
|
-
f"消费停滞,但有 {len(self._processing_message_ids)} 个消息正在处理,暂不重启")
|
|
330
|
-
else:
|
|
331
|
-
# 确实无消息处理,再重启
|
|
332
|
-
await self.stop_consuming()
|
|
333
|
-
await asyncio.sleep(1)
|
|
334
|
-
await self.start_consuming()
|
|
335
|
-
except Exception as e:
|
|
336
|
-
logger.error(f"监控任务出错: {str(e)}")
|
|
337
|
-
await asyncio.sleep(1)
|
|
338
|
-
|
|
339
|
-
await asyncio.sleep(30)
|
|
340
|
-
|
|
341
|
-
self._monitor_task = asyncio.create_task(monitor())
|
|
342
|
-
|
|
343
|
-
async def _recreate_channel(self) -> None:
|
|
344
|
-
try:
|
|
345
|
-
# 无需手动释放,上下文管理器会自动处理
|
|
346
|
-
self.channel = await self._get_channel()
|
|
347
|
-
await self.channel.set_qos(prefetch_count=self.prefetch_count)
|
|
348
|
-
|
|
349
|
-
# 重新获取交换机
|
|
350
|
-
self.exchange = await self.channel.get_exchange(self.exchange_name)
|
|
351
|
-
|
|
352
|
-
# 重新绑定队列
|
|
353
|
-
if self.queue_name:
|
|
354
|
-
self.queue = await self.channel.get_queue(self.queue_name)
|
|
355
|
-
if self.queue and self.exchange:
|
|
356
|
-
await self._bind_queue(self.channel, self.queue, self.exchange)
|
|
357
|
-
|
|
358
|
-
# 重新开始消费
|
|
359
|
-
if self._is_consuming and self.message_handler:
|
|
360
|
-
await self.start_consuming()
|
|
181
|
+
# 解析消息
|
|
182
|
+
if self.auto_parse_json:
|
|
183
|
+
body = json.loads(message.body.decode('utf-8'))
|
|
184
|
+
parsed_data = MQMsgModel(**body)
|
|
185
|
+
else:
|
|
186
|
+
parsed_data = MQMsgModel(
|
|
187
|
+
msg=message.body.decode('utf-8'))
|
|
361
188
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
189
|
+
# 调用处理器
|
|
190
|
+
await self._message_handler(parsed_data, message)
|
|
191
|
+
# 手动确认消息
|
|
192
|
+
await message.ack()
|
|
193
|
+
except Exception as e:
|
|
194
|
+
logger.error(
|
|
195
|
+
f"处理消息失败 (delivery_tag: {message.delivery_tag}): {str(e)}", exc_info=True)
|
|
196
|
+
# 消费失败时重新入队(最多重试3次)
|
|
197
|
+
if message.redelivered:
|
|
198
|
+
logger.warning(
|
|
199
|
+
f"消息已重试过,拒绝入队 (delivery_tag: {message.delivery_tag})")
|
|
200
|
+
await message.reject(requeue=False)
|
|
201
|
+
else:
|
|
202
|
+
await message.nack(requeue=True)
|
|
367
203
|
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
204
|
+
# 启动消费
|
|
205
|
+
self._consumer_tag = await queue.consume(consume_callback)
|
|
206
|
+
logger.info(
|
|
207
|
+
f"开始消费队列 '{queue.name}',consumer_tag: {self._consumer_tag}")
|
|
208
|
+
return self._consumer_tag
|
|
372
209
|
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
f"连接 {self.connection_pool.heartbeat*1.5}s 无活动,执行保活检查")
|
|
210
|
+
async def stop_consuming(self) -> None:
|
|
211
|
+
"""停止消费"""
|
|
212
|
+
async with self._consume_lock:
|
|
213
|
+
if self._consumer_tag and not self._closed:
|
|
214
|
+
_, _, queue = await self._get_connection_resources()
|
|
215
|
+
if queue and not queue.is_closed:
|
|
380
216
|
try:
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
await self.connect(force_reconnect=True)
|
|
384
|
-
return
|
|
385
|
-
|
|
386
|
-
# 轻量级操作保持连接活跃
|
|
387
|
-
await asyncio.wait_for(
|
|
388
|
-
self.channel.declare_exchange(
|
|
389
|
-
name=self.exchange_name,
|
|
390
|
-
type=self.exchange_type,
|
|
391
|
-
passive=True
|
|
392
|
-
),
|
|
393
|
-
timeout=5
|
|
394
|
-
)
|
|
395
|
-
self._update_activity_timestamp()
|
|
217
|
+
await queue.cancel(self._consumer_tag)
|
|
218
|
+
logger.info(f"停止消费,consumer_tag: {self._consumer_tag}")
|
|
396
219
|
except Exception as e:
|
|
397
|
-
logger.
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
await asyncio.sleep(self.connection_pool.heartbeat / 2)
|
|
401
|
-
|
|
402
|
-
self._keepalive_task = asyncio.create_task(keepalive())
|
|
403
|
-
|
|
404
|
-
async def _schedule_reconnect(self) -> None:
|
|
405
|
-
"""安排重新连接"""
|
|
406
|
-
if self._reconnect_task and not self._reconnect_task.done():
|
|
407
|
-
return
|
|
408
|
-
|
|
409
|
-
logger.info(f"将在 {self.reconnection_delay} 秒后尝试重新连接...")
|
|
410
|
-
|
|
411
|
-
async def reconnect():
|
|
412
|
-
try:
|
|
413
|
-
await asyncio.sleep(self.reconnection_delay)
|
|
414
|
-
if not self._closed:
|
|
415
|
-
await self.connect(force_reconnect=True)
|
|
416
|
-
except Exception as e:
|
|
417
|
-
logger.error(f"重连任务失败: {str(e)}")
|
|
418
|
-
if not self._closed:
|
|
419
|
-
await self._schedule_reconnect()
|
|
420
|
-
|
|
421
|
-
self._reconnect_task = asyncio.create_task(reconnect())
|
|
422
|
-
|
|
423
|
-
async def close(self) -> None:
|
|
424
|
-
"""关闭客户端并释放资源"""
|
|
425
|
-
self._closed = True
|
|
426
|
-
self._is_consuming = False
|
|
427
|
-
|
|
428
|
-
# 取消所有任务
|
|
429
|
-
for task in [self._keepalive_task, self._reconnect_task,
|
|
430
|
-
self._consuming_task, self._monitor_task]:
|
|
431
|
-
if task and not task.done():
|
|
432
|
-
task.cancel()
|
|
433
|
-
try:
|
|
434
|
-
await task
|
|
435
|
-
except asyncio.CancelledError:
|
|
436
|
-
pass
|
|
437
|
-
|
|
438
|
-
# 重置状态
|
|
439
|
-
self.channel = None
|
|
440
|
-
self.exchange = None
|
|
441
|
-
self.queue = None
|
|
442
|
-
self._consumer_tag = None
|
|
443
|
-
self._processing_message_ids.clear()
|
|
444
|
-
|
|
445
|
-
logger.info("RabbitMQ客户端已关闭")
|
|
220
|
+
logger.error(f"停止消费失败: {str(e)}")
|
|
221
|
+
self._consumer_tag = None
|
|
446
222
|
|
|
447
223
|
async def publish(
|
|
448
224
|
self,
|
|
449
|
-
message_body: Union[str, Dict[str, Any]],
|
|
450
|
-
routing_key: Optional[str] = None,
|
|
451
|
-
content_type: str = "application/json",
|
|
225
|
+
message_body: Union[str, Dict[str, Any], MQMsgModel],
|
|
452
226
|
headers: Optional[Dict[str, Any]] = None,
|
|
227
|
+
content_type: str = "application/json",
|
|
453
228
|
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
|
|
454
229
|
) -> None:
|
|
455
|
-
"""
|
|
456
|
-
if
|
|
457
|
-
|
|
458
|
-
await self.connect(force_reconnect=True)
|
|
230
|
+
"""发布消息(修复 is_closed 问题)"""
|
|
231
|
+
if self._closed:
|
|
232
|
+
raise RuntimeError("客户端已关闭,无法发布消息")
|
|
459
233
|
|
|
460
|
-
|
|
461
|
-
|
|
234
|
+
# 确保连接已建立
|
|
235
|
+
if not await self.is_connected:
|
|
236
|
+
await self.connect()
|
|
462
237
|
|
|
463
238
|
# 处理消息体
|
|
464
|
-
if isinstance(message_body,
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
239
|
+
if isinstance(message_body, MQMsgModel):
|
|
240
|
+
body = json.dumps(message_body.__dict__,
|
|
241
|
+
ensure_ascii=False).encode('utf-8')
|
|
242
|
+
elif isinstance(message_body, dict):
|
|
243
|
+
body = json.dumps(message_body, ensure_ascii=False).encode('utf-8')
|
|
468
244
|
else:
|
|
469
|
-
|
|
245
|
+
body = str(message_body).encode('utf-8')
|
|
470
246
|
|
|
471
|
-
#
|
|
247
|
+
# 创建消息
|
|
472
248
|
message = Message(
|
|
473
|
-
body=
|
|
474
|
-
content_type=content_type,
|
|
249
|
+
body=body,
|
|
475
250
|
headers=headers or {},
|
|
251
|
+
content_type=content_type,
|
|
476
252
|
delivery_mode=delivery_mode
|
|
477
253
|
)
|
|
478
254
|
|
|
479
|
-
#
|
|
480
|
-
retry_count = 0
|
|
481
|
-
max_retries = 2
|
|
482
|
-
while retry_count < max_retries:
|
|
483
|
-
try:
|
|
484
|
-
# 从池获取新通道用于发布(避免长时间占用消费通道)
|
|
485
|
-
async with self.connection_pool.channel_pool.acquire() as publish_channel:
|
|
486
|
-
exchange = await publish_channel.get_exchange(self.exchange_name)
|
|
487
|
-
confirmed = await exchange.publish(
|
|
488
|
-
message,
|
|
489
|
-
routing_key=routing_key or self.routing_key or '#',
|
|
490
|
-
mandatory=True,
|
|
491
|
-
timeout=5.0
|
|
492
|
-
)
|
|
493
|
-
if not confirmed:
|
|
494
|
-
raise Exception("消息未被服务器确认接收")
|
|
495
|
-
|
|
496
|
-
self._update_activity_timestamp()
|
|
497
|
-
logger.debug(f"消息已发布到交换机 '{self.exchange_name}'")
|
|
498
|
-
return
|
|
499
|
-
except (ConnectionClosed, ChannelInvalidStateError):
|
|
500
|
-
retry_count += 1
|
|
501
|
-
logger.warning(f"连接已关闭,尝试重连后重新发布 (重试次数: {retry_count})")
|
|
502
|
-
await self.connect(force_reconnect=True)
|
|
503
|
-
except Exception as e:
|
|
504
|
-
retry_count += 1
|
|
505
|
-
logger.error(f"消息发布失败 (重试次数: {retry_count}): {str(e)}")
|
|
506
|
-
if retry_count < max_retries:
|
|
507
|
-
await asyncio.sleep(1)
|
|
508
|
-
|
|
509
|
-
raise Exception(f"消息发布失败,经过{retry_count}次重试仍未成功")
|
|
510
|
-
|
|
511
|
-
# 以下方法(消息消费相关)逻辑与原有保持一致,仅适配连接池
|
|
512
|
-
def set_message_handler(self, handler):
|
|
513
|
-
self.message_handler = handler
|
|
514
|
-
|
|
515
|
-
async def start_consuming(self) -> ConsumerTag:
|
|
516
|
-
if self._is_consuming:
|
|
517
|
-
logger.debug("已经在消费中,返回现有consumer_tag")
|
|
518
|
-
if self._consumer_tag:
|
|
519
|
-
return self._consumer_tag
|
|
520
|
-
# 如果_is_consuming为True但无consumer_tag,重置状态并重新启动
|
|
521
|
-
logger.warning("检测到消费状态异常(无consumer_tag),重置状态后重试")
|
|
522
|
-
self._is_consuming = False # 重置状态
|
|
523
|
-
|
|
524
|
-
if not self.is_connected:
|
|
525
|
-
await self.connect()
|
|
526
|
-
|
|
527
|
-
if not self.queue:
|
|
528
|
-
raise Exception("队列未初始化,无法开始消费")
|
|
529
|
-
|
|
530
|
-
if not self.message_handler:
|
|
531
|
-
raise Exception("未设置消息处理函数")
|
|
532
|
-
|
|
533
|
-
self._is_consuming = True
|
|
534
|
-
logger.info(f"开始消费队列: {self.actual_queue_name}")
|
|
535
|
-
|
|
255
|
+
# 发布消息
|
|
536
256
|
try:
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
257
|
+
async with self._connect_lock:
|
|
258
|
+
if not self._exchange:
|
|
259
|
+
# 交换机未初始化,重新声明
|
|
260
|
+
logger.warning("交换机未初始化,重新声明")
|
|
261
|
+
self._exchange = await self._channel.declare_exchange(
|
|
262
|
+
name=self.exchange_name,
|
|
263
|
+
type=self.exchange_type,
|
|
264
|
+
durable=self.durable,
|
|
265
|
+
auto_delete=self.auto_delete
|
|
266
|
+
)
|
|
267
|
+
await self._exchange.publish(
|
|
268
|
+
message=message,
|
|
269
|
+
routing_key=self.routing_key or self.queue_name or "#"
|
|
270
|
+
)
|
|
271
|
+
logger.debug(f"消息发布成功(routing_key: {self.routing_key})")
|
|
550
272
|
except Exception as e:
|
|
551
|
-
|
|
552
|
-
|
|
273
|
+
logger.error(f"发布消息失败: {str(e)}", exc_info=True)
|
|
274
|
+
# 发布失败时清理状态,下次自动重连
|
|
275
|
+
self._exchange = None
|
|
553
276
|
raise
|
|
554
277
|
|
|
555
|
-
async def
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
try:
|
|
560
|
-
await asyncio.wait_for(
|
|
561
|
-
self.queue.cancel(self._consumer_tag),
|
|
562
|
-
timeout=self.rpc_timeout
|
|
563
|
-
)
|
|
564
|
-
logger.info(f"消费者 {self._consumer_tag} 已取消")
|
|
565
|
-
return True
|
|
566
|
-
except Exception as e:
|
|
567
|
-
logger.error(f"取消消费者异常: {str(e)}")
|
|
568
|
-
return False
|
|
569
|
-
|
|
570
|
-
async def stop_consuming(self) -> None:
|
|
571
|
-
if not self._is_consuming:
|
|
572
|
-
return
|
|
573
|
-
|
|
574
|
-
self._is_consuming = False
|
|
575
|
-
|
|
576
|
-
if self._consumer_tag and self.queue:
|
|
577
|
-
await self._safe_cancel_consumer()
|
|
578
|
-
|
|
579
|
-
# 等待所有正在处理的消息完成
|
|
580
|
-
if self._processing_message_ids:
|
|
581
|
-
logger.info(
|
|
582
|
-
f"等待 {len(self._processing_message_ids)} 个正在处理的消息完成...")
|
|
583
|
-
while self._processing_message_ids and not self._closed:
|
|
584
|
-
await asyncio.sleep(0.1)
|
|
585
|
-
|
|
586
|
-
# 清理状态
|
|
587
|
-
self._consumer_tag = None
|
|
588
|
-
self._processing_message_ids.clear()
|
|
589
|
-
|
|
590
|
-
logger.info(f"已停止消费队列: {self.actual_queue_name}")
|
|
591
|
-
|
|
592
|
-
async def _parse_message(self, message: AbstractIncomingMessage) -> Union[Dict[str, Any], str]:
|
|
593
|
-
try:
|
|
594
|
-
body_str = message.body.decode('utf-8')
|
|
595
|
-
self._update_activity_timestamp()
|
|
596
|
-
|
|
597
|
-
if self.auto_parse_json:
|
|
598
|
-
return json.loads(body_str)
|
|
599
|
-
return body_str
|
|
600
|
-
except json.JSONDecodeError:
|
|
601
|
-
logger.warning(f"消息解析JSON失败,返回原始字符串")
|
|
602
|
-
return body_str
|
|
603
|
-
except Exception as e:
|
|
604
|
-
logger.error(f"消息解析出错: {str(e)}")
|
|
605
|
-
return message.body.decode('utf-8')
|
|
606
|
-
|
|
607
|
-
async def _message_wrapper(self, message: AbstractIncomingMessage) -> None:
|
|
608
|
-
if not self.message_handler or not self._is_consuming:
|
|
609
|
-
logger.warning("未设置消息处理器或已停止消费")
|
|
610
|
-
# await message.ack()
|
|
611
|
-
try:
|
|
612
|
-
await message.reject(requeue=True)
|
|
613
|
-
except Exception as e:
|
|
614
|
-
logger.error(f"拒绝消息失败: {e}")
|
|
615
|
-
return
|
|
616
|
-
|
|
617
|
-
message_id = message.message_id or str(id(message))
|
|
618
|
-
if message_id in self._processing_message_ids:
|
|
619
|
-
logger.warning(f"检测到重复处理的消息ID: {message_id},直接确认")
|
|
620
|
-
await message.ack()
|
|
278
|
+
async def close(self) -> None:
|
|
279
|
+
"""关闭客户端(释放通道)"""
|
|
280
|
+
if self._closed:
|
|
621
281
|
return
|
|
622
282
|
|
|
623
|
-
self.
|
|
624
|
-
|
|
625
|
-
try:
|
|
626
|
-
logger.debug(f"收到队列 {self.actual_queue_name} 的消息: {message_id}")
|
|
283
|
+
self._closed = True
|
|
284
|
+
logger.info("开始关闭RabbitMQ客户端...")
|
|
627
285
|
|
|
628
|
-
|
|
629
|
-
|
|
286
|
+
# 先停止消费
|
|
287
|
+
await self.stop_consuming()
|
|
630
288
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
self.
|
|
634
|
-
|
|
289
|
+
# 释放通道到连接池
|
|
290
|
+
async with self._connect_lock:
|
|
291
|
+
if self._channel and not self._channel.is_closed:
|
|
292
|
+
try:
|
|
293
|
+
await self.connection_pool.release_channel(self._channel)
|
|
294
|
+
except Exception as e:
|
|
295
|
+
logger.warning(f"释放通道失败: {str(e)}")
|
|
296
|
+
self._channel = None
|
|
297
|
+
self._exchange = None
|
|
298
|
+
self._queue = None
|
|
635
299
|
|
|
636
|
-
|
|
637
|
-
current_headers = message.headers or {}
|
|
638
|
-
retry_count = current_headers.get('x-retry-count', 0)
|
|
639
|
-
retry_count += 1
|
|
640
|
-
|
|
641
|
-
logger.error(
|
|
642
|
-
f"消息 {message_id} 处理出错(第{retry_count}次重试): {str(e)}",
|
|
643
|
-
exc_info=True
|
|
644
|
-
)
|
|
645
|
-
|
|
646
|
-
if retry_count >= MAX_RETRY_COUNT:
|
|
647
|
-
logger.error(
|
|
648
|
-
f"消息 {message_id} 已达到最大重试次数({MAX_RETRY_COUNT}次),标记为失败")
|
|
649
|
-
await message.ack()
|
|
650
|
-
self._update_activity_timestamp()
|
|
651
|
-
return
|
|
652
|
-
|
|
653
|
-
new_headers = current_headers.copy()
|
|
654
|
-
new_headers['x-retry-count'] = retry_count
|
|
655
|
-
|
|
656
|
-
new_message = Message(
|
|
657
|
-
body=message.body,
|
|
658
|
-
content_type=message.content_type,
|
|
659
|
-
headers=new_headers,
|
|
660
|
-
delivery_mode=message.delivery_mode
|
|
661
|
-
)
|
|
662
|
-
|
|
663
|
-
await message.reject(requeue=False)
|
|
664
|
-
|
|
665
|
-
if self.exchange:
|
|
666
|
-
await self.exchange.publish(
|
|
667
|
-
new_message,
|
|
668
|
-
routing_key=self.routing_key or '#',
|
|
669
|
-
mandatory=True,
|
|
670
|
-
timeout=5.0
|
|
671
|
-
)
|
|
672
|
-
self._update_activity_timestamp()
|
|
673
|
-
logger.info(f"消息 {message_id} 已重新发布,当前重试次数: {retry_count}")
|
|
674
|
-
finally:
|
|
675
|
-
if message_id in self._processing_message_ids:
|
|
676
|
-
self._processing_message_ids.remove(message_id)
|
|
677
|
-
|
|
678
|
-
async def __aenter__(self):
|
|
679
|
-
await self.connect()
|
|
680
|
-
return self
|
|
681
|
-
|
|
682
|
-
async def __aexit__(self, exc_type, exc, tb):
|
|
683
|
-
await self.close()
|
|
300
|
+
logger.info("RabbitMQ客户端已关闭")
|