sycommon-python-lib 0.1.37__tar.gz → 0.1.39__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/PKG-INFO +1 -1
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/pyproject.toml +1 -1
- sycommon_python_lib-0.1.39/src/sycommon/rabbitmq/rabbitmq_client.py +300 -0
- sycommon_python_lib-0.1.39/src/sycommon/rabbitmq/rabbitmq_pool.py +180 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/rabbitmq/rabbitmq_service.py +41 -18
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/PKG-INFO +1 -1
- sycommon_python_lib-0.1.37/src/sycommon/rabbitmq/rabbitmq_client.py +0 -981
- sycommon_python_lib-0.1.37/src/sycommon/rabbitmq/rabbitmq_pool.py +0 -104
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/README.md +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/setup.cfg +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/command/cli.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/Config.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/DatabaseConfig.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/EmbeddingConfig.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/LLMConfig.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/MQConfig.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/RerankerConfig.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/config/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/database/base_db_service.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/database/database_service.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/health/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/health/health_check.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/health/metrics.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/health/ping.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/logging/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/logging/kafka_log.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/logging/logger_wrapper.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/logging/sql_logger.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/context.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/cors.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/docs.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/exception.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/middleware.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/monitor_memory.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/mq.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/timeout.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/middleware/traceid.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/base_http.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/log.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/mqlistener_config.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/mqmsg_model.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/mqsend_config.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/models/sso_user.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/services.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/sse/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/sse/event.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/sse/sse.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/example.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/example2.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/feign.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/feign_client.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/nacos_service.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/synacos/param.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/tools/__init__.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/tools/docs.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/tools/snowflake.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/tools/timing.py +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/SOURCES.txt +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/dependency_links.txt +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/requires.txt +0 -0
- {sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon_python_lib.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,300 @@
|
|
|
1
|
+
from aio_pika import Channel
|
|
2
|
+
from typing import Optional
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
from typing import Callable, Coroutine, Dict, Any, Union
|
|
6
|
+
from aio_pika import Message, DeliveryMode, ExchangeType
|
|
7
|
+
from aio_pika.abc import (
|
|
8
|
+
AbstractExchange,
|
|
9
|
+
AbstractQueue,
|
|
10
|
+
AbstractIncomingMessage,
|
|
11
|
+
ConsumerTag,
|
|
12
|
+
)
|
|
13
|
+
from src.sycommon.rabbitmq.rabbitmq_pool import RabbitMQConnectionPool
|
|
14
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
15
|
+
from sycommon.models.mqmsg_model import MQMsgModel
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
# 最大重试次数限制
|
|
19
|
+
MAX_RETRY_COUNT = 3
|
|
20
|
+
|
|
21
|
+
logger = SYLogger
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class RabbitMQClient:
|
|
25
|
+
"""
|
|
26
|
+
RabbitMQ客户端(基于连接池),支持集群、自动重连、消息发布/消费
|
|
27
|
+
依赖 aio_pika 的内置重连机制,移除手动重连逻辑
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
connection_pool: RabbitMQConnectionPool,
|
|
33
|
+
exchange_name: str = "system.topic.exchange",
|
|
34
|
+
exchange_type: str = "topic",
|
|
35
|
+
queue_name: Optional[str] = None,
|
|
36
|
+
routing_key: str = "#",
|
|
37
|
+
durable: bool = True,
|
|
38
|
+
auto_delete: bool = False,
|
|
39
|
+
auto_parse_json: bool = True,
|
|
40
|
+
create_if_not_exists: bool = True,
|
|
41
|
+
rpc_timeout: int = 10,
|
|
42
|
+
prefetch_count: int = 2,
|
|
43
|
+
consumption_stall_threshold: int = 60,
|
|
44
|
+
):
|
|
45
|
+
self.connection_pool = connection_pool
|
|
46
|
+
if not self.connection_pool._initialized:
|
|
47
|
+
raise RuntimeError("连接池未初始化,请先调用 connection_pool.init_pools()")
|
|
48
|
+
|
|
49
|
+
# 交换机配置
|
|
50
|
+
self.exchange_name = exchange_name
|
|
51
|
+
self.exchange_type = ExchangeType(exchange_type.lower())
|
|
52
|
+
# 队列配置
|
|
53
|
+
self.queue_name = queue_name
|
|
54
|
+
self.routing_key = routing_key
|
|
55
|
+
self.durable = durable
|
|
56
|
+
self.auto_delete = auto_delete
|
|
57
|
+
self.auto_parse_json = auto_parse_json
|
|
58
|
+
self.create_if_not_exists = create_if_not_exists
|
|
59
|
+
|
|
60
|
+
# 运行时配置
|
|
61
|
+
self.rpc_timeout = rpc_timeout
|
|
62
|
+
self.prefetch_count = prefetch_count
|
|
63
|
+
self.consumption_stall_threshold = consumption_stall_threshold
|
|
64
|
+
|
|
65
|
+
# 内部状态
|
|
66
|
+
self._channel: Optional[Channel] = None
|
|
67
|
+
self._exchange: Optional[AbstractExchange] = None
|
|
68
|
+
self._queue: Optional[AbstractQueue] = None
|
|
69
|
+
self._consumer_tag: Optional[ConsumerTag] = None
|
|
70
|
+
self._message_handler: Optional[Callable] = None
|
|
71
|
+
self._closed = False
|
|
72
|
+
|
|
73
|
+
# 细粒度锁
|
|
74
|
+
self._consume_lock = asyncio.Lock()
|
|
75
|
+
self._connect_lock = asyncio.Lock()
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
async def is_connected(self) -> bool:
|
|
79
|
+
"""异步检查客户端连接状态(属性,不是函数)"""
|
|
80
|
+
if self._closed:
|
|
81
|
+
return False
|
|
82
|
+
try:
|
|
83
|
+
# 检查通道是否有效
|
|
84
|
+
if self._channel and not self._channel.is_closed:
|
|
85
|
+
# 检查交换机和队列(如果需要)
|
|
86
|
+
if self.create_if_not_exists and self.queue_name:
|
|
87
|
+
return bool(self._exchange and self._queue)
|
|
88
|
+
return True
|
|
89
|
+
return False
|
|
90
|
+
except Exception as e:
|
|
91
|
+
logger.warning(f"检查连接状态失败: {str(e)}")
|
|
92
|
+
return False
|
|
93
|
+
|
|
94
|
+
async def _get_connection_resources(self) -> tuple[Optional[Channel], Optional[AbstractExchange], Optional[AbstractQueue]]:
|
|
95
|
+
"""原子获取连接资源(通道、交换机、队列)"""
|
|
96
|
+
async with self._connect_lock:
|
|
97
|
+
return self._channel, self._exchange, self._queue
|
|
98
|
+
|
|
99
|
+
async def connect(self) -> None:
|
|
100
|
+
"""建立连接并初始化交换机/队列(使用新的通道获取方式)"""
|
|
101
|
+
if self._closed:
|
|
102
|
+
raise RuntimeError("客户端已关闭,无法重新连接")
|
|
103
|
+
|
|
104
|
+
async with self._connect_lock:
|
|
105
|
+
try:
|
|
106
|
+
# 关键修复:使用连接池的 acquire_channel 方法获取通道
|
|
107
|
+
self._channel = await self.connection_pool.acquire_channel()
|
|
108
|
+
# 设置预取计数
|
|
109
|
+
await self._channel.set_qos(prefetch_count=self.prefetch_count)
|
|
110
|
+
|
|
111
|
+
# 声明交换机
|
|
112
|
+
self._exchange = await self._channel.declare_exchange(
|
|
113
|
+
name=self.exchange_name,
|
|
114
|
+
type=self.exchange_type,
|
|
115
|
+
durable=self.durable,
|
|
116
|
+
auto_delete=self.auto_delete,
|
|
117
|
+
passive=not self.create_if_not_exists # 不创建时使用passive模式检查
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# 声明队列(如果配置了队列名)
|
|
121
|
+
if self.queue_name:
|
|
122
|
+
self._queue = await self._channel.declare_queue(
|
|
123
|
+
name=self.queue_name,
|
|
124
|
+
durable=self.durable,
|
|
125
|
+
auto_delete=self.auto_delete,
|
|
126
|
+
passive=not self.create_if_not_exists
|
|
127
|
+
)
|
|
128
|
+
# 绑定队列到交换机
|
|
129
|
+
await self._queue.bind(
|
|
130
|
+
exchange=self._exchange,
|
|
131
|
+
routing_key=self.routing_key or self.queue_name
|
|
132
|
+
)
|
|
133
|
+
logger.info(
|
|
134
|
+
f"队列 '{self.queue_name}' 已声明并绑定到交换机 '{self.exchange_name}' "
|
|
135
|
+
f"(exchange_type: {self.exchange_type}, routing_key: {self.routing_key})"
|
|
136
|
+
)
|
|
137
|
+
else:
|
|
138
|
+
logger.info(
|
|
139
|
+
f"未配置队列名,仅初始化交换机 '{self.exchange_name}' (exchange_type: {self.exchange_type})")
|
|
140
|
+
|
|
141
|
+
logger.info(f"RabbitMQ客户端连接成功(exchange: {self.exchange_name})")
|
|
142
|
+
except Exception as e:
|
|
143
|
+
logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
|
|
144
|
+
# 清理异常状态
|
|
145
|
+
if self._channel:
|
|
146
|
+
await self.connection_pool.release_channel(self._channel)
|
|
147
|
+
self._channel = None
|
|
148
|
+
self._exchange = None
|
|
149
|
+
self._queue = None
|
|
150
|
+
raise
|
|
151
|
+
|
|
152
|
+
async def set_message_handler(self, handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]):
|
|
153
|
+
"""设置消息处理器(消费消息时使用)"""
|
|
154
|
+
async with self._consume_lock:
|
|
155
|
+
self._message_handler = handler
|
|
156
|
+
logger.info("消息处理器已设置")
|
|
157
|
+
|
|
158
|
+
async def start_consuming(self) -> Optional[ConsumerTag]:
|
|
159
|
+
"""启动消费(返回消费者标签)"""
|
|
160
|
+
if self._closed:
|
|
161
|
+
logger.warning("客户端已关闭,无法启动消费")
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
async with self._consume_lock:
|
|
165
|
+
# 检查前置条件
|
|
166
|
+
if not self._message_handler:
|
|
167
|
+
raise RuntimeError("未设置消息处理器,请先调用 set_message_handler")
|
|
168
|
+
|
|
169
|
+
# 确保连接已建立
|
|
170
|
+
if not await self.is_connected:
|
|
171
|
+
await self.connect()
|
|
172
|
+
|
|
173
|
+
# 确保队列已初始化
|
|
174
|
+
_, _, queue = await self._get_connection_resources()
|
|
175
|
+
if not queue:
|
|
176
|
+
raise RuntimeError("队列未初始化,无法启动消费")
|
|
177
|
+
|
|
178
|
+
# 定义消费回调
|
|
179
|
+
async def consume_callback(message: AbstractIncomingMessage):
|
|
180
|
+
try:
|
|
181
|
+
# 解析消息
|
|
182
|
+
if self.auto_parse_json:
|
|
183
|
+
body = json.loads(message.body.decode('utf-8'))
|
|
184
|
+
parsed_data = MQMsgModel(**body)
|
|
185
|
+
else:
|
|
186
|
+
parsed_data = MQMsgModel(
|
|
187
|
+
msg=message.body.decode('utf-8'))
|
|
188
|
+
|
|
189
|
+
# 调用处理器
|
|
190
|
+
await self._message_handler(parsed_data, message)
|
|
191
|
+
# 手动确认消息
|
|
192
|
+
await message.ack()
|
|
193
|
+
except Exception as e:
|
|
194
|
+
logger.error(
|
|
195
|
+
f"处理消息失败 (delivery_tag: {message.delivery_tag}): {str(e)}", exc_info=True)
|
|
196
|
+
# 消费失败时重新入队(最多重试3次)
|
|
197
|
+
if message.redelivered:
|
|
198
|
+
logger.warning(
|
|
199
|
+
f"消息已重试过,拒绝入队 (delivery_tag: {message.delivery_tag})")
|
|
200
|
+
await message.reject(requeue=False)
|
|
201
|
+
else:
|
|
202
|
+
await message.nack(requeue=True)
|
|
203
|
+
|
|
204
|
+
# 启动消费
|
|
205
|
+
self._consumer_tag = await queue.consume(consume_callback)
|
|
206
|
+
logger.info(
|
|
207
|
+
f"开始消费队列 '{queue.name}',consumer_tag: {self._consumer_tag}")
|
|
208
|
+
return self._consumer_tag
|
|
209
|
+
|
|
210
|
+
async def stop_consuming(self) -> None:
|
|
211
|
+
"""停止消费"""
|
|
212
|
+
async with self._consume_lock:
|
|
213
|
+
if self._consumer_tag and not self._closed:
|
|
214
|
+
_, _, queue = await self._get_connection_resources()
|
|
215
|
+
if queue and not queue.is_closed:
|
|
216
|
+
try:
|
|
217
|
+
await queue.cancel(self._consumer_tag)
|
|
218
|
+
logger.info(f"停止消费,consumer_tag: {self._consumer_tag}")
|
|
219
|
+
except Exception as e:
|
|
220
|
+
logger.error(f"停止消费失败: {str(e)}")
|
|
221
|
+
self._consumer_tag = None
|
|
222
|
+
|
|
223
|
+
async def publish(
|
|
224
|
+
self,
|
|
225
|
+
message_body: Union[str, Dict[str, Any], MQMsgModel],
|
|
226
|
+
headers: Optional[Dict[str, Any]] = None,
|
|
227
|
+
content_type: str = "application/json",
|
|
228
|
+
delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT
|
|
229
|
+
) -> None:
|
|
230
|
+
"""发布消息(修复 is_closed 问题)"""
|
|
231
|
+
if self._closed:
|
|
232
|
+
raise RuntimeError("客户端已关闭,无法发布消息")
|
|
233
|
+
|
|
234
|
+
# 确保连接已建立
|
|
235
|
+
if not await self.is_connected:
|
|
236
|
+
await self.connect()
|
|
237
|
+
|
|
238
|
+
# 处理消息体
|
|
239
|
+
if isinstance(message_body, MQMsgModel):
|
|
240
|
+
body = json.dumps(message_body.__dict__,
|
|
241
|
+
ensure_ascii=False).encode('utf-8')
|
|
242
|
+
elif isinstance(message_body, dict):
|
|
243
|
+
body = json.dumps(message_body, ensure_ascii=False).encode('utf-8')
|
|
244
|
+
else:
|
|
245
|
+
body = str(message_body).encode('utf-8')
|
|
246
|
+
|
|
247
|
+
# 创建消息
|
|
248
|
+
message = Message(
|
|
249
|
+
body=body,
|
|
250
|
+
headers=headers or {},
|
|
251
|
+
content_type=content_type,
|
|
252
|
+
delivery_mode=delivery_mode
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
# 发布消息
|
|
256
|
+
try:
|
|
257
|
+
async with self._connect_lock:
|
|
258
|
+
if not self._exchange:
|
|
259
|
+
# 交换机未初始化,重新声明
|
|
260
|
+
logger.warning("交换机未初始化,重新声明")
|
|
261
|
+
self._exchange = await self._channel.declare_exchange(
|
|
262
|
+
name=self.exchange_name,
|
|
263
|
+
type=self.exchange_type,
|
|
264
|
+
durable=self.durable,
|
|
265
|
+
auto_delete=self.auto_delete
|
|
266
|
+
)
|
|
267
|
+
await self._exchange.publish(
|
|
268
|
+
message=message,
|
|
269
|
+
routing_key=self.routing_key or self.queue_name or "#"
|
|
270
|
+
)
|
|
271
|
+
logger.debug(f"消息发布成功(routing_key: {self.routing_key})")
|
|
272
|
+
except Exception as e:
|
|
273
|
+
logger.error(f"发布消息失败: {str(e)}", exc_info=True)
|
|
274
|
+
# 发布失败时清理状态,下次自动重连
|
|
275
|
+
self._exchange = None
|
|
276
|
+
raise
|
|
277
|
+
|
|
278
|
+
async def close(self) -> None:
|
|
279
|
+
"""关闭客户端(释放通道)"""
|
|
280
|
+
if self._closed:
|
|
281
|
+
return
|
|
282
|
+
|
|
283
|
+
self._closed = True
|
|
284
|
+
logger.info("开始关闭RabbitMQ客户端...")
|
|
285
|
+
|
|
286
|
+
# 先停止消费
|
|
287
|
+
await self.stop_consuming()
|
|
288
|
+
|
|
289
|
+
# 释放通道到连接池
|
|
290
|
+
async with self._connect_lock:
|
|
291
|
+
if self._channel and not self._channel.is_closed:
|
|
292
|
+
try:
|
|
293
|
+
await self.connection_pool.release_channel(self._channel)
|
|
294
|
+
except Exception as e:
|
|
295
|
+
logger.warning(f"释放通道失败: {str(e)}")
|
|
296
|
+
self._channel = None
|
|
297
|
+
self._exchange = None
|
|
298
|
+
self._queue = None
|
|
299
|
+
|
|
300
|
+
logger.info("RabbitMQ客户端已关闭")
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from typing import List, Set
|
|
3
|
+
from aio_pika import connect_robust, Channel
|
|
4
|
+
from aio_pika.abc import AbstractRobustConnection
|
|
5
|
+
|
|
6
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
7
|
+
|
|
8
|
+
logger = SYLogger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class RabbitMQConnectionPool:
|
|
12
|
+
"""RabbitMQ连接池管理(简化实现,避免上下文管理器冲突)"""
|
|
13
|
+
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
hosts: List[str],
|
|
17
|
+
port: int,
|
|
18
|
+
username: str,
|
|
19
|
+
password: str,
|
|
20
|
+
virtualhost: str = "/",
|
|
21
|
+
connection_pool_size: int = 2,
|
|
22
|
+
channel_pool_size: int = 10,
|
|
23
|
+
heartbeat: int = 10,
|
|
24
|
+
app_name: str = ""
|
|
25
|
+
):
|
|
26
|
+
self.hosts = [host.strip() for host in hosts if host.strip()]
|
|
27
|
+
if not self.hosts:
|
|
28
|
+
raise ValueError("至少需要提供一个RabbitMQ主机地址")
|
|
29
|
+
self.port = port
|
|
30
|
+
self.username = username
|
|
31
|
+
self.password = password
|
|
32
|
+
self.virtualhost = virtualhost
|
|
33
|
+
self.app_name = app_name or "rabbitmq-client"
|
|
34
|
+
self.heartbeat = heartbeat
|
|
35
|
+
|
|
36
|
+
# 连接池配置
|
|
37
|
+
self.connection_pool_size = connection_pool_size
|
|
38
|
+
self.channel_pool_size = channel_pool_size
|
|
39
|
+
|
|
40
|
+
# 实际存储的连接和通道
|
|
41
|
+
self._connections: List[AbstractRobustConnection] = []
|
|
42
|
+
self._free_channels: List[Channel] = []
|
|
43
|
+
self._used_channels: Set[Channel] = set()
|
|
44
|
+
|
|
45
|
+
# 锁用于线程安全
|
|
46
|
+
self._conn_lock = asyncio.Lock()
|
|
47
|
+
self._chan_lock = asyncio.Lock()
|
|
48
|
+
|
|
49
|
+
# 连接状态
|
|
50
|
+
self._initialized = False
|
|
51
|
+
|
|
52
|
+
async def init_pools(self):
|
|
53
|
+
"""初始化连接池(创建指定数量的连接)"""
|
|
54
|
+
if self._initialized:
|
|
55
|
+
logger.warning("连接池已初始化,无需重复调用")
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
try:
|
|
59
|
+
# 创建核心连接(数量=connection_pool_size)
|
|
60
|
+
for i in range(self.connection_pool_size):
|
|
61
|
+
conn = await self._create_connection()
|
|
62
|
+
self._connections.append(conn)
|
|
63
|
+
# 为每个连接创建初始通道(数量=channel_pool_size//connection_pool_size)
|
|
64
|
+
chan_count_per_conn = self.channel_pool_size // self.connection_pool_size
|
|
65
|
+
for _ in range(chan_count_per_conn):
|
|
66
|
+
chan = await conn.channel()
|
|
67
|
+
self._free_channels.append(chan)
|
|
68
|
+
|
|
69
|
+
self._initialized = True
|
|
70
|
+
logger.info(
|
|
71
|
+
f"RabbitMQ连接池初始化成功 - 连接数: {len(self._connections)}, "
|
|
72
|
+
f"空闲通道数: {len(self._free_channels)}, 集群节点: {self.hosts}"
|
|
73
|
+
)
|
|
74
|
+
except Exception as e:
|
|
75
|
+
logger.error(f"连接池初始化失败: {str(e)}", exc_info=True)
|
|
76
|
+
# 清理异常状态
|
|
77
|
+
await self.close()
|
|
78
|
+
raise
|
|
79
|
+
|
|
80
|
+
async def _create_connection(self) -> AbstractRobustConnection:
|
|
81
|
+
"""创建单个RabbitMQ连接(支持集群节点轮询)"""
|
|
82
|
+
hosts = self.hosts.copy()
|
|
83
|
+
while hosts:
|
|
84
|
+
host = hosts.pop(0)
|
|
85
|
+
try:
|
|
86
|
+
connection = await connect_robust(
|
|
87
|
+
host=host,
|
|
88
|
+
port=self.port,
|
|
89
|
+
login=self.username,
|
|
90
|
+
password=self.password,
|
|
91
|
+
virtualhost=self.virtualhost,
|
|
92
|
+
heartbeat=self.heartbeat,
|
|
93
|
+
client_properties={
|
|
94
|
+
"connection_name": f"{self.app_name}@{host}"
|
|
95
|
+
},
|
|
96
|
+
reconnect_interval=2 # aio_pika 内置重连间隔
|
|
97
|
+
)
|
|
98
|
+
logger.info(f"成功连接到 RabbitMQ 节点: {host}:{self.port}")
|
|
99
|
+
return connection
|
|
100
|
+
except Exception as e:
|
|
101
|
+
logger.warning(
|
|
102
|
+
f"连接主机 {host}:{self.port} 失败,尝试下一个节点: {str(e)}")
|
|
103
|
+
if not hosts:
|
|
104
|
+
raise # 所有节点失败时抛出异常
|
|
105
|
+
|
|
106
|
+
async def acquire_channel(self) -> Channel:
|
|
107
|
+
"""获取通道(从空闲通道池获取,无则创建新通道)"""
|
|
108
|
+
if not self._initialized:
|
|
109
|
+
raise RuntimeError("连接池未初始化,请先调用 init_pools()")
|
|
110
|
+
|
|
111
|
+
async with self._chan_lock:
|
|
112
|
+
# 优先从空闲通道池获取
|
|
113
|
+
if self._free_channels:
|
|
114
|
+
channel = self._free_channels.pop()
|
|
115
|
+
# 检查通道是否有效
|
|
116
|
+
if not channel.is_closed:
|
|
117
|
+
self._used_channels.add(channel)
|
|
118
|
+
return channel
|
|
119
|
+
else:
|
|
120
|
+
logger.warning("发现无效通道,已自动清理")
|
|
121
|
+
|
|
122
|
+
# 空闲通道不足,创建新通道(不超过最大限制)
|
|
123
|
+
if len(self._used_channels) < self.channel_pool_size:
|
|
124
|
+
# 选择一个空闲连接创建通道
|
|
125
|
+
async with self._conn_lock:
|
|
126
|
+
for conn in self._connections:
|
|
127
|
+
if not conn.is_closed:
|
|
128
|
+
try:
|
|
129
|
+
channel = await conn.channel()
|
|
130
|
+
self._used_channels.add(channel)
|
|
131
|
+
logger.info(
|
|
132
|
+
f"创建新通道,当前通道数: {len(self._used_channels)}/{self.channel_pool_size}")
|
|
133
|
+
return channel
|
|
134
|
+
except Exception as e:
|
|
135
|
+
logger.warning(f"使用连接创建通道失败: {str(e)}")
|
|
136
|
+
# 所有连接都无效,尝试重新创建连接
|
|
137
|
+
conn = await self._create_connection()
|
|
138
|
+
self._connections.append(conn)
|
|
139
|
+
channel = await conn.channel()
|
|
140
|
+
self._used_channels.add(channel)
|
|
141
|
+
return channel
|
|
142
|
+
else:
|
|
143
|
+
raise RuntimeError(f"通道池已达最大限制: {self.channel_pool_size}")
|
|
144
|
+
|
|
145
|
+
async def release_channel(self, channel: Channel):
|
|
146
|
+
"""释放通道(归还到空闲通道池)"""
|
|
147
|
+
async with self._chan_lock:
|
|
148
|
+
if channel in self._used_channels:
|
|
149
|
+
self._used_channels.remove(channel)
|
|
150
|
+
# 通道有效则归还,无效则丢弃
|
|
151
|
+
if not channel.is_closed:
|
|
152
|
+
self._free_channels.append(channel)
|
|
153
|
+
else:
|
|
154
|
+
logger.warning("释放无效通道,已自动丢弃")
|
|
155
|
+
|
|
156
|
+
async def close(self):
|
|
157
|
+
"""关闭连接池(释放所有连接和通道)"""
|
|
158
|
+
# 释放所有通道
|
|
159
|
+
async with self._chan_lock:
|
|
160
|
+
for channel in self._free_channels + list(self._used_channels):
|
|
161
|
+
try:
|
|
162
|
+
if not channel.is_closed:
|
|
163
|
+
await channel.close()
|
|
164
|
+
except Exception as e:
|
|
165
|
+
logger.warning(f"关闭通道失败: {str(e)}")
|
|
166
|
+
self._free_channels.clear()
|
|
167
|
+
self._used_channels.clear()
|
|
168
|
+
|
|
169
|
+
# 关闭所有连接
|
|
170
|
+
async with self._conn_lock:
|
|
171
|
+
for conn in self._connections:
|
|
172
|
+
try:
|
|
173
|
+
if not conn.is_closed:
|
|
174
|
+
await conn.close()
|
|
175
|
+
except Exception as e:
|
|
176
|
+
logger.warning(f"关闭连接失败: {str(e)}")
|
|
177
|
+
self._connections.clear()
|
|
178
|
+
|
|
179
|
+
self._initialized = False
|
|
180
|
+
logger.info("RabbitMQ连接池已完全关闭")
|
{sycommon_python_lib-0.1.37 → sycommon_python_lib-0.1.39}/src/sycommon/rabbitmq/rabbitmq_service.py
RENAMED
|
@@ -179,18 +179,21 @@ class RabbitMQService:
|
|
|
179
179
|
auto_delete=kwargs.get('auto_delete', False),
|
|
180
180
|
auto_parse_json=kwargs.get('auto_parse_json', True),
|
|
181
181
|
create_if_not_exists=create_if_not_exists,
|
|
182
|
-
connection_timeout=kwargs.get('connection_timeout', 10),
|
|
183
182
|
rpc_timeout=kwargs.get('rpc_timeout', 10),
|
|
184
|
-
reconnection_delay=kwargs.get('reconnection_delay', 1),
|
|
185
|
-
max_reconnection_attempts=kwargs.get(
|
|
186
|
-
'max_reconnection_attempts', 5),
|
|
187
183
|
prefetch_count=kwargs.get('prefetch_count', 2),
|
|
188
184
|
consumption_stall_threshold=kwargs.get(
|
|
189
185
|
'consumption_stall_threshold', 60), # 延长停滞阈值
|
|
190
186
|
)
|
|
191
187
|
|
|
192
|
-
#
|
|
193
|
-
await client.connect(
|
|
188
|
+
# 新客户端connect无declare_queue参数,自动根据create_if_not_exists处理
|
|
189
|
+
await client.connect()
|
|
190
|
+
|
|
191
|
+
# 监听器客户端连接后延迟1秒,确保消费状态就绪(仅首次启动)
|
|
192
|
+
if not is_sender and create_if_not_exists:
|
|
193
|
+
logger.info(
|
|
194
|
+
f"监听器客户端 '{processed_queue_name}' 连接成功,延迟1秒启动消费(解决启动时序问题)")
|
|
195
|
+
await asyncio.sleep(1)
|
|
196
|
+
|
|
194
197
|
return client
|
|
195
198
|
|
|
196
199
|
@classmethod
|
|
@@ -238,13 +241,13 @@ class RabbitMQService:
|
|
|
238
241
|
if not queue:
|
|
239
242
|
logger.info(f"客户端 '{client_name}' 队列未初始化,重新连接")
|
|
240
243
|
client.create_if_not_exists = True
|
|
241
|
-
await client.connect(force_reconnect
|
|
244
|
+
await client.connect() # 移除force_reconnect和declare_queue
|
|
242
245
|
return client
|
|
243
246
|
else:
|
|
244
247
|
logger.info(f"客户端 '{client_name}' 连接已关闭,重新连接")
|
|
245
248
|
if not is_sender:
|
|
246
249
|
client.create_if_not_exists = True
|
|
247
|
-
await client.connect(declare_queue
|
|
250
|
+
await client.connect() # 移除force_reconnect和declare_queue
|
|
248
251
|
return client
|
|
249
252
|
|
|
250
253
|
# 创建新客户端
|
|
@@ -260,7 +263,7 @@ class RabbitMQService:
|
|
|
260
263
|
app_name=cls._config.get("APP_NAME", ""),
|
|
261
264
|
**kwargs
|
|
262
265
|
)
|
|
263
|
-
await client.connect(declare_queue=False
|
|
266
|
+
await client.connect() # 移除declare_queue=False
|
|
264
267
|
cls._clients[client_name] = client
|
|
265
268
|
return client
|
|
266
269
|
|
|
@@ -273,7 +276,7 @@ class RabbitMQService:
|
|
|
273
276
|
client = await cls._create_client(
|
|
274
277
|
initial_queue_name, ** kwargs
|
|
275
278
|
)
|
|
276
|
-
await client.connect(declare_queue=True
|
|
279
|
+
await client.connect() # 移除declare_queue=True
|
|
277
280
|
cls._clients[client_name] = client
|
|
278
281
|
return client
|
|
279
282
|
|
|
@@ -285,14 +288,14 @@ class RabbitMQService:
|
|
|
285
288
|
)
|
|
286
289
|
|
|
287
290
|
client.create_if_not_exists = True
|
|
288
|
-
await client.connect(declare_queue=True
|
|
291
|
+
await client.connect() # 移除declare_queue=True
|
|
289
292
|
|
|
290
293
|
# 验证队列是否创建成功(异步获取队列)
|
|
291
294
|
_, _, queue = await client._get_connection_resources()
|
|
292
295
|
if not queue:
|
|
293
296
|
logger.error(f"队列 '{initial_queue_name}' 创建失败,尝试重新创建")
|
|
294
297
|
client.create_if_not_exists = True
|
|
295
|
-
await client.connect(
|
|
298
|
+
await client.connect()
|
|
296
299
|
_, _, queue = await client._get_connection_resources()
|
|
297
300
|
if not queue:
|
|
298
301
|
raise Exception(f"无法创建队列 '{initial_queue_name}'")
|
|
@@ -341,7 +344,7 @@ class RabbitMQService:
|
|
|
341
344
|
if normalized_name in cls._clients:
|
|
342
345
|
client = cls._clients[normalized_name]
|
|
343
346
|
if not await client.is_connected:
|
|
344
|
-
await client.connect(declare_queue=False
|
|
347
|
+
await client.connect() # 移除declare_queue=False
|
|
345
348
|
else:
|
|
346
349
|
client = await cls.get_client(
|
|
347
350
|
client_name=normalized_name,
|
|
@@ -508,6 +511,11 @@ class RabbitMQService:
|
|
|
508
511
|
logger.info("服务关闭中,取消启动消费者")
|
|
509
512
|
return
|
|
510
513
|
|
|
514
|
+
# 监听器启动消费前额外延迟1秒,确保consumer_tag完全生成(不删消息)
|
|
515
|
+
if cls._has_listeners and not client_name.startswith("sender-"):
|
|
516
|
+
logger.info(f"消费者 '{client_name}' 准备启动,延迟1秒等待消费状态就绪(不删除积压消息)")
|
|
517
|
+
await asyncio.sleep(1)
|
|
518
|
+
|
|
511
519
|
# 创建停止事件
|
|
512
520
|
stop_event = asyncio.Event()
|
|
513
521
|
cls._consumer_events[client_name] = stop_event
|
|
@@ -522,6 +530,20 @@ class RabbitMQService:
|
|
|
522
530
|
|
|
523
531
|
while attempt < max_attempts and not stop_event.is_set() and not cls._is_shutdown:
|
|
524
532
|
try:
|
|
533
|
+
# 启动消费前再次校验连接和队列状态
|
|
534
|
+
if not await client.is_connected: # 确保这里没有括号
|
|
535
|
+
logger.info(f"消费者 '{client_name}' 连接断开,尝试重连")
|
|
536
|
+
await client.connect()
|
|
537
|
+
|
|
538
|
+
# 确保队列已就绪
|
|
539
|
+
_, _, queue = await client._get_connection_resources()
|
|
540
|
+
if not queue:
|
|
541
|
+
raise Exception("队列未初始化完成")
|
|
542
|
+
|
|
543
|
+
# 确保消息处理器已设置
|
|
544
|
+
if not hasattr(client, '_message_handler') or not client._message_handler:
|
|
545
|
+
raise Exception("消息处理器未设置")
|
|
546
|
+
|
|
525
547
|
consumer_tag = await client.start_consuming()
|
|
526
548
|
if consumer_tag:
|
|
527
549
|
break
|
|
@@ -530,7 +552,7 @@ class RabbitMQService:
|
|
|
530
552
|
logger.warning(
|
|
531
553
|
f"启动消费者尝试 {attempt}/{max_attempts} 失败: {str(e)}")
|
|
532
554
|
if attempt < max_attempts:
|
|
533
|
-
await asyncio.sleep(
|
|
555
|
+
await asyncio.sleep(2) # 延长重试间隔
|
|
534
556
|
|
|
535
557
|
if cls._is_shutdown:
|
|
536
558
|
logger.info("服务关闭中,消费者启动中止")
|
|
@@ -541,7 +563,8 @@ class RabbitMQService:
|
|
|
541
563
|
|
|
542
564
|
# 记录消费者标签
|
|
543
565
|
cls._consumer_tags[client_name] = consumer_tag
|
|
544
|
-
logger.info(
|
|
566
|
+
logger.info(
|
|
567
|
+
f"消费者 '{client_name}' 开始消费,tag: {consumer_tag}(将处理队列中所有积压消息)")
|
|
545
568
|
|
|
546
569
|
# 等待停止事件
|
|
547
570
|
await stop_event.wait()
|
|
@@ -621,7 +644,7 @@ class RabbitMQService:
|
|
|
621
644
|
else:
|
|
622
645
|
logger.info(f"发送器 '{queue_name}' 连接已断开,尝试重连")
|
|
623
646
|
try:
|
|
624
|
-
await client.connect(declare_queue=False
|
|
647
|
+
await client.connect() # 移除declare_queue=False
|
|
625
648
|
if await client.is_connected:
|
|
626
649
|
return client
|
|
627
650
|
except Exception as e:
|
|
@@ -639,7 +662,7 @@ class RabbitMQService:
|
|
|
639
662
|
else:
|
|
640
663
|
logger.info(f"发送器 '{suffixed_name}' 连接已断开,尝试重连")
|
|
641
664
|
try:
|
|
642
|
-
await client.connect(declare_queue=False
|
|
665
|
+
await client.connect() # 移除declare_queue=False
|
|
643
666
|
if await client.is_connected:
|
|
644
667
|
return client
|
|
645
668
|
except Exception as e:
|
|
@@ -675,7 +698,7 @@ class RabbitMQService:
|
|
|
675
698
|
while retry_count < max_retry and not cls._is_shutdown:
|
|
676
699
|
try:
|
|
677
700
|
# 尝试重连,每次重试间隔1秒
|
|
678
|
-
await sender.connect(force_reconnect
|
|
701
|
+
await sender.connect() # 移除force_reconnect和declare_queue
|
|
679
702
|
if await sender.is_connected:
|
|
680
703
|
logger.info(
|
|
681
704
|
f"发送器 '{queue_name}' 第 {retry_count + 1} 次重连成功")
|