sycommon-python-lib 0.1.46__py3-none-any.whl → 0.1.57b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/config/Config.py +29 -4
- sycommon/config/LangfuseConfig.py +15 -0
- sycommon/config/RerankerConfig.py +1 -0
- sycommon/config/SentryConfig.py +13 -0
- sycommon/database/async_base_db_service.py +36 -0
- sycommon/database/async_database_service.py +96 -0
- sycommon/llm/__init__.py +0 -0
- sycommon/llm/embedding.py +204 -0
- sycommon/llm/get_llm.py +37 -0
- sycommon/llm/llm_logger.py +126 -0
- sycommon/llm/llm_tokens.py +119 -0
- sycommon/llm/struct_token.py +192 -0
- sycommon/llm/sy_langfuse.py +103 -0
- sycommon/llm/usage_token.py +117 -0
- sycommon/logging/async_sql_logger.py +65 -0
- sycommon/logging/kafka_log.py +200 -434
- sycommon/logging/logger_levels.py +23 -0
- sycommon/middleware/context.py +2 -0
- sycommon/middleware/exception.py +10 -16
- sycommon/middleware/timeout.py +2 -1
- sycommon/middleware/traceid.py +179 -51
- sycommon/notice/__init__.py +0 -0
- sycommon/notice/uvicorn_monitor.py +200 -0
- sycommon/rabbitmq/rabbitmq_client.py +267 -290
- sycommon/rabbitmq/rabbitmq_pool.py +277 -465
- sycommon/rabbitmq/rabbitmq_service.py +23 -891
- sycommon/rabbitmq/rabbitmq_service_client_manager.py +211 -0
- sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +73 -0
- sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +285 -0
- sycommon/rabbitmq/rabbitmq_service_core.py +117 -0
- sycommon/rabbitmq/rabbitmq_service_producer_manager.py +238 -0
- sycommon/sentry/__init__.py +0 -0
- sycommon/sentry/sy_sentry.py +35 -0
- sycommon/services.py +144 -115
- sycommon/synacos/feign.py +18 -7
- sycommon/synacos/feign_client.py +26 -8
- sycommon/synacos/nacos_client_base.py +119 -0
- sycommon/synacos/nacos_config_manager.py +107 -0
- sycommon/synacos/nacos_heartbeat_manager.py +144 -0
- sycommon/synacos/nacos_service.py +65 -769
- sycommon/synacos/nacos_service_discovery.py +157 -0
- sycommon/synacos/nacos_service_registration.py +270 -0
- sycommon/tools/env.py +62 -0
- sycommon/tools/merge_headers.py +117 -0
- sycommon/tools/snowflake.py +238 -23
- {sycommon_python_lib-0.1.46.dist-info → sycommon_python_lib-0.1.57b1.dist-info}/METADATA +18 -11
- sycommon_python_lib-0.1.57b1.dist-info/RECORD +89 -0
- sycommon_python_lib-0.1.46.dist-info/RECORD +0 -59
- {sycommon_python_lib-0.1.46.dist-info → sycommon_python_lib-0.1.57b1.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.46.dist-info → sycommon_python_lib-0.1.57b1.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.46.dist-info → sycommon_python_lib-0.1.57b1.dist-info}/top_level.txt +0 -0
|
@@ -1,19 +1,29 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
|
|
3
|
-
from
|
|
2
|
+
import random
|
|
3
|
+
from typing import Optional, List, Dict, Callable, Tuple
|
|
4
|
+
from aio_pika import connect_robust, RobustChannel, Message
|
|
4
5
|
from aio_pika.abc import (
|
|
5
6
|
AbstractRobustConnection, AbstractQueue, AbstractExchange, AbstractMessage
|
|
6
7
|
)
|
|
7
|
-
from aio_pika.exceptions import ChannelClosed
|
|
8
|
-
import aiormq.exceptions
|
|
9
|
-
|
|
10
8
|
from sycommon.logging.kafka_log import SYLogger
|
|
11
9
|
|
|
12
10
|
logger = SYLogger
|
|
13
11
|
|
|
14
12
|
|
|
13
|
+
class AsyncProperty:
|
|
14
|
+
"""实现 await obj.attr 的支持"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, method):
|
|
17
|
+
self.method = method
|
|
18
|
+
|
|
19
|
+
def __get__(self, obj, objtype=None):
|
|
20
|
+
if obj is None:
|
|
21
|
+
return self
|
|
22
|
+
return self.method(obj)
|
|
23
|
+
|
|
24
|
+
|
|
15
25
|
class RabbitMQConnectionPool:
|
|
16
|
-
"""
|
|
26
|
+
"""单连接单通道RabbitMQ客户端 (严格执行“先清理后连接”策略)"""
|
|
17
27
|
|
|
18
28
|
def __init__(
|
|
19
29
|
self,
|
|
@@ -22,14 +32,12 @@ class RabbitMQConnectionPool:
|
|
|
22
32
|
username: str,
|
|
23
33
|
password: str,
|
|
24
34
|
virtualhost: str = "/",
|
|
25
|
-
|
|
26
|
-
heartbeat: int = 30,
|
|
35
|
+
heartbeat: int = 15,
|
|
27
36
|
app_name: str = "",
|
|
28
|
-
connection_timeout: int =
|
|
29
|
-
reconnect_interval: int =
|
|
37
|
+
connection_timeout: int = 15,
|
|
38
|
+
reconnect_interval: int = 5,
|
|
30
39
|
prefetch_count: int = 2,
|
|
31
40
|
):
|
|
32
|
-
# 基础配置校验与初始化
|
|
33
41
|
self.hosts = [host.strip() for host in hosts if host.strip()]
|
|
34
42
|
if not self.hosts:
|
|
35
43
|
raise ValueError("至少需要提供一个RabbitMQ主机地址")
|
|
@@ -43,544 +51,348 @@ class RabbitMQConnectionPool:
|
|
|
43
51
|
self.connection_timeout = connection_timeout
|
|
44
52
|
self.reconnect_interval = reconnect_interval
|
|
45
53
|
self.prefetch_count = prefetch_count
|
|
46
|
-
self.channel_pool_size = max(1, channel_pool_size) # 确保池大小不小于1
|
|
47
54
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
self._current_host: Optional[str] = None
|
|
55
|
+
self._current_host: str = random.choice(self.hosts)
|
|
56
|
+
logger.info(f"[INIT] 随机选择RabbitMQ主机: {self._current_host}")
|
|
51
57
|
|
|
52
|
-
#
|
|
53
|
-
self._connection: Optional[AbstractRobustConnection] = None
|
|
54
|
-
self.
|
|
55
|
-
self.
|
|
56
|
-
self._consumer_channels: Dict[str, Tuple[Channel,
|
|
57
|
-
AbstractRobustConnection, Callable, bool, dict]] = {} # 消费者通道跟踪
|
|
58
|
+
# 核心资源
|
|
59
|
+
self._connection: Optional[AbstractRobustConnection] = None
|
|
60
|
+
self._channel: Optional[RobustChannel] = None
|
|
61
|
+
self._consumer_channels: Dict[str, RobustChannel] = {}
|
|
58
62
|
|
|
59
|
-
#
|
|
60
|
-
self._lock = asyncio.Lock()
|
|
63
|
+
# 状态控制
|
|
64
|
+
self._lock = asyncio.Lock()
|
|
61
65
|
self._initialized = False
|
|
62
66
|
self._is_shutdown = False
|
|
63
|
-
self._reconnecting = False # 避免并发重连
|
|
64
|
-
self._connection_version = 0 # 连接版本号(区分新旧连接/通道)
|
|
65
|
-
|
|
66
|
-
def _create_host_iterator(self) -> Iterator[str]:
|
|
67
|
-
"""创建无限循环的节点轮询迭代器"""
|
|
68
|
-
while True:
|
|
69
|
-
for host in self.hosts:
|
|
70
|
-
yield host
|
|
71
|
-
|
|
72
|
-
async def _is_connection_valid(self) -> bool:
|
|
73
|
-
"""原子化检查连接有效性(加锁保证无竞态)"""
|
|
74
|
-
async with self._lock:
|
|
75
|
-
return (
|
|
76
|
-
self._connection is not None
|
|
77
|
-
and not self._connection.is_closed
|
|
78
|
-
and not self._reconnecting
|
|
79
|
-
)
|
|
80
67
|
|
|
81
|
-
@
|
|
68
|
+
@AsyncProperty
|
|
82
69
|
async def is_alive(self) -> bool:
|
|
83
|
-
"""
|
|
84
|
-
if self._is_shutdown:
|
|
85
|
-
return False
|
|
86
|
-
return await self._is_connection_valid()
|
|
87
|
-
|
|
88
|
-
async def _safe_close_old_resources(self):
|
|
89
|
-
"""强制关闭所有旧资源(加锁保证原子性,重连前必调用)"""
|
|
70
|
+
"""对外暴露的连接存活状态"""
|
|
90
71
|
async with self._lock:
|
|
91
|
-
|
|
72
|
+
if self._is_shutdown:
|
|
73
|
+
return False
|
|
74
|
+
if not self._initialized:
|
|
75
|
+
return False
|
|
76
|
+
if self._connection is None or self._connection.is_closed:
|
|
77
|
+
return False
|
|
78
|
+
if self._channel is None or self._channel.is_closed:
|
|
79
|
+
return False
|
|
80
|
+
return True
|
|
92
81
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
except Exception as e:
|
|
100
|
-
logger.warning(f"关闭消费者通道 {queue_name} 失败: {str(e)}")
|
|
101
|
-
self._consumer_channels.clear()
|
|
82
|
+
async def _cleanup_resources(self):
|
|
83
|
+
"""
|
|
84
|
+
彻底清理旧资源
|
|
85
|
+
必须在持有 self._lock 的情况下调用
|
|
86
|
+
"""
|
|
87
|
+
logger.info("🧹 [CLEANUP] 开始清理旧资源...")
|
|
102
88
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
if not channel.is_closed:
|
|
108
|
-
await channel.close()
|
|
109
|
-
except Exception as e:
|
|
110
|
-
logger.warning(f"关闭旧通道失败: {str(e)}")
|
|
111
|
-
self._free_channels.clear()
|
|
112
|
-
self._used_channels.clear()
|
|
89
|
+
# 1. 清理所有消费者通道
|
|
90
|
+
if self._consumer_channels:
|
|
91
|
+
channels_to_close = list(self._consumer_channels.values())
|
|
92
|
+
self._consumer_channels.clear()
|
|
113
93
|
|
|
114
|
-
|
|
115
|
-
if self._connection:
|
|
94
|
+
for ch in channels_to_close:
|
|
116
95
|
try:
|
|
117
|
-
if not
|
|
118
|
-
await
|
|
119
|
-
logger.info(
|
|
120
|
-
f"已关闭旧连接: {self._current_host}:{self.port}(版本: {self._connection_version})")
|
|
96
|
+
if not ch.is_closed:
|
|
97
|
+
await ch.close()
|
|
121
98
|
except Exception as e:
|
|
122
|
-
logger.warning(f"
|
|
123
|
-
self._connection = None # 置空,确保单连接
|
|
124
|
-
|
|
125
|
-
logger.info("旧资源释放完成(所有旧自动恢复逻辑已终止)")
|
|
126
|
-
|
|
127
|
-
async def _create_single_connection(self) -> AbstractRobustConnection:
|
|
128
|
-
"""创建唯一活跃连接(重连前已释放旧资源,确保单连接)"""
|
|
129
|
-
max_attempts = len(self.hosts) * 2 # 每个节点尝试2次
|
|
130
|
-
attempts = 0
|
|
131
|
-
last_error: Optional[Exception] = None
|
|
132
|
-
|
|
133
|
-
while attempts < max_attempts and not self._is_shutdown:
|
|
134
|
-
self._current_host = next(self._host_iterator)
|
|
135
|
-
conn_url = f"amqp://{self.username}:{self.password}@{self._current_host}:{self.port}/{self.virtualhost}"
|
|
99
|
+
logger.warning(f"⚠️ [CLEANUP_CH] 关闭消费者通道失败: {e}")
|
|
136
100
|
|
|
101
|
+
# 2. 关闭主通道
|
|
102
|
+
if self._channel:
|
|
137
103
|
try:
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
f"尝试创建连接: {self._current_host}:{self.port} "
|
|
141
|
-
f"(目标版本: {target_version},{attempts+1}/{max_attempts})"
|
|
142
|
-
)
|
|
143
|
-
# 创建连接(保留aio-pika原生自动恢复)
|
|
144
|
-
conn = await connect_robust(
|
|
145
|
-
conn_url,
|
|
146
|
-
properties={
|
|
147
|
-
"connection_name": f"{self.app_name}_conn_v{target_version}",
|
|
148
|
-
"product": self.app_name
|
|
149
|
-
},
|
|
150
|
-
heartbeat=self.heartbeat,
|
|
151
|
-
timeout=self.connection_timeout,
|
|
152
|
-
reconnect_interval=5, # 单节点内部短间隔重连(原生自动恢复)
|
|
153
|
-
max_reconnect_attempts=3, # 单节点最大重试3次
|
|
154
|
-
)
|
|
155
|
-
logger.info(
|
|
156
|
-
f"连接创建成功: {self._current_host}:{self.port}(版本: {target_version})")
|
|
157
|
-
return conn
|
|
104
|
+
if not self._channel.is_closed:
|
|
105
|
+
await self._channel.close()
|
|
158
106
|
except Exception as e:
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
f"连接节点 {self._current_host}:{self.port} 失败({attempts}/{max_attempts}): {str(e)}",
|
|
163
|
-
exc_info=True
|
|
164
|
-
)
|
|
165
|
-
await asyncio.sleep(min(5 * attempts, self.reconnect_interval))
|
|
166
|
-
|
|
167
|
-
raise ConnectionError(
|
|
168
|
-
f"所有节点创建连接失败(节点列表: {self.hosts})"
|
|
169
|
-
) from last_error
|
|
170
|
-
|
|
171
|
-
async def _init_channel_pool(self):
|
|
172
|
-
"""初始化通道池(加锁保证原子性,绑定当前连接)"""
|
|
173
|
-
async with self._lock:
|
|
174
|
-
if self._is_shutdown:
|
|
175
|
-
raise RuntimeError("通道池已关闭,无法初始化")
|
|
107
|
+
logger.warning(f"⚠️ [CLEANUP_MAIN_CH] 关闭主通道失败: {e}")
|
|
108
|
+
finally:
|
|
109
|
+
self._channel = None
|
|
176
110
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
channel = await self._connection.channel() # 新通道自带自动恢复
|
|
188
|
-
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
189
|
-
self._free_channels.append(channel)
|
|
190
|
-
except Exception as e:
|
|
191
|
-
logger.error(
|
|
192
|
-
f"创建通道失败(第{i+1}个,连接版本: {self._connection_version}): {str(e)}",
|
|
193
|
-
exc_info=True
|
|
194
|
-
)
|
|
195
|
-
continue
|
|
196
|
-
|
|
197
|
-
logger.info(
|
|
198
|
-
f"通道池初始化完成 - 连接: {self._current_host}:{self.port}(版本: {self._connection_version}), "
|
|
199
|
-
f"可用通道数: {len(self._free_channels)}/{self.channel_pool_size}(均带自动恢复)"
|
|
200
|
-
)
|
|
111
|
+
# 3. 关闭连接
|
|
112
|
+
if self._connection:
|
|
113
|
+
try:
|
|
114
|
+
if not self._connection.is_closed:
|
|
115
|
+
# close() 可能是同步的,也可能是异步的,aio_pika 中通常是异步的
|
|
116
|
+
await self._connection.close()
|
|
117
|
+
except Exception as e:
|
|
118
|
+
logger.warning(f"⚠️ [CLEANUP_CONN] 关闭连接失败: {e}")
|
|
119
|
+
finally:
|
|
120
|
+
self._connection = None
|
|
201
121
|
|
|
202
|
-
|
|
203
|
-
"""连接失效时重连(加锁保护,严格单连接+释放旧资源)"""
|
|
204
|
-
# 快速判断,避免无效加锁
|
|
205
|
-
if self._is_shutdown or self._reconnecting:
|
|
206
|
-
return False
|
|
122
|
+
logger.info("✅ [CLEANUP] 资源清理完成")
|
|
207
123
|
|
|
208
|
-
|
|
124
|
+
async def _create_connection_impl(self, host: str) -> AbstractRobustConnection:
|
|
125
|
+
conn_url = (
|
|
126
|
+
f"amqp://{self.username}:{self.password}@{host}:{self.port}/"
|
|
127
|
+
f"{self.virtualhost}?name={self.app_name}&heartbeat={self.heartbeat}"
|
|
128
|
+
f"&reconnect_interval={self.reconnect_interval}&fail_fast=1"
|
|
129
|
+
)
|
|
130
|
+
logger.info(f"🔌 [CONNECT] 尝试连接节点: {host}")
|
|
209
131
|
try:
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
async with self._lock:
|
|
217
|
-
self._connection_version += 1
|
|
218
|
-
target_version = self._connection_version
|
|
219
|
-
|
|
220
|
-
# 3. 创建新连接(保留原生自动恢复)
|
|
221
|
-
new_conn = await self._create_single_connection()
|
|
222
|
-
|
|
223
|
-
# 4. 绑定新连接(加锁保证原子性)
|
|
224
|
-
async with self._lock:
|
|
225
|
-
self._connection = new_conn
|
|
226
|
-
|
|
227
|
-
# 5. 重新初始化通道池(新通道带自动恢复)
|
|
228
|
-
await self._init_channel_pool()
|
|
229
|
-
|
|
230
|
-
# 6. 恢复消费者通道(新通道带自动恢复)
|
|
231
|
-
await self._restore_consumer_channels()
|
|
232
|
-
|
|
233
|
-
logger.info(f"重连成功(新连接版本: {target_version}),所有通道均带自动恢复")
|
|
234
|
-
async with self._lock:
|
|
235
|
-
self._initialized = True # 重连成功后标记为已初始化
|
|
236
|
-
return True
|
|
132
|
+
conn = await asyncio.wait_for(
|
|
133
|
+
connect_robust(conn_url),
|
|
134
|
+
timeout=self.connection_timeout + 5
|
|
135
|
+
)
|
|
136
|
+
logger.info(f"✅ [CONNECT_OK] 节点连接成功: {host}")
|
|
137
|
+
return conn
|
|
237
138
|
except Exception as e:
|
|
238
|
-
logger.error(f"
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
139
|
+
logger.error(f"❌ [CONNECT_FAIL] 节点 {host} 连接失败: {str(e)}")
|
|
140
|
+
raise ConnectionError(f"无法连接RabbitMQ {host}") from e
|
|
141
|
+
|
|
142
|
+
async def _ensure_main_channel(self) -> RobustChannel:
|
|
143
|
+
"""
|
|
144
|
+
确保主通道有效
|
|
145
|
+
逻辑:
|
|
146
|
+
1. 检查连接状态
|
|
147
|
+
2. 如果断开 -> 清理 -> 轮询重试
|
|
148
|
+
3. 如果连接在但通道断开 -> 仅重建通道
|
|
149
|
+
"""
|
|
247
150
|
async with self._lock:
|
|
248
|
-
if
|
|
249
|
-
|
|
250
|
-
logger.info(
|
|
251
|
-
f"开始恢复 {len(self._consumer_channels)} 个消费者通道(连接版本: {self._connection_version})")
|
|
151
|
+
if self._is_shutdown:
|
|
152
|
+
raise RuntimeError("客户端已关闭")
|
|
252
153
|
|
|
253
|
-
#
|
|
254
|
-
|
|
255
|
-
self._consumer_channels.clear()
|
|
154
|
+
# --- 阶段A:连接恢复逻辑 (如果连接断了) ---
|
|
155
|
+
if self._connection is None or self._connection.is_closed:
|
|
256
156
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
try:
|
|
260
|
-
await self.consume_queue(queue_name, callback, auto_ack, **kwargs)
|
|
261
|
-
except Exception as e:
|
|
262
|
-
logger.error(
|
|
263
|
-
f"恢复消费者队列 {queue_name} 失败: {str(e)}", exc_info=True)
|
|
157
|
+
# 1. 【强制】先彻底清理所有旧资源
|
|
158
|
+
await self._cleanup_resources()
|
|
264
159
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
return
|
|
160
|
+
retry_hosts = self.hosts.copy()
|
|
161
|
+
random.shuffle(retry_hosts)
|
|
162
|
+
last_error = None
|
|
163
|
+
max_attempts = min(len(retry_hosts), 3)
|
|
270
164
|
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
and not self._reconnecting
|
|
276
|
-
)
|
|
277
|
-
if not current_valid:
|
|
278
|
-
# 连接失效,触发重连(不加锁,避免死锁)
|
|
279
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
280
|
-
return
|
|
165
|
+
# 2. 轮询尝试新连接
|
|
166
|
+
for _ in range(max_attempts):
|
|
167
|
+
if not retry_hosts:
|
|
168
|
+
break
|
|
281
169
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
try:
|
|
286
|
-
if not chan.is_closed and chan.connection == self._connection:
|
|
287
|
-
valid_free.append(chan)
|
|
288
|
-
else:
|
|
289
|
-
logger.warning(f"清理失效空闲通道(连接版本不匹配或已关闭)")
|
|
290
|
-
except Exception:
|
|
291
|
-
logger.warning(f"清理异常空闲通道")
|
|
292
|
-
self._free_channels = valid_free
|
|
170
|
+
host = retry_hosts.pop()
|
|
171
|
+
self._current_host = host
|
|
172
|
+
temp_conn = None
|
|
293
173
|
|
|
294
|
-
# 3. 清理使用中通道(仅保留当前连接的有效通道)
|
|
295
|
-
valid_used = set()
|
|
296
|
-
for chan in self._used_channels:
|
|
297
|
-
try:
|
|
298
|
-
if not chan.is_closed and chan.connection == self._connection:
|
|
299
|
-
valid_used.add(chan)
|
|
300
|
-
else:
|
|
301
|
-
logger.warning(f"清理失效使用中通道(连接版本不匹配或已关闭)")
|
|
302
|
-
except Exception:
|
|
303
|
-
logger.warning(f"清理异常使用中通道")
|
|
304
|
-
self._used_channels = valid_used
|
|
305
|
-
|
|
306
|
-
# 4. 补充通道到指定大小(新通道带自动恢复)
|
|
307
|
-
total_valid = len(self._free_channels) + len(self._used_channels)
|
|
308
|
-
missing = self.channel_pool_size - total_valid
|
|
309
|
-
if missing > 0:
|
|
310
|
-
logger.info(
|
|
311
|
-
f"通道池缺少{missing}个通道,补充中(连接版本: {self._connection_version})...")
|
|
312
|
-
for _ in range(missing):
|
|
313
174
|
try:
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
self.
|
|
317
|
-
|
|
318
|
-
|
|
175
|
+
temp_conn = await self._create_connection_impl(host)
|
|
176
|
+
|
|
177
|
+
# 3. 只有在连接成功后,才更新 self._connection
|
|
178
|
+
self._connection = temp_conn
|
|
179
|
+
temp_conn = None # 转移所有权
|
|
180
|
+
self._initialized = True
|
|
181
|
+
last_error = None
|
|
182
|
+
logger.info(f"🔗 [RECONNECT_OK] 切换到节点: {host}")
|
|
319
183
|
break
|
|
320
184
|
|
|
185
|
+
except Exception as e:
|
|
186
|
+
logger.warning(f"⚠️ [RECONNECT_RETRY] 节点 {host} 不可用")
|
|
187
|
+
if temp_conn is not None:
|
|
188
|
+
# 尝试连接失败了,必须把这个“半成品”连接关掉
|
|
189
|
+
try:
|
|
190
|
+
await temp_conn.close()
|
|
191
|
+
except Exception:
|
|
192
|
+
pass
|
|
193
|
+
last_error = e
|
|
194
|
+
await asyncio.sleep(self.reconnect_interval)
|
|
195
|
+
|
|
196
|
+
# 4. 如果所有尝试都失败
|
|
197
|
+
if last_error:
|
|
198
|
+
# 确保状态是干净的
|
|
199
|
+
self._connection = None
|
|
200
|
+
self._initialized = False
|
|
201
|
+
logger.error("💥 [RECONNECT_FATAL] 所有节点重试失败")
|
|
202
|
+
raise ConnectionError("所有 RabbitMQ 节点连接失败") from last_error
|
|
203
|
+
|
|
204
|
+
# --- 阶段B:通道恢复逻辑 (如果连接在但通道断了) ---
|
|
205
|
+
# 注意:这里不需要清理连接,只重置通道
|
|
206
|
+
if self._channel is None or self._channel.is_closed:
|
|
207
|
+
try:
|
|
208
|
+
self._channel = await self._connection.channel()
|
|
209
|
+
await self._channel.set_qos(prefetch_count=self.prefetch_count)
|
|
210
|
+
logger.info(f"✅ [CHANNEL_OK] 主通道已恢复")
|
|
211
|
+
except Exception as e:
|
|
212
|
+
# 如果连通道都创建不了,说明这个连接也是坏的,回滚到阶段A
|
|
213
|
+
logger.error(f"❌ [CHANNEL_FAIL] 通道创建失败,标记连接无效: {e}")
|
|
214
|
+
# 强制清理连接,触发下一次进入阶段A
|
|
215
|
+
await self._cleanup_resources()
|
|
216
|
+
raise
|
|
217
|
+
|
|
218
|
+
return self._channel
|
|
219
|
+
|
|
321
220
|
async def init_pools(self):
|
|
322
|
-
"""
|
|
221
|
+
"""初始化入口"""
|
|
323
222
|
async with self._lock:
|
|
223
|
+
if self._is_shutdown:
|
|
224
|
+
raise RuntimeError("客户端已关闭")
|
|
324
225
|
if self._initialized:
|
|
325
|
-
logger.warning("通道池已初始化,无需重复调用")
|
|
326
226
|
return
|
|
327
|
-
if self._is_shutdown:
|
|
328
|
-
raise RuntimeError("通道池已关闭,无法初始化")
|
|
329
227
|
|
|
330
|
-
try
|
|
331
|
-
|
|
332
|
-
new_conn = await self._create_single_connection()
|
|
228
|
+
# 在 try 之前声明变量,确保 except 块能访问
|
|
229
|
+
conn_created_in_this_try = None
|
|
333
230
|
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
231
|
+
try:
|
|
232
|
+
# 锁外创建连接,减少锁持有时间
|
|
233
|
+
init_host = random.choice(self.hosts)
|
|
234
|
+
conn = await self._create_connection_impl(init_host)
|
|
338
235
|
|
|
339
|
-
#
|
|
340
|
-
|
|
236
|
+
# 记录本次创建的连接
|
|
237
|
+
conn_created_in_this_try = conn
|
|
341
238
|
|
|
342
|
-
# 4. 标记为已初始化(加锁保证原子性)
|
|
343
239
|
async with self._lock:
|
|
240
|
+
if self._is_shutdown:
|
|
241
|
+
raise RuntimeError("客户端已关闭")
|
|
242
|
+
|
|
243
|
+
# 提交新资源
|
|
244
|
+
self._connection = conn
|
|
245
|
+
self._channel = await self._connection.channel()
|
|
246
|
+
await self._channel.set_qos(prefetch_count=self.prefetch_count)
|
|
344
247
|
self._initialized = True
|
|
345
248
|
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
logger.error(f"初始化失败: {str(e)}", exc_info=True)
|
|
349
|
-
await self._safe_close_old_resources()
|
|
350
|
-
raise
|
|
249
|
+
# 所有权转移成功,清空临时引用,防止 finally 重复关闭
|
|
250
|
+
conn_created_in_this_try = None
|
|
351
251
|
|
|
352
|
-
|
|
353
|
-
"""获取通道(加锁保证原子性,返回当前连接+带自动恢复的通道)"""
|
|
354
|
-
# 快速判断,避免无效加锁
|
|
355
|
-
async with self._lock:
|
|
356
|
-
if not self._initialized:
|
|
357
|
-
raise RuntimeError("通道池未初始化,请先调用init_pools()")
|
|
358
|
-
if self._is_shutdown:
|
|
359
|
-
raise RuntimeError("通道池已关闭,无法获取通道")
|
|
252
|
+
logger.info(f"🚀 [INIT_OK] 连接池初始化完成: {init_host}")
|
|
360
253
|
|
|
361
|
-
|
|
362
|
-
|
|
254
|
+
except Exception as e:
|
|
255
|
+
logger.error(f"💥 [INIT_FAIL] 初始化异常: {str(e)}")
|
|
363
256
|
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
)
|
|
371
|
-
if not current_valid:
|
|
372
|
-
# 连接失效,触发重连(不加锁,避免死锁)
|
|
373
|
-
reconnect_success = await self._reconnect_if_needed()
|
|
374
|
-
if not reconnect_success:
|
|
375
|
-
raise RuntimeError("连接失效且重连失败,无法获取通道")
|
|
376
|
-
|
|
377
|
-
# 优先从空闲池获取(带自动恢复的通道)
|
|
378
|
-
if self._free_channels:
|
|
379
|
-
channel = self._free_channels.pop()
|
|
380
|
-
self._used_channels.add(channel)
|
|
381
|
-
return channel, self._connection
|
|
382
|
-
|
|
383
|
-
# 通道池已满,创建临时通道(带自动恢复,用完关闭)
|
|
384
|
-
try:
|
|
385
|
-
channel = await self._connection.channel() # 临时通道带自动恢复
|
|
386
|
-
await channel.set_qos(prefetch_count=self.prefetch_count)
|
|
387
|
-
self._used_channels.add(channel)
|
|
388
|
-
logger.warning(
|
|
389
|
-
f"通道池已达上限({self.channel_pool_size}),创建临时通道(带自动恢复,用完自动关闭)"
|
|
390
|
-
)
|
|
391
|
-
return channel, self._connection
|
|
392
|
-
except Exception as e:
|
|
393
|
-
logger.error(f"获取通道失败: {str(e)}", exc_info=True)
|
|
394
|
-
raise
|
|
257
|
+
# 这里现在可以合法访问 conn_created_in_this_try
|
|
258
|
+
if conn_created_in_this_try is not None:
|
|
259
|
+
try:
|
|
260
|
+
await conn_created_in_this_try.close()
|
|
261
|
+
except Exception:
|
|
262
|
+
pass
|
|
395
263
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
if not channel or not conn or self._is_shutdown:
|
|
400
|
-
return
|
|
264
|
+
if not self._is_shutdown:
|
|
265
|
+
await self.close()
|
|
266
|
+
raise
|
|
401
267
|
|
|
268
|
+
async def force_reconnect(self):
|
|
269
|
+
"""
|
|
270
|
+
强制重连
|
|
271
|
+
严格执行:清理所有资源 -> 尝试建立新资源
|
|
272
|
+
"""
|
|
402
273
|
async with self._lock:
|
|
403
|
-
|
|
404
|
-
if conn != self._connection:
|
|
405
|
-
try:
|
|
406
|
-
if not channel.is_closed:
|
|
407
|
-
await channel.close()
|
|
408
|
-
logger.warning(f"已关闭非当前连接的通道(版本不匹配,自动恢复终止)")
|
|
409
|
-
except Exception as e:
|
|
410
|
-
logger.warning(f"关闭非当前连接通道失败: {str(e)}")
|
|
274
|
+
if self._is_shutdown:
|
|
411
275
|
return
|
|
412
276
|
|
|
413
|
-
|
|
414
|
-
if channel not in self._used_channels:
|
|
415
|
-
return
|
|
277
|
+
logger.warning("🔄 [FORCE_RECONNECT] 开始强制重连...")
|
|
416
278
|
|
|
417
|
-
#
|
|
418
|
-
self.
|
|
279
|
+
# 1. 【关键】标记未初始化,迫使 _ensure_main_channel 走清理流程
|
|
280
|
+
self._initialized = False
|
|
419
281
|
|
|
420
|
-
#
|
|
421
|
-
|
|
422
|
-
self._connection is not None
|
|
423
|
-
and not self._connection.is_closed
|
|
424
|
-
and not self._reconnecting
|
|
425
|
-
)
|
|
426
|
-
if current_valid and not channel.is_closed and len(self._free_channels) < self.channel_pool_size:
|
|
427
|
-
self._free_channels.append(channel)
|
|
428
|
-
else:
|
|
429
|
-
# 无效通道直接关闭(终止自动恢复)
|
|
430
|
-
try:
|
|
431
|
-
if not channel.is_closed:
|
|
432
|
-
await channel.close()
|
|
433
|
-
except Exception as e:
|
|
434
|
-
logger.warning(f"关闭通道失败: {str(e)}")
|
|
282
|
+
# 2. 【关键】立即清理旧资源 (在锁内)
|
|
283
|
+
await self._cleanup_resources()
|
|
435
284
|
|
|
436
|
-
|
|
437
|
-
"""声明队列(使用池内通道,带自动恢复)"""
|
|
438
|
-
channel, conn = await self.acquire_channel()
|
|
439
|
-
try:
|
|
440
|
-
return await channel.declare_queue(queue_name, **kwargs)
|
|
441
|
-
finally:
|
|
442
|
-
await self.release_channel(channel, conn)
|
|
285
|
+
# 此时 self._connection 和 self._channel 均为 None
|
|
443
286
|
|
|
444
|
-
|
|
445
|
-
"""声明交换机(使用池内通道,带自动恢复)"""
|
|
446
|
-
channel, conn = await self.acquire_channel()
|
|
287
|
+
# 3. 锁外触发恢复 (避免阻塞锁太久)
|
|
447
288
|
try:
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
289
|
+
await self.acquire_channel()
|
|
290
|
+
logger.info("✅ [FORCE_RECONNECT_OK] 强制重连成功")
|
|
291
|
+
except Exception as e:
|
|
292
|
+
logger.error(f"❌ [FORCE_RECONNECT_FAIL] 强制重连失败: {e}")
|
|
293
|
+
raise
|
|
294
|
+
|
|
295
|
+
async def acquire_channel(self) -> Tuple[RobustChannel, AbstractRobustConnection]:
|
|
296
|
+
"""获取主通道"""
|
|
297
|
+
if not self._initialized and not self._is_shutdown:
|
|
298
|
+
await self.init_pools()
|
|
299
|
+
return await self._ensure_main_channel(), self._connection
|
|
451
300
|
|
|
452
301
|
async def publish_message(self, routing_key: str, message_body: bytes, exchange_name: str = "", **kwargs):
|
|
453
|
-
|
|
454
|
-
channel, conn = await self.acquire_channel()
|
|
302
|
+
channel, _ = await self.acquire_channel()
|
|
455
303
|
try:
|
|
456
304
|
exchange = channel.default_exchange if not exchange_name else await channel.get_exchange(exchange_name)
|
|
457
305
|
message = Message(body=message_body, **kwargs)
|
|
458
306
|
await exchange.publish(message, routing_key=routing_key)
|
|
459
|
-
logger.debug(
|
|
460
|
-
f"消息发布成功 - 连接: {self._current_host}:{self.port}(版本: {self._connection_version}), "
|
|
461
|
-
f"交换机: {exchange.name}, 路由键: {routing_key}"
|
|
462
|
-
)
|
|
463
307
|
except Exception as e:
|
|
464
|
-
logger.error(f"
|
|
465
|
-
# 发布失败触发重连(下次使用新通道)
|
|
466
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
308
|
+
logger.error(f"❌ [PUBLISH_FAIL] 发布失败: {str(e)}")
|
|
467
309
|
raise
|
|
468
|
-
finally:
|
|
469
|
-
await self.release_channel(channel, conn)
|
|
470
310
|
|
|
471
311
|
async def consume_queue(self, queue_name: str, callback: Callable[[AbstractMessage], asyncio.Future], auto_ack: bool = False, **kwargs):
|
|
472
|
-
|
|
473
|
-
|
|
312
|
+
if not self._initialized:
|
|
313
|
+
await self.init_pools()
|
|
314
|
+
|
|
315
|
+
# 检查是否已存在
|
|
474
316
|
async with self._lock:
|
|
475
|
-
if not self._initialized:
|
|
476
|
-
raise RuntimeError("通道池未初始化,请先调用init_pools()")
|
|
477
317
|
if self._is_shutdown:
|
|
478
|
-
raise RuntimeError("
|
|
318
|
+
raise RuntimeError("客户端已关闭")
|
|
319
|
+
if queue_name in self._consumer_channels:
|
|
320
|
+
logger.warning(f"⚠️ [CONSUMER_EXISTS] 队列 {queue_name} 已在消费中")
|
|
321
|
+
return
|
|
322
|
+
if not self._connection or self._connection.is_closed:
|
|
323
|
+
raise RuntimeError("连接不可用,无法启动消费")
|
|
479
324
|
|
|
480
|
-
#
|
|
325
|
+
# 声明队列 (使用主通道)
|
|
481
326
|
await self.declare_queue(queue_name, **kwargs)
|
|
482
327
|
|
|
483
|
-
|
|
484
|
-
|
|
328
|
+
try:
|
|
329
|
+
# 获取最新连接
|
|
330
|
+
_, conn = await self.acquire_channel()
|
|
485
331
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
self.
|
|
489
|
-
channel, conn, callback, auto_ack, kwargs)
|
|
332
|
+
# 创建消费者通道
|
|
333
|
+
consumer_channel = await conn.channel()
|
|
334
|
+
await consumer_channel.set_qos(prefetch_count=self.prefetch_count)
|
|
490
335
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
current_conn_valid = (
|
|
499
|
-
self._connection is not None
|
|
500
|
-
and not self._connection.is_closed
|
|
501
|
-
and not self._reconnecting
|
|
502
|
-
)
|
|
503
|
-
|
|
504
|
-
# 通道/连接失效,触发重连恢复
|
|
505
|
-
if not conn_matched or not channel_valid or not current_conn_valid:
|
|
506
|
-
logger.warning(
|
|
507
|
-
f"消费者通道 {queue_name} 失效(连接版本不匹配/通道关闭),触发重连恢复")
|
|
508
|
-
|
|
509
|
-
# 移除旧消费者记录(加锁保证原子性)
|
|
510
|
-
async with self._lock:
|
|
511
|
-
if self._consumer_channels.get(queue_name) == (channel, conn, callback, auto_ack, kwargs):
|
|
512
|
-
del self._consumer_channels[queue_name]
|
|
513
|
-
|
|
514
|
-
# 释放旧通道(加锁保证原子性)
|
|
515
|
-
await self.release_channel(channel, conn)
|
|
516
|
-
|
|
517
|
-
# 重新创建消费者(新通道带自动恢复)
|
|
518
|
-
asyncio.create_task(self.consume_queue(
|
|
519
|
-
queue_name, callback, auto_ack, **kwargs))
|
|
520
|
-
|
|
521
|
-
# Nack消息(避免丢失)
|
|
522
|
-
if not auto_ack:
|
|
523
|
-
await message.nack(requeue=True)
|
|
336
|
+
async with self._lock:
|
|
337
|
+
# 再次检查,防止并发创建
|
|
338
|
+
if self._is_shutdown:
|
|
339
|
+
await consumer_channel.close()
|
|
340
|
+
return
|
|
341
|
+
if queue_name in self._consumer_channels:
|
|
342
|
+
await consumer_channel.close() # 其他协程已经创建了
|
|
524
343
|
return
|
|
525
344
|
|
|
526
|
-
|
|
527
|
-
await callback(message)
|
|
528
|
-
if not auto_ack:
|
|
529
|
-
await message.ack()
|
|
530
|
-
except ChannelClosed as e:
|
|
531
|
-
logger.error(f"消费者通道 {queue_name} 关闭: {str(e)}", exc_info=True)
|
|
532
|
-
if not auto_ack:
|
|
533
|
-
await message.nack(requeue=True)
|
|
534
|
-
asyncio.create_task(self._reconnect_if_needed())
|
|
535
|
-
except aiormq.exceptions.ChannelInvalidStateError as e:
|
|
536
|
-
logger.error(
|
|
537
|
-
f"消费者通道 {queue_name} 状态异常: {str(e)}", exc_info=True)
|
|
538
|
-
if not auto_ack:
|
|
539
|
-
await message.nack(requeue=True)
|
|
540
|
-
asyncio.create_task(self.consume_queue(
|
|
541
|
-
queue_name, callback, auto_ack, **kwargs))
|
|
542
|
-
except Exception as e:
|
|
543
|
-
logger.error(
|
|
544
|
-
f"消费消息失败(队列: {queue_name}): {str(e)}", exc_info=True)
|
|
545
|
-
if not auto_ack:
|
|
546
|
-
await message.nack(requeue=True)
|
|
547
|
-
|
|
548
|
-
# 日志输出(加锁获取当前连接信息)
|
|
549
|
-
async with self._lock:
|
|
550
|
-
current_host = self._current_host
|
|
551
|
-
current_version = self._connection_version
|
|
345
|
+
self._consumer_channels[queue_name] = consumer_channel
|
|
552
346
|
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
347
|
+
async def consume_callback_wrapper(message: AbstractMessage):
|
|
348
|
+
try:
|
|
349
|
+
await callback(message)
|
|
350
|
+
if not auto_ack:
|
|
351
|
+
await message.ack()
|
|
352
|
+
except Exception as e:
|
|
353
|
+
logger.error(f"❌ [CALLBACK_ERR] {queue_name}: {e}")
|
|
354
|
+
if not auto_ack:
|
|
355
|
+
await message.nack(requeue=True)
|
|
557
356
|
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
await channel.basic_consume(
|
|
561
|
-
queue_name,
|
|
562
|
-
consumer_callback=consume_callback_wrapper,
|
|
563
|
-
auto_ack=auto_ack,
|
|
564
|
-
**kwargs
|
|
357
|
+
await consumer_channel.basic_consume(
|
|
358
|
+
queue_name, consumer_callback=consume_callback_wrapper, auto_ack=auto_ack
|
|
565
359
|
)
|
|
360
|
+
logger.info(f"🎧 [CONSUME_START] {queue_name}")
|
|
361
|
+
|
|
566
362
|
except Exception as e:
|
|
567
|
-
logger.error(f"
|
|
568
|
-
#
|
|
569
|
-
await self.release_channel(channel, conn)
|
|
363
|
+
logger.error(f"💥 [CONSUME_ERR] {queue_name}: {e}")
|
|
364
|
+
# 失败时清理字典
|
|
570
365
|
async with self._lock:
|
|
571
|
-
if self._consumer_channels
|
|
572
|
-
|
|
366
|
+
if queue_name in self._consumer_channels:
|
|
367
|
+
# 注意:这里清理的是字典里的引用,通道本身应该在 try 块里被关闭了吗?
|
|
368
|
+
# 如果 consumer_channel 创建成功但 basic_consume 失败,需要手动关闭
|
|
369
|
+
ch = self._consumer_channels.pop(queue_name, None)
|
|
370
|
+
if ch:
|
|
371
|
+
try:
|
|
372
|
+
await ch.close()
|
|
373
|
+
except:
|
|
374
|
+
pass
|
|
573
375
|
raise
|
|
574
376
|
|
|
575
377
|
async def close(self):
|
|
576
|
-
"""
|
|
378
|
+
"""资源销毁"""
|
|
577
379
|
async with self._lock:
|
|
578
380
|
if self._is_shutdown:
|
|
579
|
-
logger.warning("通道池已关闭,无需重复操作")
|
|
580
381
|
return
|
|
581
382
|
self._is_shutdown = True
|
|
383
|
+
self._initialized = False
|
|
384
|
+
|
|
385
|
+
logger.info("🛑 [CLOSE] 开始关闭连接池...")
|
|
386
|
+
|
|
387
|
+
# 1. 清理所有资源
|
|
388
|
+
await self._cleanup_resources()
|
|
582
389
|
|
|
583
|
-
logger.info("
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
390
|
+
logger.info("🏁 [CLOSE] 连接池已关闭")
|
|
391
|
+
|
|
392
|
+
async def declare_queue(self, queue_name: str, **kwargs) -> AbstractQueue:
|
|
393
|
+
channel, _ = await self.acquire_channel()
|
|
394
|
+
return await channel.declare_queue(queue_name, **kwargs)
|
|
395
|
+
|
|
396
|
+
async def declare_exchange(self, exchange_name: str, exchange_type: str = "direct", **kwargs) -> AbstractExchange:
|
|
397
|
+
channel, _ = await self.acquire_channel()
|
|
398
|
+
return await channel.declare_exchange(exchange_name, exchange_type, **kwargs)
|