sycommon-python-lib 0.2.0b6__tar.gz → 0.2.0b7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/PKG-INFO +2 -1
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/pyproject.toml +2 -1
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/embedding.py +74 -63
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_client.py +35 -114
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/PKG-INFO +2 -1
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/requires.txt +1 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/README.md +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/setup.cfg +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/command/cli.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/Config.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/DatabaseConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/EmbeddingConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/LLMConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/LangfuseConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/MQConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/RerankerConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/SentryConfig.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/database/async_base_db_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/database/async_database_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/database/base_db_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/database/database_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/health/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/health/health_check.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/health/metrics.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/health/ping.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/get_llm.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/llm_logger.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/llm_tokens.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/struct_token.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/sy_langfuse.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/usage_token.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/async_sql_logger.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/kafka_log.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/logger_levels.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/logger_wrapper.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/sql_logger.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/context.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/cors.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/docs.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/exception.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/middleware.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/monitor_memory.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/mq.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/timeout.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/traceid.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/base_http.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/log.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqlistener_config.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqmsg_model.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqsend_config.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/sso_user.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/notice/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/notice/uvicorn_monitor.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_pool.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service_core.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sentry/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sentry/sy_sentry.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/services.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sse/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sse/event.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sse/sse.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/example.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/example2.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/feign.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/feign_client.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_client_base.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_config_manager.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_heartbeat_manager.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_service.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_service_discovery.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_service_registration.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/param.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tests/test_email.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/__init__.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/docs.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/env.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/merge_headers.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/snowflake.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/syemail.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/timing.py +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/SOURCES.txt +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/dependency_links.txt +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/entry_points.txt +0 -0
- {sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon_python_lib.egg-info/top_level.txt +0 -0
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sycommon-python-lib
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.0b7
|
|
4
4
|
Summary: Add your description here
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: aio-pika>=9.5.8
|
|
8
8
|
Requires-Dist: aiohttp>=3.13.3
|
|
9
9
|
Requires-Dist: aiomysql>=0.3.2
|
|
10
|
+
Requires-Dist: anyio>=4.12.0
|
|
10
11
|
Requires-Dist: decorator>=5.2.1
|
|
11
12
|
Requires-Dist: fastapi>=0.128.0
|
|
12
13
|
Requires-Dist: kafka-python>=2.3.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sycommon-python-lib"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.0b7"
|
|
4
4
|
description = "Add your description here"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -8,6 +8,7 @@ dependencies = [
|
|
|
8
8
|
"aio-pika>=9.5.8",
|
|
9
9
|
"aiohttp>=3.13.3",
|
|
10
10
|
"aiomysql>=0.3.2",
|
|
11
|
+
"anyio>=4.12.0",
|
|
11
12
|
"decorator>=5.2.1",
|
|
12
13
|
"fastapi>=0.128.0",
|
|
13
14
|
"kafka-python>=2.3.0",
|
|
@@ -22,7 +22,7 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
22
22
|
self.reranker_base_url = RerankerConfig.from_config(
|
|
23
23
|
self.default_reranker_model).baseUrl
|
|
24
24
|
|
|
25
|
-
#
|
|
25
|
+
# 缓存配置URL
|
|
26
26
|
self._embedding_url_cache: Dict[str, str] = {
|
|
27
27
|
self.default_embedding_model: self.embeddings_base_url
|
|
28
28
|
}
|
|
@@ -30,25 +30,77 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
30
30
|
self.default_reranker_model: self.reranker_base_url
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
#
|
|
34
|
-
self.semaphore = asyncio.Semaphore(self.max_concurrency)
|
|
33
|
+
# 不存储 Semaphore 实例,动态获取当前 Loop 的
|
|
35
34
|
self.default_timeout = aiohttp.ClientTimeout(total=None)
|
|
36
|
-
|
|
37
|
-
# 核心优化:创建全局可复用的ClientSession(连接池复用)
|
|
38
35
|
self.session = None
|
|
39
36
|
|
|
40
|
-
# [修复] 注册退出钩子,确保程序结束时关闭连接池
|
|
41
37
|
atexit.register(self._sync_close_session)
|
|
42
38
|
|
|
39
|
+
def _get_loop_limiter(self):
|
|
40
|
+
"""获取当前 Loop 的原生 Semaphore"""
|
|
41
|
+
loop = asyncio.get_running_loop()
|
|
42
|
+
storage_key = '_embedding_limiter'
|
|
43
|
+
|
|
44
|
+
if not hasattr(loop, storage_key):
|
|
45
|
+
setattr(loop, storage_key, asyncio.Semaphore(self.max_concurrency))
|
|
46
|
+
return getattr(loop, storage_key)
|
|
47
|
+
|
|
43
48
|
async def init_session(self):
|
|
44
|
-
"""
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
"""
|
|
50
|
+
【终极修复】初始化全局ClientSession
|
|
51
|
+
"""
|
|
52
|
+
current_loop = asyncio.get_running_loop()
|
|
53
|
+
need_new = False
|
|
54
|
+
|
|
55
|
+
# 情况 1: Session 从未创建
|
|
56
|
+
if self.session is None:
|
|
57
|
+
need_new = True
|
|
58
|
+
|
|
59
|
+
# 情况 2: Session 已经被显式关闭
|
|
60
|
+
elif self.session.closed:
|
|
61
|
+
need_new = True
|
|
62
|
+
|
|
63
|
+
else:
|
|
64
|
+
# 情况 3: 【关键】检查 Session 绑定的 Loop 是否与当前 Loop 一致
|
|
65
|
+
# 这一步能捕获 "Loop 已关闭但对象还在" 的情况
|
|
66
|
+
try:
|
|
67
|
+
sess_loop = self.session._loop
|
|
68
|
+
|
|
69
|
+
# 如果 Loop 对象引用不同,说明跨线程了
|
|
70
|
+
if sess_loop is not current_loop:
|
|
71
|
+
SYLogger.warning(
|
|
72
|
+
f"⚠️ Session attached to a different loop "
|
|
73
|
+
f"(Old: {id(sess_loop)}, New: {id(current_loop)}). Recreating..."
|
|
74
|
+
)
|
|
75
|
+
need_new = True
|
|
76
|
+
|
|
77
|
+
# 如果 Loop 引用相同,但 Loop 自身已关闭
|
|
78
|
+
# 注意:这里需要检查 sess_loop.is_closed(),而不是 session.closed
|
|
79
|
+
elif sess_loop.is_closed():
|
|
80
|
+
SYLogger.warning(
|
|
81
|
+
"⚠️ Session attached to a closed loop. Recreating...")
|
|
82
|
+
need_new = True
|
|
83
|
+
|
|
84
|
+
except (AttributeError, RuntimeError) as e:
|
|
85
|
+
# 如果获取 _loop 失败,或者 Loop 已经完全不可用,重建
|
|
86
|
+
SYLogger.warning(
|
|
87
|
+
f"⚠️ Failed to check session loop state ({e}), Recreating...")
|
|
88
|
+
need_new = True
|
|
89
|
+
|
|
90
|
+
if need_new:
|
|
91
|
+
# 关闭旧 Session(如果存在且未关)
|
|
92
|
+
if self.session and not self.session.closed:
|
|
93
|
+
try:
|
|
94
|
+
await self.session.close()
|
|
95
|
+
except Exception:
|
|
96
|
+
pass
|
|
97
|
+
|
|
98
|
+
# 创建新 Session
|
|
47
99
|
connector = aiohttp.TCPConnector(
|
|
48
|
-
limit=self.max_concurrency,
|
|
49
|
-
limit_per_host=self.max_concurrency,
|
|
50
|
-
ttl_dns_cache=300,
|
|
51
|
-
enable_cleanup_closed=True
|
|
100
|
+
limit=self.max_concurrency,
|
|
101
|
+
limit_per_host=self.max_concurrency,
|
|
102
|
+
ttl_dns_cache=300,
|
|
103
|
+
enable_cleanup_closed=True
|
|
52
104
|
)
|
|
53
105
|
self.session = aiohttp.ClientSession(
|
|
54
106
|
connector=connector,
|
|
@@ -56,7 +108,7 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
56
108
|
)
|
|
57
109
|
|
|
58
110
|
async def close_session(self):
|
|
59
|
-
"""关闭全局Session
|
|
111
|
+
"""关闭全局Session"""
|
|
60
112
|
if self.session and not self.session.closed:
|
|
61
113
|
await self.session.close()
|
|
62
114
|
|
|
@@ -65,7 +117,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
65
117
|
try:
|
|
66
118
|
loop = asyncio.get_event_loop()
|
|
67
119
|
if loop.is_running():
|
|
68
|
-
# [修复] 修正缩进,确保 create_task 的异常能被捕获
|
|
69
120
|
try:
|
|
70
121
|
loop.create_task(self.close_session())
|
|
71
122
|
except Exception:
|
|
@@ -76,18 +127,15 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
76
127
|
except Exception:
|
|
77
128
|
pass
|
|
78
129
|
except Exception:
|
|
79
|
-
# 捕获获取 loop 时的异常
|
|
80
130
|
pass
|
|
81
131
|
|
|
82
132
|
def _get_embedding_url(self, model: str) -> str:
|
|
83
|
-
"""获取Embedding URL(带缓存)"""
|
|
84
133
|
if model not in self._embedding_url_cache:
|
|
85
134
|
self._embedding_url_cache[model] = EmbeddingConfig.from_config(
|
|
86
135
|
model).baseUrl
|
|
87
136
|
return self._embedding_url_cache[model]
|
|
88
137
|
|
|
89
138
|
def _get_reranker_url(self, model: str) -> str:
|
|
90
|
-
"""获取Reranker URL(带缓存)"""
|
|
91
139
|
if model not in self._reranker_url_cache:
|
|
92
140
|
self._reranker_url_cache[model] = RerankerConfig.from_config(
|
|
93
141
|
model).baseUrl
|
|
@@ -102,12 +150,13 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
102
150
|
**kwargs
|
|
103
151
|
):
|
|
104
152
|
"""embedding请求核心逻辑"""
|
|
105
|
-
await self.init_session()
|
|
106
|
-
|
|
153
|
+
await self.init_session()
|
|
154
|
+
limiter = self._get_loop_limiter()
|
|
155
|
+
|
|
156
|
+
async with limiter:
|
|
107
157
|
request_timeout = timeout or self.default_timeout
|
|
108
158
|
target_model = model or self.default_embedding_model
|
|
109
159
|
|
|
110
|
-
# [修复] 使用缓存获取URL
|
|
111
160
|
target_base_url = self._get_embedding_url(target_model)
|
|
112
161
|
url = f"{target_base_url}/v1/embeddings"
|
|
113
162
|
|
|
@@ -118,7 +167,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
118
167
|
}
|
|
119
168
|
request_body.update(kwargs)
|
|
120
169
|
|
|
121
|
-
# 复用全局Session
|
|
122
170
|
try:
|
|
123
171
|
async with self.session.post(
|
|
124
172
|
url,
|
|
@@ -127,7 +175,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
127
175
|
) as response:
|
|
128
176
|
if response.status != 200:
|
|
129
177
|
error_detail = await response.text()
|
|
130
|
-
# [日志] 记录详细的HTTP错误响应
|
|
131
178
|
SYLogger.error(
|
|
132
179
|
f"Embedding request HTTP Error. Status: {response.status}, "
|
|
133
180
|
f"Model: {target_model}, URL: {url}. Detail: {error_detail}"
|
|
@@ -135,14 +182,12 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
135
182
|
return None
|
|
136
183
|
return await response.json()
|
|
137
184
|
except (aiohttp.ClientConnectionResetError, asyncio.TimeoutError, aiohttp.ClientError) as e:
|
|
138
|
-
# [日志] 记录网络错误
|
|
139
185
|
SYLogger.error(
|
|
140
186
|
f"Embedding request Network Error. Model: {target_model}, URL: {url}. "
|
|
141
187
|
f"Error: {e.__class__.__name__} - {str(e)}"
|
|
142
188
|
)
|
|
143
189
|
return None
|
|
144
190
|
except Exception as e:
|
|
145
|
-
# 记录其他未预期的异常
|
|
146
191
|
SYLogger.error(
|
|
147
192
|
f"Unexpected error in _get_embeddings_http_core: {str(e)}", exc_info=True)
|
|
148
193
|
return None
|
|
@@ -171,12 +216,13 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
171
216
|
timeout: aiohttp.ClientTimeout = None, ** kwargs
|
|
172
217
|
):
|
|
173
218
|
"""reranker请求核心逻辑"""
|
|
174
|
-
await self.init_session()
|
|
175
|
-
|
|
219
|
+
await self.init_session()
|
|
220
|
+
limiter = self._get_loop_limiter()
|
|
221
|
+
|
|
222
|
+
async with limiter:
|
|
176
223
|
request_timeout = timeout or self.default_timeout
|
|
177
224
|
target_model = model or self.default_reranker_model
|
|
178
225
|
|
|
179
|
-
# [修复] 使用缓存获取URL
|
|
180
226
|
target_base_url = self._get_reranker_url(target_model)
|
|
181
227
|
url = f"{target_base_url}/v1/rerank"
|
|
182
228
|
|
|
@@ -191,7 +237,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
191
237
|
}
|
|
192
238
|
request_body.update(kwargs)
|
|
193
239
|
|
|
194
|
-
# 复用全局Session
|
|
195
240
|
try:
|
|
196
241
|
async with self.session.post(
|
|
197
242
|
url,
|
|
@@ -200,7 +245,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
200
245
|
) as response:
|
|
201
246
|
if response.status != 200:
|
|
202
247
|
error_detail = await response.text()
|
|
203
|
-
# [日志] 记录详细的HTTP错误响应
|
|
204
248
|
SYLogger.error(
|
|
205
249
|
f"Reranker request HTTP Error. Status: {response.status}, "
|
|
206
250
|
f"Model: {target_model}, URL: {url}. Detail: {error_detail}"
|
|
@@ -208,14 +252,12 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
208
252
|
return None
|
|
209
253
|
return await response.json()
|
|
210
254
|
except (aiohttp.ClientConnectionResetError, asyncio.TimeoutError, aiohttp.ClientError) as e:
|
|
211
|
-
# [日志] 记录网络错误
|
|
212
255
|
SYLogger.error(
|
|
213
256
|
f"Reranker request Network Error. Model: {target_model}, URL: {url}. "
|
|
214
257
|
f"Error: {e.__class__.__name__} - {str(e)}"
|
|
215
258
|
)
|
|
216
259
|
return None
|
|
217
260
|
except Exception as e:
|
|
218
|
-
# 记录其他未预期的异常
|
|
219
261
|
SYLogger.error(
|
|
220
262
|
f"Unexpected error in _get_reranker_http_core: {str(e)}", exc_info=True)
|
|
221
263
|
return None
|
|
@@ -245,7 +287,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
245
287
|
return int(config.dimension)
|
|
246
288
|
except Exception:
|
|
247
289
|
pass
|
|
248
|
-
# 默认兜底 1024
|
|
249
290
|
return 1024
|
|
250
291
|
|
|
251
292
|
async def get_embeddings(
|
|
@@ -254,16 +295,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
254
295
|
model: str = None,
|
|
255
296
|
timeout: Optional[Union[int, float]] = None
|
|
256
297
|
):
|
|
257
|
-
"""
|
|
258
|
-
获取语料库的嵌入向量,结果顺序与输入语料库顺序一致
|
|
259
|
-
|
|
260
|
-
Args:
|
|
261
|
-
corpus: 待生成嵌入向量的文本列表
|
|
262
|
-
model: 可选,指定使用的embedding模型名称,默认使用bge-large-zh-v1.5
|
|
263
|
-
timeout: 可选,超时时间(秒):
|
|
264
|
-
- 传int/float:表示总超时时间(秒)
|
|
265
|
-
- 不传/None:使用默认永不超时配置
|
|
266
|
-
"""
|
|
267
298
|
request_timeout = None
|
|
268
299
|
if timeout is not None:
|
|
269
300
|
if isinstance(timeout, (int, float)):
|
|
@@ -278,18 +309,13 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
278
309
|
f"Requesting embeddings for corpus: {len(corpus)} items (model: {actual_model}, max_concurrency: {self.max_concurrency}, timeout: {timeout or 'None'})")
|
|
279
310
|
|
|
280
311
|
all_vectors = []
|
|
281
|
-
|
|
282
|
-
# [修复] 增加 Chunk 处理逻辑,防止 corpus 过大导致内存溢出或协程过多
|
|
283
|
-
# 每次最多处理 max_concurrency * 2 个请求,避免一次性创建几十万个协程
|
|
284
312
|
batch_size = self.max_concurrency * 2
|
|
285
313
|
|
|
286
314
|
for i in range(0, len(corpus), batch_size):
|
|
287
315
|
batch_texts = corpus[i: i + batch_size]
|
|
288
|
-
|
|
289
316
|
SYLogger.info(
|
|
290
317
|
f"Requesting embeddings for text: {len(batch_texts)} items (model: {actual_model}, timeout: {timeout or 'None'})")
|
|
291
318
|
|
|
292
|
-
# 给每个异步任务传入模型名称和超时配置
|
|
293
319
|
tasks = [self._get_embeddings_http_async(
|
|
294
320
|
text, model=actual_model, timeout=request_timeout) for text in batch_texts]
|
|
295
321
|
results = await asyncio.gather(*tasks)
|
|
@@ -297,15 +323,12 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
297
323
|
for result in results:
|
|
298
324
|
if result is None:
|
|
299
325
|
dim = self._get_dimension(actual_model)
|
|
300
|
-
|
|
301
326
|
zero_vector = [0.0] * dim
|
|
302
327
|
all_vectors.append(zero_vector)
|
|
303
|
-
# [日志] 补充日志,明确是补零操作
|
|
304
328
|
SYLogger.warning(
|
|
305
329
|
f"Embedding request failed (returned None), appending zero vector ({dim}D) for model {actual_model}")
|
|
306
330
|
continue
|
|
307
331
|
|
|
308
|
-
# 从返回结果中提取向量
|
|
309
332
|
try:
|
|
310
333
|
for item in result["data"]:
|
|
311
334
|
embedding = item["embedding"]
|
|
@@ -326,17 +349,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
326
349
|
model: str = None,
|
|
327
350
|
timeout: Optional[Union[int, float]] = None
|
|
328
351
|
):
|
|
329
|
-
"""
|
|
330
|
-
对搜索结果进行重排序
|
|
331
|
-
|
|
332
|
-
Args:
|
|
333
|
-
top_results: 待重排序的文本列表
|
|
334
|
-
query: 排序参考的查询语句
|
|
335
|
-
model: 可选,指定使用的reranker模型名称,默认使用bge-reranker-large
|
|
336
|
-
timeout: 可选,超时时间(秒):
|
|
337
|
-
- 传int/float:表示总超时时间(秒)
|
|
338
|
-
- 不传/None:使用默认永不超时配置
|
|
339
|
-
"""
|
|
340
352
|
request_timeout = None
|
|
341
353
|
if timeout is not None:
|
|
342
354
|
if isinstance(timeout, (int, float)):
|
|
@@ -348,7 +360,6 @@ class Embedding(metaclass=SingletonMeta):
|
|
|
348
360
|
actual_model = model or self.default_reranker_model
|
|
349
361
|
SYLogger.info(
|
|
350
362
|
f"Requesting reranker for top_results: {top_results} (model: {actual_model}, max_concurrency: {self.max_concurrency}, timeout: {timeout or 'None'})")
|
|
351
|
-
# 打印请求参数
|
|
352
363
|
SYLogger.info(
|
|
353
364
|
f"Requesting reranker for top_results: {top_results} (model: {actual_model}) (query: {query}) (timeout: {timeout or 'None'})")
|
|
354
365
|
data = await self._get_reranker_http_async(
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_client.py
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
-
import
|
|
2
|
+
import anyio
|
|
3
3
|
import json
|
|
4
4
|
from typing import Optional, Callable, Coroutine, Dict, Any, Union
|
|
5
5
|
from aio_pika import Channel, Message, DeliveryMode, ExchangeType
|
|
@@ -20,6 +20,8 @@ logger = SYLogger
|
|
|
20
20
|
class RabbitMQClient:
|
|
21
21
|
"""
|
|
22
22
|
RabbitMQ 客户端
|
|
23
|
+
|
|
24
|
+
【架构升级】全面使用 anyio 替代 asyncio 原语,以支持跨线程/跨 Loop 调用。
|
|
23
25
|
"""
|
|
24
26
|
|
|
25
27
|
def __init__(
|
|
@@ -65,37 +67,30 @@ class RabbitMQClient:
|
|
|
65
67
|
MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]]] = None
|
|
66
68
|
self._closed = False
|
|
67
69
|
|
|
68
|
-
#
|
|
69
|
-
self._consume_lock =
|
|
70
|
-
self._connect_lock =
|
|
71
|
-
self._reconnect_lock =
|
|
70
|
+
# 【修改】使用 anyio 锁,支持跨线程调用
|
|
71
|
+
self._consume_lock = anyio.Lock()
|
|
72
|
+
self._connect_lock = anyio.Lock()
|
|
73
|
+
self._reconnect_lock = anyio.Lock()
|
|
72
74
|
|
|
73
75
|
# 防止并发重连覆盖
|
|
74
76
|
self._connecting = False
|
|
75
|
-
self._connect_condition =
|
|
77
|
+
self._connect_condition = anyio.Condition() # anyio Condition
|
|
76
78
|
|
|
77
79
|
self._conn_close_callback: Optional[Callable] = None
|
|
78
|
-
self._reconnect_semaphore =
|
|
80
|
+
self._reconnect_semaphore = anyio.Semaphore(1)
|
|
79
81
|
self._current_reconnect_task: Optional[asyncio.Task] = None
|
|
80
82
|
self._RECONNECT_INTERVAL = 15
|
|
81
83
|
|
|
82
84
|
@property
|
|
83
85
|
def is_connected(self) -> bool:
|
|
84
|
-
"""
|
|
85
|
-
同步检查连接状态
|
|
86
|
-
【修复】改为同步属性,避免异步调用错误和布尔上下文误判
|
|
87
|
-
"""
|
|
86
|
+
"""同步检查连接状态"""
|
|
88
87
|
if self._closed:
|
|
89
88
|
return False
|
|
90
89
|
try:
|
|
91
|
-
# 检查通道是否有效
|
|
92
90
|
if not self._channel or self._channel.is_closed:
|
|
93
91
|
return False
|
|
94
|
-
|
|
95
|
-
# 检查连接是否有效
|
|
96
92
|
if self._channel_conn and self._channel_conn.is_closed:
|
|
97
93
|
return False
|
|
98
|
-
|
|
99
94
|
return (
|
|
100
95
|
self._exchange is not None
|
|
101
96
|
and (not self.queue_name or self._queue is not None)
|
|
@@ -108,7 +103,6 @@ class RabbitMQClient:
|
|
|
108
103
|
if not self._channel or self._channel.is_closed:
|
|
109
104
|
raise RuntimeError("无有效通道,无法重建资源")
|
|
110
105
|
|
|
111
|
-
# 1. 无条件声明交换机
|
|
112
106
|
exchange_inst = await self._channel.declare_exchange(
|
|
113
107
|
name=self.exchange_name,
|
|
114
108
|
type=self.exchange_type,
|
|
@@ -119,7 +113,6 @@ class RabbitMQClient:
|
|
|
119
113
|
self._exchange = exchange_inst
|
|
120
114
|
logger.info(f"交换机重建成功: {self.exchange_name}")
|
|
121
115
|
|
|
122
|
-
# 2. 仅在有队列名且符合条件时声明队列
|
|
123
116
|
if self.queue_name and self.queue_name.endswith(f".{self.app_name}"):
|
|
124
117
|
self._queue = await self._channel.declare_queue(
|
|
125
118
|
name=self.queue_name,
|
|
@@ -131,72 +124,58 @@ class RabbitMQClient:
|
|
|
131
124
|
logger.info(f"队列重建成功: {self.queue_name}")
|
|
132
125
|
|
|
133
126
|
async def _ensure_connection_alive(self) -> AbstractRobustConnection:
|
|
134
|
-
"""
|
|
135
|
-
【新增安全辅助方法】
|
|
136
|
-
动态获取当前有效的连接对象,避免引用池中已过期的连接。
|
|
137
|
-
如果连接无效,会抛出异常,交由上层逻辑处理重连。
|
|
138
|
-
"""
|
|
139
127
|
if not self.connection_pool or not self.connection_pool._initialized:
|
|
140
128
|
raise RuntimeError("连接池未初始化")
|
|
141
|
-
|
|
142
129
|
conn = self.connection_pool._connection
|
|
143
130
|
if not conn or conn.is_closed:
|
|
144
131
|
raise RuntimeError("连接池底层连接已关闭")
|
|
145
132
|
return conn
|
|
146
133
|
|
|
147
134
|
async def connect(self) -> None:
|
|
148
|
-
"""
|
|
135
|
+
"""连接方法"""
|
|
149
136
|
if self._closed:
|
|
150
137
|
raise RuntimeError("客户端已关闭,无法重新连接")
|
|
151
138
|
|
|
152
|
-
# === 阶段 A: 并发控制循环 ===
|
|
139
|
+
# === 阶段 A: 并发控制循环 (使用 anyio.Condition) ===
|
|
153
140
|
while True:
|
|
154
141
|
await self._connect_condition.acquire()
|
|
155
142
|
try:
|
|
156
|
-
# 1. 检查是否已连接
|
|
157
143
|
if self.is_connected:
|
|
158
144
|
if self._connect_condition.locked():
|
|
159
145
|
self._connect_condition.release()
|
|
160
146
|
return
|
|
161
147
|
|
|
162
|
-
# 2. 检查是否已有协程在连接
|
|
163
148
|
if self._connecting:
|
|
164
149
|
try:
|
|
165
150
|
logger.debug("连接正在进行中,等待现有连接完成...")
|
|
166
|
-
#
|
|
167
|
-
|
|
168
|
-
|
|
151
|
+
# anyio Condition 支持 wait
|
|
152
|
+
with anyio.fail_after(60.0):
|
|
153
|
+
await self._connect_condition.wait()
|
|
154
|
+
except TimeoutError:
|
|
169
155
|
logger.warning("⚠️ 等待前序连接超时,重新竞争连接权...")
|
|
170
156
|
|
|
171
|
-
# 【核心修复】
|
|
172
|
-
# 无论是因为超时还是被唤醒,都 continue 重新循环
|
|
173
|
-
# 这样确保不会出现“所有协程同时醒来”的羊群效应
|
|
174
|
-
# 只有抢到锁的那个协程会进入连接逻辑,其他继续 wait
|
|
175
157
|
if self._connect_condition.locked():
|
|
176
158
|
self._connect_condition.release()
|
|
177
159
|
continue
|
|
178
160
|
|
|
179
|
-
# 3. 抢到连接权,标记开始
|
|
180
161
|
self._connecting = True
|
|
181
162
|
self._connect_condition.release()
|
|
182
|
-
break
|
|
163
|
+
break
|
|
183
164
|
|
|
184
165
|
except Exception as e:
|
|
185
166
|
if self._connect_condition.locked():
|
|
186
167
|
self._connect_condition.release()
|
|
187
168
|
raise
|
|
188
169
|
|
|
189
|
-
# === 阶段 C:
|
|
170
|
+
# === 阶段 C: 执行连接逻辑 ===
|
|
190
171
|
connection_failed = False
|
|
191
172
|
was_consuming = False
|
|
192
173
|
is_consumer = self._message_handler is not None
|
|
193
174
|
old_channel = self._channel
|
|
194
175
|
|
|
195
176
|
try:
|
|
196
|
-
# --- 步骤 1: 清理旧资源 ---
|
|
197
177
|
was_consuming = self._consumer_tag is not None
|
|
198
178
|
|
|
199
|
-
# 清理旧连接的回调
|
|
200
179
|
if self._channel_conn:
|
|
201
180
|
try:
|
|
202
181
|
if self._channel_conn.close_callbacks:
|
|
@@ -210,14 +189,13 @@ class RabbitMQClient:
|
|
|
210
189
|
except Exception:
|
|
211
190
|
pass
|
|
212
191
|
|
|
213
|
-
# 强制重置
|
|
214
192
|
self._channel = None
|
|
215
193
|
self._channel_conn = None
|
|
216
194
|
self._exchange = None
|
|
217
195
|
self._queue = None
|
|
218
196
|
self._consumer_tag = None
|
|
219
197
|
|
|
220
|
-
#
|
|
198
|
+
# 获取通道
|
|
221
199
|
if is_consumer:
|
|
222
200
|
logger.debug("获取消费者独立通道...")
|
|
223
201
|
self._channel = await self.connection_pool.acquire_consumer_channel()
|
|
@@ -225,36 +203,29 @@ class RabbitMQClient:
|
|
|
225
203
|
logger.debug("获取生产者主通道...")
|
|
226
204
|
self._channel, self._channel_conn = await self.connection_pool.acquire_channel()
|
|
227
205
|
|
|
228
|
-
# --- 步骤 3: 统一获取并注册连接回调 ---
|
|
229
206
|
if not self._channel_conn:
|
|
230
|
-
# 消费者路径需要手动获取连接引用
|
|
231
|
-
# 直接引用连接池的连接对象,确保引用时效性
|
|
232
207
|
self._channel_conn = self.connection_pool._connection
|
|
233
208
|
|
|
234
209
|
loop = asyncio.get_running_loop()
|
|
235
210
|
|
|
236
211
|
def on_conn_closed(conn, exc):
|
|
237
|
-
# 【核心修复】不再判断 _connecting,直接委托给 _safe_reconnect
|
|
238
|
-
# _safe_reconnect 内部有锁,会自动处理并发问题
|
|
239
212
|
if not self._closed:
|
|
240
213
|
logger.warning(f"⚠️ 检测到底层连接关闭: {exc}")
|
|
214
|
+
# anyio 会自动处理 loop 线程问题,这里依然使用 asyncio 的调用方式即可
|
|
241
215
|
asyncio.run_coroutine_threadsafe(
|
|
242
216
|
self._safe_reconnect(), loop)
|
|
243
217
|
|
|
244
|
-
# 注册回调前,再次防御性清理
|
|
245
218
|
if self._channel_conn.close_callbacks:
|
|
246
219
|
self._channel_conn.close_callbacks.clear()
|
|
247
220
|
|
|
248
221
|
self._channel_conn.close_callbacks.add(on_conn_closed)
|
|
249
222
|
|
|
250
|
-
# --- 步骤 4: 重建基础资源 ---
|
|
251
223
|
await self._rebuild_resources()
|
|
252
224
|
|
|
253
225
|
except Exception as e:
|
|
254
226
|
connection_failed = True
|
|
255
227
|
logger.error(f"❌ 客户端连接失败: {str(e)}", exc_info=True)
|
|
256
228
|
|
|
257
|
-
# 异常时清理
|
|
258
229
|
if self._channel_conn and self._channel_conn.close_callbacks:
|
|
259
230
|
self._channel_conn.close_callbacks.clear()
|
|
260
231
|
|
|
@@ -262,7 +233,6 @@ class RabbitMQClient:
|
|
|
262
233
|
self._channel_conn = None
|
|
263
234
|
self._exchange = None
|
|
264
235
|
self._queue = None
|
|
265
|
-
self._consumer_tag = None
|
|
266
236
|
raise
|
|
267
237
|
|
|
268
238
|
finally:
|
|
@@ -270,7 +240,6 @@ class RabbitMQClient:
|
|
|
270
240
|
try:
|
|
271
241
|
await self._connect_condition.acquire()
|
|
272
242
|
except Exception:
|
|
273
|
-
# 如果 acquire 本身失败,确保状态复位,防止死锁
|
|
274
243
|
self._connecting = False
|
|
275
244
|
self._connect_condition.notify_all()
|
|
276
245
|
raise
|
|
@@ -289,13 +258,11 @@ class RabbitMQClient:
|
|
|
289
258
|
passive=not self.create_if_not_exists,
|
|
290
259
|
)
|
|
291
260
|
|
|
292
|
-
# 确保绑定
|
|
293
261
|
if self._exchange:
|
|
294
262
|
await self._queue.bind(exchange=self._exchange, routing_key=self.routing_key)
|
|
295
263
|
logger.info(
|
|
296
264
|
f"✅ 重连绑定成功: {self.queue_name} -> {self.routing_key}")
|
|
297
265
|
else:
|
|
298
|
-
# 防御性编程
|
|
299
266
|
logger.error(
|
|
300
267
|
"🔥 Exchange missing, forcing declare...")
|
|
301
268
|
temp_ex = await self._channel.declare_exchange(
|
|
@@ -316,18 +283,12 @@ class RabbitMQClient:
|
|
|
316
283
|
logger.error(f"❌ 自动恢复消费失败: {e}")
|
|
317
284
|
self._consumer_tag = None
|
|
318
285
|
finally:
|
|
319
|
-
# 无论恢复逻辑成功与否,必须解锁
|
|
320
286
|
self._connecting = False
|
|
321
287
|
self._connect_condition.notify_all()
|
|
322
288
|
if self._connect_condition.locked():
|
|
323
289
|
self._connect_condition.release()
|
|
324
290
|
|
|
325
291
|
async def _safe_reconnect(self):
|
|
326
|
-
"""
|
|
327
|
-
安全重连入口
|
|
328
|
-
【核心修复】使用锁防止并发风暴,确保同一时间只有一个重连任务在执行。
|
|
329
|
-
"""
|
|
330
|
-
# 如果锁已经被占用,说明已经有重连任务在进行中,直接忽略,防止无限递归
|
|
331
292
|
if self._reconnect_lock.locked():
|
|
332
293
|
logger.debug("⏳ 重连任务已在执行中,忽略本次触发,避免并发风暴")
|
|
333
294
|
return
|
|
@@ -339,10 +300,8 @@ class RabbitMQClient:
|
|
|
339
300
|
return
|
|
340
301
|
|
|
341
302
|
logger.warning("🔄 触发底层连接重连...")
|
|
342
|
-
# 调用实际的 connect 方法
|
|
343
303
|
await self.connect()
|
|
344
304
|
except Exception as e:
|
|
345
|
-
# 即使重连失败,也不要在这里递归调用自己,而是依赖外部的定时任务或下一次网络事件
|
|
346
305
|
logger.error(f"❌ 安全重连执行失败: {str(e)}", exc_info=True)
|
|
347
306
|
|
|
348
307
|
async def set_message_handler(self, handler: Callable[..., Coroutine]) -> None:
|
|
@@ -353,28 +312,21 @@ class RabbitMQClient:
|
|
|
353
312
|
|
|
354
313
|
async def _process_message_callback(self, message: AbstractIncomingMessage):
|
|
355
314
|
"""
|
|
356
|
-
消息处理回调
|
|
357
|
-
|
|
358
|
-
【架构原理】
|
|
359
|
-
1. 优先尝试:
|
|
360
|
-
- 在子线程中构建完全独立的 asyncio 运行环境。
|
|
361
|
-
- XX.Semaphore 绑定到子线程 Loop,彻底隔离 CPU 压力。
|
|
315
|
+
消息处理回调 (Anyio 兼容版)
|
|
362
316
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
- 此时直接在主线程 await 执行,利用异步 I/O 特性保持心跳响应。
|
|
317
|
+
既然使用了 anyio,大部分跨 Loop 问题已解决。
|
|
318
|
+
保留 asyncio.to_thread 用于在子线程中隔离 CPU 密集型或阻塞型任务。
|
|
366
319
|
"""
|
|
367
|
-
msg_obj = None
|
|
320
|
+
msg_obj = None
|
|
368
321
|
trace_id = "unknown"
|
|
369
322
|
|
|
370
323
|
try:
|
|
371
|
-
# === 阶段 1: 消息解析与 TraceId 准备 (主线程) ===
|
|
372
324
|
try:
|
|
373
325
|
body_dict = json.loads(message.body.decode("utf-8"))
|
|
374
326
|
msg_obj: MQMsgModel = MQMsgModel(**body_dict)
|
|
375
327
|
except Exception as e:
|
|
376
328
|
logger.error(f"❌ 消息解析失败: {str(e)}")
|
|
377
|
-
await message.ack()
|
|
329
|
+
await message.ack()
|
|
378
330
|
return
|
|
379
331
|
|
|
380
332
|
if not msg_obj.traceId:
|
|
@@ -384,52 +336,36 @@ class RabbitMQClient:
|
|
|
384
336
|
trace_id = msg_obj.traceId
|
|
385
337
|
SYLogger.set_trace_id(trace_id)
|
|
386
338
|
|
|
387
|
-
# === 阶段 2: 智能执行 ===
|
|
388
339
|
if self._message_handler:
|
|
389
|
-
|
|
390
|
-
# --- 模式 A: 终极隔离方案 (默认首选) ---
|
|
340
|
+
# 定义子线程运行函数
|
|
391
341
|
def run_in_isolated_thread(current_trace_id):
|
|
392
|
-
# 【关键修复】通过参数传递 TraceId,防止闭包捕获导致并发 ID 错乱
|
|
393
342
|
SYLogger.set_trace_id(current_trace_id)
|
|
394
|
-
|
|
395
343
|
try:
|
|
396
344
|
return asyncio.run(self._message_handler(msg_obj, message))
|
|
397
345
|
except Exception as e:
|
|
398
|
-
# 捕获异常并抛出,避免在子线程中崩溃导致主线程无法感知
|
|
399
346
|
raise e
|
|
400
347
|
|
|
348
|
+
# 【关键】使用 to_thread 在子线程运行,避免阻塞主线程心跳
|
|
349
|
+
# anyio 兼容 asyncio 的线程池接口
|
|
401
350
|
try:
|
|
402
|
-
# 1. 扔进线程池执行 (主线程等待子线程完成)
|
|
403
351
|
await asyncio.to_thread(run_in_isolated_thread, trace_id)
|
|
404
|
-
|
|
405
352
|
except RuntimeError as e:
|
|
406
|
-
#
|
|
353
|
+
# 保留这个防御性检查,以防万一
|
|
407
354
|
if "bound to a different event loop" in str(e):
|
|
408
355
|
logger.warning(
|
|
409
356
|
f"⚠️ 检测到跨 Loop 冲突 (TraceId: {trace_id}),"
|
|
410
|
-
f"
|
|
357
|
+
f"自动切换至【混合模式】。"
|
|
411
358
|
)
|
|
412
|
-
|
|
413
|
-
# --- 模式 B: 混合模式 (降级方案) ---
|
|
414
|
-
# 既然 Semaphore 绑定了主 Loop,且必须保证流控 (Ack 在最后),
|
|
415
|
-
# 那么直接在主线程 await 执行是唯一正确的解法。
|
|
416
|
-
# 只要业务逻辑是异步 I/O,主线程会在 await 期间处理心跳。
|
|
417
359
|
await self._message_handler(msg_obj, message)
|
|
418
|
-
|
|
419
360
|
else:
|
|
420
|
-
# 其他 RuntimeError 直接抛出
|
|
421
361
|
raise
|
|
422
362
|
|
|
423
|
-
# === 阶段 3: 消息确认 ===
|
|
424
|
-
# 只有代码执行到这里,说明业务已成功处理 (或已处理完异常)
|
|
425
|
-
# 此时 Ack 才能正确触发 MQ 的流控机制
|
|
426
363
|
await message.ack()
|
|
427
364
|
|
|
428
365
|
except Exception as e:
|
|
429
366
|
logger.error(
|
|
430
367
|
f"❌ 消息处理异常 (TraceId: {trace_id}): {str(e)}", exc_info=True)
|
|
431
368
|
try:
|
|
432
|
-
# 即使异常也 Ack,避免死信循环(根据业务需求也可改为 Nack/Requeue)
|
|
433
369
|
await message.ack()
|
|
434
370
|
except Exception:
|
|
435
371
|
pass
|
|
@@ -473,7 +409,6 @@ class RabbitMQClient:
|
|
|
473
409
|
self._consumer_tag = None
|
|
474
410
|
|
|
475
411
|
async def _handle_publish_failure(self):
|
|
476
|
-
# 如果当前正在重连,或者已经关闭,直接返回,避免冲突
|
|
477
412
|
if self._connecting or self._closed:
|
|
478
413
|
logger.warning("⚠️ 正在重连或已关闭,跳过故障转移触发")
|
|
479
414
|
return
|
|
@@ -481,7 +416,6 @@ class RabbitMQClient:
|
|
|
481
416
|
try:
|
|
482
417
|
logger.info("检测到发布异常,强制连接池切换节点...")
|
|
483
418
|
await self.connection_pool.force_reconnect()
|
|
484
|
-
# 连接池切换后,必须刷新客户端资源
|
|
485
419
|
await self.connect()
|
|
486
420
|
logger.info("故障转移完成,资源已刷新")
|
|
487
421
|
except Exception as e:
|
|
@@ -552,12 +486,9 @@ class RabbitMQClient:
|
|
|
552
486
|
raise RuntimeError(f"消息发布最终失败: {last_exception}")
|
|
553
487
|
|
|
554
488
|
async def close(self) -> None:
|
|
555
|
-
"""关闭客户端(支持独立通道的清理与死锁修复)"""
|
|
556
|
-
# 1. 先标记关闭
|
|
557
489
|
self._closed = True
|
|
558
490
|
logger.info("开始关闭RabbitMQ客户端...")
|
|
559
491
|
|
|
560
|
-
# 2. 取消可能存在的后台重连任务
|
|
561
492
|
if self._current_reconnect_task and not self._current_reconnect_task.done():
|
|
562
493
|
self._current_reconnect_task.cancel()
|
|
563
494
|
try:
|
|
@@ -565,17 +496,17 @@ class RabbitMQClient:
|
|
|
565
496
|
except asyncio.CancelledError:
|
|
566
497
|
pass
|
|
567
498
|
|
|
568
|
-
# 3. 停止消费
|
|
569
499
|
await self.stop_consuming()
|
|
570
500
|
|
|
571
|
-
#
|
|
501
|
+
# 【修改】使用 anyio 处理锁超时和释放
|
|
572
502
|
try:
|
|
573
|
-
|
|
574
|
-
|
|
503
|
+
# anyio 没有直接的 wait_for,使用 fail_after
|
|
504
|
+
with anyio.fail_after(2.0):
|
|
505
|
+
await self._connect_condition.acquire()
|
|
506
|
+
except TimeoutError:
|
|
575
507
|
logger.warning("获取连接锁超时,强制清理资源...")
|
|
576
508
|
|
|
577
509
|
try:
|
|
578
|
-
# 清理回调
|
|
579
510
|
if self._channel_conn:
|
|
580
511
|
try:
|
|
581
512
|
if self._channel_conn.close_callbacks:
|
|
@@ -583,22 +514,13 @@ class RabbitMQClient:
|
|
|
583
514
|
except Exception:
|
|
584
515
|
pass
|
|
585
516
|
|
|
586
|
-
# 【关键修改】显式关闭持有的通道
|
|
587
|
-
# 无论是生产者(主通道,但 Client 只是持有者,通常不关 Pool 管理的主通道),
|
|
588
|
-
# 还是消费者(独立通道,必须显式关闭),这里都需要处理。
|
|
589
|
-
# 由于我们引入了独立消费者通道,这里必须显式关闭 self._channel
|
|
590
517
|
if self._channel and not self._channel.is_closed:
|
|
591
518
|
try:
|
|
592
|
-
# 注意:如果是主通道,这里关闭可能会影响其他 Producer。
|
|
593
|
-
# 但由于我们的架构中,Consumer 用独立通道,这里大概率是 Consumer 关闭。
|
|
594
|
-
# 为了安全,可以增加判断:如果 shared_channel 标志为 False 才关?
|
|
595
|
-
# 简化策略:统一关闭,因为 Client 被销毁意味着不再需要该通道。
|
|
596
519
|
await self._channel.close()
|
|
597
520
|
logger.debug("客户端通道已关闭")
|
|
598
521
|
except Exception as e:
|
|
599
522
|
logger.warning(f"关闭客户端通道异常: {e}")
|
|
600
523
|
|
|
601
|
-
# 置空资源引用
|
|
602
524
|
self._channel = None
|
|
603
525
|
self._channel_conn = None
|
|
604
526
|
self._exchange = None
|
|
@@ -607,7 +529,6 @@ class RabbitMQClient:
|
|
|
607
529
|
self._conn_close_callback = None
|
|
608
530
|
|
|
609
531
|
finally:
|
|
610
|
-
# 强制重置状态并唤醒所有等待者
|
|
611
532
|
self._connecting = False
|
|
612
533
|
self._connect_condition.notify_all()
|
|
613
534
|
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sycommon-python-lib
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.0b7
|
|
4
4
|
Summary: Add your description here
|
|
5
5
|
Requires-Python: >=3.11
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: aio-pika>=9.5.8
|
|
8
8
|
Requires-Dist: aiohttp>=3.13.3
|
|
9
9
|
Requires-Dist: aiomysql>=0.3.2
|
|
10
|
+
Requires-Dist: anyio>=4.12.0
|
|
10
11
|
Requires-Dist: decorator>=5.2.1
|
|
11
12
|
Requires-Dist: fastapi>=0.128.0
|
|
12
13
|
Requires-Dist: kafka-python>=2.3.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/DatabaseConfig.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/EmbeddingConfig.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/LLMConfig.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/LangfuseConfig.py
RENAMED
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/RerankerConfig.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/config/SentryConfig.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/database/base_db_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/health/health_check.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/llm/struct_token.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/__init__.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/async_sql_logger.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/kafka_log.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/logger_levels.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/logger_wrapper.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/logging/sql_logger.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/__init__.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/context.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/exception.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/middleware.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/timeout.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/middleware/traceid.py
RENAMED
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/base_http.py
RENAMED
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqlistener_config.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqmsg_model.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/models/mqsend_config.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/notice/uvicorn_monitor.py
RENAMED
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/rabbitmq/rabbitmq_pool.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/sentry/sy_sentry.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/example2.py
RENAMED
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/feign_client.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/synacos/nacos_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tests/test_email.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{sycommon_python_lib-0.2.0b6 → sycommon_python_lib-0.2.0b7}/src/sycommon/tools/merge_headers.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|