sycommon-python-lib 0.1.56__py3-none-any.whl → 0.1.56b2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sycommon/config/Config.py +3 -24
- sycommon/llm/embedding.py +23 -78
- sycommon/llm/get_llm.py +164 -24
- sycommon/logging/kafka_log.py +433 -187
- sycommon/middleware/exception.py +16 -10
- sycommon/middleware/timeout.py +1 -2
- sycommon/middleware/traceid.py +76 -81
- sycommon/rabbitmq/rabbitmq_client.py +242 -232
- sycommon/rabbitmq/rabbitmq_pool.py +218 -278
- sycommon/rabbitmq/rabbitmq_service.py +843 -25
- sycommon/services.py +96 -122
- sycommon/synacos/nacos_service.py +779 -63
- sycommon/tools/merge_headers.py +0 -20
- sycommon/tools/snowflake.py +153 -101
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/METADATA +8 -10
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/RECORD +19 -40
- sycommon/config/LangfuseConfig.py +0 -15
- sycommon/config/SentryConfig.py +0 -13
- sycommon/llm/llm_tokens.py +0 -119
- sycommon/llm/struct_token.py +0 -192
- sycommon/llm/sy_langfuse.py +0 -103
- sycommon/llm/usage_token.py +0 -117
- sycommon/notice/__init__.py +0 -0
- sycommon/notice/uvicorn_monitor.py +0 -200
- sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -206
- sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -73
- sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -285
- sycommon/rabbitmq/rabbitmq_service_core.py +0 -117
- sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -238
- sycommon/sentry/__init__.py +0 -0
- sycommon/sentry/sy_sentry.py +0 -35
- sycommon/synacos/nacos_client_base.py +0 -119
- sycommon/synacos/nacos_config_manager.py +0 -107
- sycommon/synacos/nacos_heartbeat_manager.py +0 -144
- sycommon/synacos/nacos_service_discovery.py +0 -157
- sycommon/synacos/nacos_service_registration.py +0 -270
- sycommon/tools/env.py +0 -62
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/WHEEL +0 -0
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/top_level.txt +0 -0
sycommon/services.py
CHANGED
|
@@ -12,32 +12,30 @@ from sycommon.models.mqlistener_config import RabbitMQListenerConfig
|
|
|
12
12
|
from sycommon.models.mqsend_config import RabbitMQSendConfig
|
|
13
13
|
from sycommon.rabbitmq.rabbitmq_service import RabbitMQService
|
|
14
14
|
from sycommon.tools.docs import custom_redoc_html, custom_swagger_ui_html
|
|
15
|
-
from sycommon.sentry.sy_sentry import sy_sentry_init
|
|
16
15
|
|
|
17
16
|
|
|
18
17
|
class Services(metaclass=SingletonMeta):
|
|
19
18
|
_loop: Optional[asyncio.AbstractEventLoop] = None
|
|
20
19
|
_config: Optional[dict] = None
|
|
21
20
|
_initialized: bool = False
|
|
21
|
+
_registered_senders: List[str] = []
|
|
22
22
|
_instance: Optional['Services'] = None
|
|
23
23
|
_app: Optional[FastAPI] = None
|
|
24
24
|
_user_lifespan: Optional[Callable] = None
|
|
25
25
|
_shutdown_lock: asyncio.Lock = asyncio.Lock()
|
|
26
26
|
|
|
27
|
+
# 用于存储待执行的异步数据库初始化任务
|
|
28
|
+
_pending_async_db_setup: List[Tuple[Callable, str]] = []
|
|
29
|
+
|
|
27
30
|
def __init__(self, config: dict, app: FastAPI):
|
|
28
|
-
super().__init__()
|
|
29
31
|
if not Services._config:
|
|
30
32
|
Services._config = config
|
|
31
33
|
Services._instance = self
|
|
32
34
|
Services._app = app
|
|
33
|
-
|
|
34
|
-
# 在实例初始化时定义变量,防止类变量污染
|
|
35
|
-
self._pending_async_db_setup: List[Tuple[Callable, str]] = []
|
|
36
|
-
|
|
37
35
|
self._init_event_loop()
|
|
38
36
|
|
|
39
37
|
def _init_event_loop(self):
|
|
40
|
-
"""
|
|
38
|
+
"""初始化事件循环,确保全局只有一个循环实例"""
|
|
41
39
|
if not Services._loop:
|
|
42
40
|
try:
|
|
43
41
|
Services._loop = asyncio.get_running_loop()
|
|
@@ -64,20 +62,14 @@ class Services(metaclass=SingletonMeta):
|
|
|
64
62
|
setup_logger_levels()
|
|
65
63
|
cls._app = app
|
|
66
64
|
cls._config = config
|
|
67
|
-
# 保存原始的用户 lifespan
|
|
68
65
|
cls._user_lifespan = app.router.lifespan_context
|
|
69
66
|
|
|
70
67
|
applications.get_swagger_ui_html = custom_swagger_ui_html
|
|
71
68
|
applications.get_redoc_html = custom_redoc_html
|
|
72
69
|
|
|
73
70
|
if not cls._config:
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
config = yaml.safe_load(f)
|
|
77
|
-
cls._config = config
|
|
78
|
-
except FileNotFoundError:
|
|
79
|
-
logging.warning("未找到 app.yaml,将使用空配置启动")
|
|
80
|
-
cls._config = {}
|
|
71
|
+
config = yaml.safe_load(open('app.yaml', 'r', encoding='utf-8'))
|
|
72
|
+
cls._config = config
|
|
81
73
|
|
|
82
74
|
app.state.config = {
|
|
83
75
|
"host": cls._config.get('Host', '0.0.0.0'),
|
|
@@ -87,95 +79,94 @@ class Services(metaclass=SingletonMeta):
|
|
|
87
79
|
}
|
|
88
80
|
|
|
89
81
|
if middleware:
|
|
90
|
-
middleware(app,
|
|
82
|
+
middleware(app, config)
|
|
91
83
|
|
|
92
84
|
if nacos_service:
|
|
93
|
-
nacos_service(
|
|
85
|
+
nacos_service(config)
|
|
94
86
|
|
|
95
87
|
if logging_service:
|
|
96
|
-
logging_service(
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
88
|
+
logging_service(config)
|
|
89
|
+
|
|
90
|
+
# ========== 处理数据库服务 ==========
|
|
91
|
+
# 清空之前的待执行列表(防止热重载时重复)
|
|
92
|
+
cls._pending_async_db_setup = []
|
|
93
|
+
|
|
94
|
+
if database_service:
|
|
95
|
+
# 解析配置并区分同步/异步
|
|
96
|
+
items = [database_service] if isinstance(
|
|
97
|
+
database_service, tuple) else database_service
|
|
98
|
+
for item in items:
|
|
99
|
+
db_setup_func, db_name = item
|
|
100
|
+
if asyncio.iscoroutinefunction(db_setup_func):
|
|
101
|
+
# 如果是异步函数,加入待执行列表
|
|
102
|
+
logging.info(f"检测到异步数据库服务: {db_name},将在应用启动时初始化")
|
|
103
|
+
cls._pending_async_db_setup.append(item)
|
|
104
|
+
else:
|
|
105
|
+
# 如果是同步函数,立即执行
|
|
106
|
+
logging.info(f"执行同步数据库服务: {db_name}")
|
|
107
|
+
try:
|
|
108
|
+
db_setup_func(config, db_name)
|
|
109
|
+
except Exception as e:
|
|
110
|
+
logging.error(
|
|
111
|
+
f"同步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
112
|
+
raise
|
|
113
|
+
|
|
114
|
+
# 创建组合生命周期管理器
|
|
100
115
|
@asynccontextmanager
|
|
101
|
-
async def combined_lifespan(
|
|
102
|
-
#
|
|
103
|
-
instance = cls(config,
|
|
116
|
+
async def combined_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|
117
|
+
# 1. 执行Services自身的初始化
|
|
118
|
+
instance = cls(config, app)
|
|
119
|
+
|
|
120
|
+
# ========== 执行挂起的异步数据库初始化 ==========
|
|
121
|
+
if cls._pending_async_db_setup:
|
|
122
|
+
logging.info("开始执行异步数据库初始化...")
|
|
123
|
+
for db_setup_func, db_name in cls._pending_async_db_setup:
|
|
124
|
+
try:
|
|
125
|
+
await db_setup_func(config, db_name)
|
|
126
|
+
logging.info(f"异步数据库服务 {db_name} 初始化成功")
|
|
127
|
+
except Exception as e:
|
|
128
|
+
logging.error(
|
|
129
|
+
f"异步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
130
|
+
raise
|
|
131
|
+
|
|
132
|
+
# ========== 初始化 MQ ==========
|
|
133
|
+
has_valid_listeners = bool(
|
|
134
|
+
rabbitmq_listeners and len(rabbitmq_listeners) > 0)
|
|
135
|
+
has_valid_senders = bool(
|
|
136
|
+
rabbitmq_senders and len(rabbitmq_senders) > 0)
|
|
104
137
|
|
|
105
138
|
try:
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
logging.info(f"执行同步数据库服务: {db_name}")
|
|
119
|
-
try:
|
|
120
|
-
db_setup_func(config, db_name)
|
|
121
|
-
except Exception as e:
|
|
122
|
-
logging.error(
|
|
123
|
-
f"同步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
124
|
-
raise
|
|
125
|
-
|
|
126
|
-
# 2. 执行挂起的异步数据库初始化
|
|
127
|
-
if instance._pending_async_db_setup:
|
|
128
|
-
logging.info("开始执行异步数据库初始化...")
|
|
129
|
-
for db_setup_func, db_name in instance._pending_async_db_setup:
|
|
130
|
-
try:
|
|
131
|
-
await db_setup_func(config, db_name)
|
|
132
|
-
logging.info(f"异步数据库服务 {db_name} 初始化成功")
|
|
133
|
-
except Exception as e:
|
|
134
|
-
logging.error(
|
|
135
|
-
f"异步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
136
|
-
raise
|
|
137
|
-
|
|
138
|
-
# 3. 初始化 MQ
|
|
139
|
-
has_valid_listeners = bool(
|
|
140
|
-
rabbitmq_listeners and len(rabbitmq_listeners) > 0)
|
|
141
|
-
has_valid_senders = bool(
|
|
142
|
-
rabbitmq_senders and len(rabbitmq_senders) > 0)
|
|
143
|
-
|
|
144
|
-
try:
|
|
145
|
-
if has_valid_listeners or has_valid_senders:
|
|
146
|
-
await instance._setup_mq_async(
|
|
147
|
-
rabbitmq_listeners=rabbitmq_listeners if has_valid_listeners else None,
|
|
148
|
-
rabbitmq_senders=rabbitmq_senders if has_valid_senders else None,
|
|
149
|
-
has_listeners=has_valid_listeners,
|
|
150
|
-
has_senders=has_valid_senders
|
|
151
|
-
)
|
|
152
|
-
cls._initialized = True
|
|
153
|
-
logging.info("Services初始化完成")
|
|
154
|
-
except Exception as e:
|
|
155
|
-
logging.error(f"MQ初始化失败: {str(e)}", exc_info=True)
|
|
156
|
-
raise
|
|
139
|
+
if has_valid_listeners or has_valid_senders:
|
|
140
|
+
await instance._setup_mq_async(
|
|
141
|
+
rabbitmq_listeners=rabbitmq_listeners if has_valid_listeners else None,
|
|
142
|
+
rabbitmq_senders=rabbitmq_senders if has_valid_senders else None,
|
|
143
|
+
has_listeners=has_valid_listeners,
|
|
144
|
+
has_senders=has_valid_senders
|
|
145
|
+
)
|
|
146
|
+
cls._initialized = True
|
|
147
|
+
logging.info("Services初始化完成")
|
|
148
|
+
except Exception as e:
|
|
149
|
+
logging.error(f"Services初始化失败: {str(e)}", exc_info=True)
|
|
150
|
+
raise
|
|
157
151
|
|
|
158
|
-
|
|
152
|
+
app.state.services = instance
|
|
159
153
|
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
yield
|
|
164
|
-
else:
|
|
154
|
+
# 2. 执行用户定义的生命周期
|
|
155
|
+
if cls._user_lifespan:
|
|
156
|
+
async with cls._user_lifespan(app):
|
|
165
157
|
yield
|
|
158
|
+
else:
|
|
159
|
+
yield
|
|
166
160
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
raise
|
|
171
|
-
finally:
|
|
172
|
-
# 无论成功或失败,都会执行关闭逻辑
|
|
173
|
-
await cls.shutdown()
|
|
174
|
-
logging.info("Services已关闭")
|
|
161
|
+
# 3. 执行Services的关闭逻辑
|
|
162
|
+
await cls.shutdown()
|
|
163
|
+
logging.info("Services已关闭")
|
|
175
164
|
|
|
176
165
|
app.router.lifespan_context = combined_lifespan
|
|
177
166
|
return app
|
|
178
167
|
|
|
168
|
+
# 移除了 _setup_database_static,因为逻辑已内联到 plugins 中
|
|
169
|
+
|
|
179
170
|
async def _setup_mq_async(
|
|
180
171
|
self,
|
|
181
172
|
rabbitmq_listeners: Optional[List[RabbitMQListenerConfig]] = None,
|
|
@@ -191,21 +182,12 @@ class Services(metaclass=SingletonMeta):
|
|
|
191
182
|
RabbitMQService.init(self._config, has_listeners, has_senders)
|
|
192
183
|
|
|
193
184
|
start_time = asyncio.get_event_loop().time()
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
and not RabbitMQService._is_shutdown:
|
|
199
|
-
if asyncio.get_event_loop().time() - start_time > timeout:
|
|
200
|
-
logging.error("RabbitMQ连接池初始化超时")
|
|
201
|
-
raise TimeoutError(f"RabbitMQ连接池初始化超时({timeout}秒)")
|
|
202
|
-
|
|
203
|
-
logging.debug("等待RabbitMQ连接池初始化...")
|
|
185
|
+
while not (RabbitMQService._connection_pool and RabbitMQService._connection_pool._initialized) and not RabbitMQService._is_shutdown:
|
|
186
|
+
if asyncio.get_event_loop().time() - start_time > 30:
|
|
187
|
+
raise TimeoutError("RabbitMQ连接池初始化超时(30秒)")
|
|
188
|
+
logging.info("等待RabbitMQ连接池初始化...")
|
|
204
189
|
await asyncio.sleep(0.5)
|
|
205
190
|
|
|
206
|
-
if RabbitMQService._is_shutdown:
|
|
207
|
-
raise RuntimeError("RabbitMQService 在初始化期间被关闭")
|
|
208
|
-
|
|
209
191
|
if has_senders and rabbitmq_senders:
|
|
210
192
|
if has_listeners and rabbitmq_listeners:
|
|
211
193
|
for sender in rabbitmq_senders:
|
|
@@ -225,8 +207,11 @@ class Services(metaclass=SingletonMeta):
|
|
|
225
207
|
|
|
226
208
|
async def _setup_senders_async(self, rabbitmq_senders, has_listeners: bool):
|
|
227
209
|
"""设置发送器"""
|
|
210
|
+
Services._registered_senders = [
|
|
211
|
+
sender.queue_name for sender in rabbitmq_senders]
|
|
228
212
|
await RabbitMQService.setup_senders(rabbitmq_senders, has_listeners)
|
|
229
|
-
|
|
213
|
+
Services._registered_senders = RabbitMQService._sender_client_names
|
|
214
|
+
logging.info(f"已注册的RabbitMQ发送器: {Services._registered_senders}")
|
|
230
215
|
|
|
231
216
|
async def _setup_listeners_async(self, rabbitmq_listeners, has_senders: bool):
|
|
232
217
|
"""设置监听器"""
|
|
@@ -251,12 +236,14 @@ class Services(metaclass=SingletonMeta):
|
|
|
251
236
|
|
|
252
237
|
for attempt in range(max_retries):
|
|
253
238
|
try:
|
|
254
|
-
|
|
255
|
-
|
|
239
|
+
if queue_name not in cls._registered_senders:
|
|
240
|
+
cls._registered_senders = RabbitMQService._sender_client_names
|
|
241
|
+
if queue_name not in cls._registered_senders:
|
|
242
|
+
raise ValueError(f"发送器 {queue_name} 未注册")
|
|
256
243
|
|
|
244
|
+
sender = await RabbitMQService.get_sender(queue_name)
|
|
257
245
|
if not sender:
|
|
258
|
-
raise ValueError(
|
|
259
|
-
f"发送器 '{queue_name}' 不存在或未在 RabbitMQService 中注册")
|
|
246
|
+
raise ValueError(f"发送器 '{queue_name}' 不存在或连接无效")
|
|
260
247
|
|
|
261
248
|
await RabbitMQService.send_message(data, queue_name, **kwargs)
|
|
262
249
|
logging.info(f"消息发送成功(尝试 {attempt+1}/{max_retries})")
|
|
@@ -278,20 +265,7 @@ class Services(metaclass=SingletonMeta):
|
|
|
278
265
|
if RabbitMQService._is_shutdown:
|
|
279
266
|
logging.info("RabbitMQService已关闭,无需重复操作")
|
|
280
267
|
return
|
|
281
|
-
|
|
282
|
-
try:
|
|
283
|
-
await RabbitMQService.shutdown()
|
|
284
|
-
except Exception as e:
|
|
285
|
-
logging.error(f"关闭 RabbitMQService 时发生异常: {e}", exc_info=True)
|
|
286
|
-
|
|
268
|
+
await RabbitMQService.shutdown()
|
|
287
269
|
cls._initialized = False
|
|
288
|
-
|
|
289
|
-
# 清理实例数据
|
|
290
|
-
if cls._instance:
|
|
291
|
-
cls._instance._pending_async_db_setup.clear()
|
|
292
|
-
|
|
293
|
-
# 这对于热重载(reload)时防止旧实例内存泄漏至关重要
|
|
294
|
-
if cls._app:
|
|
295
|
-
cls._app.state.services = None
|
|
296
|
-
|
|
270
|
+
cls._registered_senders.clear()
|
|
297
271
|
logging.info("所有服务已关闭")
|