sycommon-python-lib 0.1.56__py3-none-any.whl → 0.1.56b2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. sycommon/config/Config.py +3 -24
  2. sycommon/llm/embedding.py +23 -78
  3. sycommon/llm/get_llm.py +164 -24
  4. sycommon/logging/kafka_log.py +433 -187
  5. sycommon/middleware/exception.py +16 -10
  6. sycommon/middleware/timeout.py +1 -2
  7. sycommon/middleware/traceid.py +76 -81
  8. sycommon/rabbitmq/rabbitmq_client.py +242 -232
  9. sycommon/rabbitmq/rabbitmq_pool.py +218 -278
  10. sycommon/rabbitmq/rabbitmq_service.py +843 -25
  11. sycommon/services.py +96 -122
  12. sycommon/synacos/nacos_service.py +779 -63
  13. sycommon/tools/merge_headers.py +0 -20
  14. sycommon/tools/snowflake.py +153 -101
  15. {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/METADATA +8 -10
  16. {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/RECORD +19 -40
  17. sycommon/config/LangfuseConfig.py +0 -15
  18. sycommon/config/SentryConfig.py +0 -13
  19. sycommon/llm/llm_tokens.py +0 -119
  20. sycommon/llm/struct_token.py +0 -192
  21. sycommon/llm/sy_langfuse.py +0 -103
  22. sycommon/llm/usage_token.py +0 -117
  23. sycommon/notice/__init__.py +0 -0
  24. sycommon/notice/uvicorn_monitor.py +0 -200
  25. sycommon/rabbitmq/rabbitmq_service_client_manager.py +0 -206
  26. sycommon/rabbitmq/rabbitmq_service_connection_monitor.py +0 -73
  27. sycommon/rabbitmq/rabbitmq_service_consumer_manager.py +0 -285
  28. sycommon/rabbitmq/rabbitmq_service_core.py +0 -117
  29. sycommon/rabbitmq/rabbitmq_service_producer_manager.py +0 -238
  30. sycommon/sentry/__init__.py +0 -0
  31. sycommon/sentry/sy_sentry.py +0 -35
  32. sycommon/synacos/nacos_client_base.py +0 -119
  33. sycommon/synacos/nacos_config_manager.py +0 -107
  34. sycommon/synacos/nacos_heartbeat_manager.py +0 -144
  35. sycommon/synacos/nacos_service_discovery.py +0 -157
  36. sycommon/synacos/nacos_service_registration.py +0 -270
  37. sycommon/tools/env.py +0 -62
  38. {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/WHEEL +0 -0
  39. {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/entry_points.txt +0 -0
  40. {sycommon_python_lib-0.1.56.dist-info → sycommon_python_lib-0.1.56b2.dist-info}/top_level.txt +0 -0
sycommon/services.py CHANGED
@@ -12,32 +12,30 @@ from sycommon.models.mqlistener_config import RabbitMQListenerConfig
12
12
  from sycommon.models.mqsend_config import RabbitMQSendConfig
13
13
  from sycommon.rabbitmq.rabbitmq_service import RabbitMQService
14
14
  from sycommon.tools.docs import custom_redoc_html, custom_swagger_ui_html
15
- from sycommon.sentry.sy_sentry import sy_sentry_init
16
15
 
17
16
 
18
17
  class Services(metaclass=SingletonMeta):
19
18
  _loop: Optional[asyncio.AbstractEventLoop] = None
20
19
  _config: Optional[dict] = None
21
20
  _initialized: bool = False
21
+ _registered_senders: List[str] = []
22
22
  _instance: Optional['Services'] = None
23
23
  _app: Optional[FastAPI] = None
24
24
  _user_lifespan: Optional[Callable] = None
25
25
  _shutdown_lock: asyncio.Lock = asyncio.Lock()
26
26
 
27
+ # 用于存储待执行的异步数据库初始化任务
28
+ _pending_async_db_setup: List[Tuple[Callable, str]] = []
29
+
27
30
  def __init__(self, config: dict, app: FastAPI):
28
- super().__init__()
29
31
  if not Services._config:
30
32
  Services._config = config
31
33
  Services._instance = self
32
34
  Services._app = app
33
-
34
- # 在实例初始化时定义变量,防止类变量污染
35
- self._pending_async_db_setup: List[Tuple[Callable, str]] = []
36
-
37
35
  self._init_event_loop()
38
36
 
39
37
  def _init_event_loop(self):
40
- """初始化事件循环"""
38
+ """初始化事件循环,确保全局只有一个循环实例"""
41
39
  if not Services._loop:
42
40
  try:
43
41
  Services._loop = asyncio.get_running_loop()
@@ -64,20 +62,14 @@ class Services(metaclass=SingletonMeta):
64
62
  setup_logger_levels()
65
63
  cls._app = app
66
64
  cls._config = config
67
- # 保存原始的用户 lifespan
68
65
  cls._user_lifespan = app.router.lifespan_context
69
66
 
70
67
  applications.get_swagger_ui_html = custom_swagger_ui_html
71
68
  applications.get_redoc_html = custom_redoc_html
72
69
 
73
70
  if not cls._config:
74
- try:
75
- with open('app.yaml', 'r', encoding='utf-8') as f:
76
- config = yaml.safe_load(f)
77
- cls._config = config
78
- except FileNotFoundError:
79
- logging.warning("未找到 app.yaml,将使用空配置启动")
80
- cls._config = {}
71
+ config = yaml.safe_load(open('app.yaml', 'r', encoding='utf-8'))
72
+ cls._config = config
81
73
 
82
74
  app.state.config = {
83
75
  "host": cls._config.get('Host', '0.0.0.0'),
@@ -87,95 +79,94 @@ class Services(metaclass=SingletonMeta):
87
79
  }
88
80
 
89
81
  if middleware:
90
- middleware(app, cls._config)
82
+ middleware(app, config)
91
83
 
92
84
  if nacos_service:
93
- nacos_service(cls._config)
85
+ nacos_service(config)
94
86
 
95
87
  if logging_service:
96
- logging_service(cls._config)
97
-
98
- sy_sentry_init()
99
-
88
+ logging_service(config)
89
+
90
+ # ========== 处理数据库服务 ==========
91
+ # 清空之前的待执行列表(防止热重载时重复)
92
+ cls._pending_async_db_setup = []
93
+
94
+ if database_service:
95
+ # 解析配置并区分同步/异步
96
+ items = [database_service] if isinstance(
97
+ database_service, tuple) else database_service
98
+ for item in items:
99
+ db_setup_func, db_name = item
100
+ if asyncio.iscoroutinefunction(db_setup_func):
101
+ # 如果是异步函数,加入待执行列表
102
+ logging.info(f"检测到异步数据库服务: {db_name},将在应用启动时初始化")
103
+ cls._pending_async_db_setup.append(item)
104
+ else:
105
+ # 如果是同步函数,立即执行
106
+ logging.info(f"执行同步数据库服务: {db_name}")
107
+ try:
108
+ db_setup_func(config, db_name)
109
+ except Exception as e:
110
+ logging.error(
111
+ f"同步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
112
+ raise
113
+
114
+ # 创建组合生命周期管理器
100
115
  @asynccontextmanager
101
- async def combined_lifespan(app_instance: FastAPI) -> AsyncGenerator[None, None]:
102
- # 获取 Services 实例
103
- instance = cls(config, app_instance)
116
+ async def combined_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
117
+ # 1. 执行Services自身的初始化
118
+ instance = cls(config, app)
119
+
120
+ # ========== 执行挂起的异步数据库初始化 ==========
121
+ if cls._pending_async_db_setup:
122
+ logging.info("开始执行异步数据库初始化...")
123
+ for db_setup_func, db_name in cls._pending_async_db_setup:
124
+ try:
125
+ await db_setup_func(config, db_name)
126
+ logging.info(f"异步数据库服务 {db_name} 初始化成功")
127
+ except Exception as e:
128
+ logging.error(
129
+ f"异步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
130
+ raise
131
+
132
+ # ========== 初始化 MQ ==========
133
+ has_valid_listeners = bool(
134
+ rabbitmq_listeners and len(rabbitmq_listeners) > 0)
135
+ has_valid_senders = bool(
136
+ rabbitmq_senders and len(rabbitmq_senders) > 0)
104
137
 
105
138
  try:
106
- # 1. 处理数据库服务
107
- if database_service:
108
- instance._pending_async_db_setup = []
109
-
110
- items = [database_service] if isinstance(
111
- database_service, tuple) else database_service
112
- for item in items:
113
- db_setup_func, db_name = item
114
- if asyncio.iscoroutinefunction(db_setup_func):
115
- logging.info(f"注册异步数据库服务: {db_name}")
116
- instance._pending_async_db_setup.append(item)
117
- else:
118
- logging.info(f"执行同步数据库服务: {db_name}")
119
- try:
120
- db_setup_func(config, db_name)
121
- except Exception as e:
122
- logging.error(
123
- f"同步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
124
- raise
125
-
126
- # 2. 执行挂起的异步数据库初始化
127
- if instance._pending_async_db_setup:
128
- logging.info("开始执行异步数据库初始化...")
129
- for db_setup_func, db_name in instance._pending_async_db_setup:
130
- try:
131
- await db_setup_func(config, db_name)
132
- logging.info(f"异步数据库服务 {db_name} 初始化成功")
133
- except Exception as e:
134
- logging.error(
135
- f"异步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
136
- raise
137
-
138
- # 3. 初始化 MQ
139
- has_valid_listeners = bool(
140
- rabbitmq_listeners and len(rabbitmq_listeners) > 0)
141
- has_valid_senders = bool(
142
- rabbitmq_senders and len(rabbitmq_senders) > 0)
143
-
144
- try:
145
- if has_valid_listeners or has_valid_senders:
146
- await instance._setup_mq_async(
147
- rabbitmq_listeners=rabbitmq_listeners if has_valid_listeners else None,
148
- rabbitmq_senders=rabbitmq_senders if has_valid_senders else None,
149
- has_listeners=has_valid_listeners,
150
- has_senders=has_valid_senders
151
- )
152
- cls._initialized = True
153
- logging.info("Services初始化完成")
154
- except Exception as e:
155
- logging.error(f"MQ初始化失败: {str(e)}", exc_info=True)
156
- raise
139
+ if has_valid_listeners or has_valid_senders:
140
+ await instance._setup_mq_async(
141
+ rabbitmq_listeners=rabbitmq_listeners if has_valid_listeners else None,
142
+ rabbitmq_senders=rabbitmq_senders if has_valid_senders else None,
143
+ has_listeners=has_valid_listeners,
144
+ has_senders=has_valid_senders
145
+ )
146
+ cls._initialized = True
147
+ logging.info("Services初始化完成")
148
+ except Exception as e:
149
+ logging.error(f"Services初始化失败: {str(e)}", exc_info=True)
150
+ raise
157
151
 
158
- app_instance.state.services = instance
152
+ app.state.services = instance
159
153
 
160
- # 4. 执行用户定义的生命周期
161
- if cls._user_lifespan:
162
- async with cls._user_lifespan(app_instance):
163
- yield
164
- else:
154
+ # 2. 执行用户定义的生命周期
155
+ if cls._user_lifespan:
156
+ async with cls._user_lifespan(app):
165
157
  yield
158
+ else:
159
+ yield
166
160
 
167
- except Exception:
168
- # 如果启动过程中发生任何异常,确保进入 shutdown
169
- logging.error("启动阶段发生异常,准备执行清理...")
170
- raise
171
- finally:
172
- # 无论成功或失败,都会执行关闭逻辑
173
- await cls.shutdown()
174
- logging.info("Services已关闭")
161
+ # 3. 执行Services的关闭逻辑
162
+ await cls.shutdown()
163
+ logging.info("Services已关闭")
175
164
 
176
165
  app.router.lifespan_context = combined_lifespan
177
166
  return app
178
167
 
168
+ # 移除了 _setup_database_static,因为逻辑已内联到 plugins 中
169
+
179
170
  async def _setup_mq_async(
180
171
  self,
181
172
  rabbitmq_listeners: Optional[List[RabbitMQListenerConfig]] = None,
@@ -191,21 +182,12 @@ class Services(metaclass=SingletonMeta):
191
182
  RabbitMQService.init(self._config, has_listeners, has_senders)
192
183
 
193
184
  start_time = asyncio.get_event_loop().time()
194
- timeout = 30 # 超时时间秒
195
-
196
- # 等待连接池初始化
197
- while not (RabbitMQService._connection_pool and RabbitMQService._connection_pool._initialized) \
198
- and not RabbitMQService._is_shutdown:
199
- if asyncio.get_event_loop().time() - start_time > timeout:
200
- logging.error("RabbitMQ连接池初始化超时")
201
- raise TimeoutError(f"RabbitMQ连接池初始化超时({timeout}秒)")
202
-
203
- logging.debug("等待RabbitMQ连接池初始化...")
185
+ while not (RabbitMQService._connection_pool and RabbitMQService._connection_pool._initialized) and not RabbitMQService._is_shutdown:
186
+ if asyncio.get_event_loop().time() - start_time > 30:
187
+ raise TimeoutError("RabbitMQ连接池初始化超时(30秒)")
188
+ logging.info("等待RabbitMQ连接池初始化...")
204
189
  await asyncio.sleep(0.5)
205
190
 
206
- if RabbitMQService._is_shutdown:
207
- raise RuntimeError("RabbitMQService 在初始化期间被关闭")
208
-
209
191
  if has_senders and rabbitmq_senders:
210
192
  if has_listeners and rabbitmq_listeners:
211
193
  for sender in rabbitmq_senders:
@@ -225,8 +207,11 @@ class Services(metaclass=SingletonMeta):
225
207
 
226
208
  async def _setup_senders_async(self, rabbitmq_senders, has_listeners: bool):
227
209
  """设置发送器"""
210
+ Services._registered_senders = [
211
+ sender.queue_name for sender in rabbitmq_senders]
228
212
  await RabbitMQService.setup_senders(rabbitmq_senders, has_listeners)
229
- logging.info(f"RabbitMQ发送器注册完成")
213
+ Services._registered_senders = RabbitMQService._sender_client_names
214
+ logging.info(f"已注册的RabbitMQ发送器: {Services._registered_senders}")
230
215
 
231
216
  async def _setup_listeners_async(self, rabbitmq_listeners, has_senders: bool):
232
217
  """设置监听器"""
@@ -251,12 +236,14 @@ class Services(metaclass=SingletonMeta):
251
236
 
252
237
  for attempt in range(max_retries):
253
238
  try:
254
- # 依赖 RabbitMQService 的内部状态
255
- sender = await RabbitMQService.get_sender(queue_name)
239
+ if queue_name not in cls._registered_senders:
240
+ cls._registered_senders = RabbitMQService._sender_client_names
241
+ if queue_name not in cls._registered_senders:
242
+ raise ValueError(f"发送器 {queue_name} 未注册")
256
243
 
244
+ sender = await RabbitMQService.get_sender(queue_name)
257
245
  if not sender:
258
- raise ValueError(
259
- f"发送器 '{queue_name}' 不存在或未在 RabbitMQService 中注册")
246
+ raise ValueError(f"发送器 '{queue_name}' 不存在或连接无效")
260
247
 
261
248
  await RabbitMQService.send_message(data, queue_name, **kwargs)
262
249
  logging.info(f"消息发送成功(尝试 {attempt+1}/{max_retries})")
@@ -278,20 +265,7 @@ class Services(metaclass=SingletonMeta):
278
265
  if RabbitMQService._is_shutdown:
279
266
  logging.info("RabbitMQService已关闭,无需重复操作")
280
267
  return
281
-
282
- try:
283
- await RabbitMQService.shutdown()
284
- except Exception as e:
285
- logging.error(f"关闭 RabbitMQService 时发生异常: {e}", exc_info=True)
286
-
268
+ await RabbitMQService.shutdown()
287
269
  cls._initialized = False
288
-
289
- # 清理实例数据
290
- if cls._instance:
291
- cls._instance._pending_async_db_setup.clear()
292
-
293
- # 这对于热重载(reload)时防止旧实例内存泄漏至关重要
294
- if cls._app:
295
- cls._app.state.services = None
296
-
270
+ cls._registered_senders.clear()
297
271
  logging.info("所有服务已关闭")