sycommon-python-lib 0.1.52a0__tar.gz → 0.1.55b1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/PKG-INFO +10 -9
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/pyproject.toml +10 -9
- sycommon_python_lib-0.1.55b1/src/sycommon/database/async_base_db_service.py +36 -0
- sycommon_python_lib-0.1.55b1/src/sycommon/database/async_database_service.py +96 -0
- sycommon_python_lib-0.1.55b1/src/sycommon/logging/async_sql_logger.py +65 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/logging/kafka_log.py +8 -8
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/middleware.py +1 -1
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/traceid.py +6 -4
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/services.py +71 -60
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/feign.py +9 -5
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/feign_client.py +4 -2
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/nacos_service.py +3 -0
- sycommon_python_lib-0.1.55b1/src/sycommon/tools/snowflake.py +300 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon_python_lib.egg-info/PKG-INFO +10 -9
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon_python_lib.egg-info/SOURCES.txt +3 -0
- sycommon_python_lib-0.1.55b1/src/sycommon_python_lib.egg-info/requires.txt +16 -0
- sycommon_python_lib-0.1.52a0/src/sycommon/tools/snowflake.py +0 -33
- sycommon_python_lib-0.1.52a0/src/sycommon_python_lib.egg-info/requires.txt +0 -15
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/README.md +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/setup.cfg +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/command/cli.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/Config.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/DatabaseConfig.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/EmbeddingConfig.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/LLMConfig.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/MQConfig.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/RerankerConfig.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/config/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/database/base_db_service.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/database/database_service.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/health/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/health/health_check.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/health/metrics.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/health/ping.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/logging/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/logging/logger_wrapper.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/logging/sql_logger.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/context.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/cors.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/docs.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/exception.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/monitor_memory.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/mq.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/timeout.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/base_http.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/log.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/mqlistener_config.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/mqmsg_model.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/mqsend_config.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/models/sso_user.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/rabbitmq/rabbitmq_client.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/rabbitmq/rabbitmq_pool.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/rabbitmq/rabbitmq_service.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/sse/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/sse/event.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/sse/sse.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/example.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/example2.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/synacos/param.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/tools/__init__.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/tools/docs.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/tools/timing.py +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon_python_lib.egg-info/dependency_links.txt +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon_python_lib.egg-info/entry_points.txt +0 -0
- {sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon_python_lib.egg-info/top_level.txt +0 -0
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sycommon-python-lib
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.55b1
|
|
4
4
|
Summary: Add your description here
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: aio-pika>=9.5.8
|
|
8
8
|
Requires-Dist: aiohttp>=3.13.2
|
|
9
|
+
Requires-Dist: aiomysql>=0.3.2
|
|
9
10
|
Requires-Dist: decorator>=5.2.1
|
|
10
|
-
Requires-Dist: fastapi>=0.
|
|
11
|
-
Requires-Dist: kafka-python>=2.
|
|
11
|
+
Requires-Dist: fastapi>=0.127.0
|
|
12
|
+
Requires-Dist: kafka-python>=2.3.0
|
|
12
13
|
Requires-Dist: loguru>=0.7.3
|
|
13
14
|
Requires-Dist: mysql-connector-python>=9.5.0
|
|
14
|
-
Requires-Dist: nacos-sdk-python
|
|
15
|
-
Requires-Dist:
|
|
15
|
+
Requires-Dist: nacos-sdk-python<3.0,>=2.0.9
|
|
16
|
+
Requires-Dist: psutil>=7.1.3
|
|
17
|
+
Requires-Dist: pydantic>=2.12.5
|
|
16
18
|
Requires-Dist: python-dotenv>=1.2.1
|
|
17
19
|
Requires-Dist: pyyaml>=6.0.3
|
|
18
|
-
Requires-Dist: sqlalchemy>=2.0.
|
|
19
|
-
Requires-Dist: starlette>=0.
|
|
20
|
-
Requires-Dist:
|
|
21
|
-
Requires-Dist: uvicorn>=0.38.0
|
|
20
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.45
|
|
21
|
+
Requires-Dist: starlette>=0.50.0
|
|
22
|
+
Requires-Dist: uvicorn>=0.40.0
|
|
22
23
|
|
|
23
24
|
# sycommon-python-lib
|
|
24
25
|
|
|
@@ -1,25 +1,26 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "sycommon-python-lib"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.55-beta1"
|
|
4
4
|
description = "Add your description here"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.10"
|
|
7
7
|
dependencies = [
|
|
8
8
|
"aio-pika>=9.5.8",
|
|
9
9
|
"aiohttp>=3.13.2",
|
|
10
|
+
"aiomysql>=0.3.2",
|
|
10
11
|
"decorator>=5.2.1",
|
|
11
|
-
"fastapi>=0.
|
|
12
|
-
"kafka-python>=2.
|
|
12
|
+
"fastapi>=0.127.0",
|
|
13
|
+
"kafka-python>=2.3.0",
|
|
13
14
|
"loguru>=0.7.3",
|
|
14
15
|
"mysql-connector-python>=9.5.0",
|
|
15
|
-
"nacos-sdk-python>=2.0.9",
|
|
16
|
-
"
|
|
16
|
+
"nacos-sdk-python>=2.0.9,<3.0",
|
|
17
|
+
"psutil>=7.1.3",
|
|
18
|
+
"pydantic>=2.12.5",
|
|
17
19
|
"python-dotenv>=1.2.1",
|
|
18
20
|
"pyyaml>=6.0.3",
|
|
19
|
-
"sqlalchemy>=2.0.
|
|
20
|
-
"starlette>=0.
|
|
21
|
-
"
|
|
22
|
-
"uvicorn>=0.38.0",
|
|
21
|
+
"sqlalchemy[asyncio]>=2.0.45",
|
|
22
|
+
"starlette>=0.50.0",
|
|
23
|
+
"uvicorn>=0.40.0",
|
|
23
24
|
]
|
|
24
25
|
|
|
25
26
|
[tool.setuptools]
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from contextlib import asynccontextmanager
|
|
2
|
+
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
3
|
+
from sycommon.config.Config import SingletonMeta
|
|
4
|
+
from sycommon.database.async_database_service import AsyncDatabaseService
|
|
5
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class AsyncBaseDBService(metaclass=SingletonMeta):
|
|
9
|
+
"""数据库操作基础服务类,封装异步会话管理功能"""
|
|
10
|
+
|
|
11
|
+
def __init__(self):
|
|
12
|
+
# 获取异步引擎 (假设 DatabaseService.engine() 返回的是 AsyncEngine)
|
|
13
|
+
self.engine = AsyncDatabaseService.engine()
|
|
14
|
+
|
|
15
|
+
# 创建异步 Session 工厂
|
|
16
|
+
# class_=AsyncSession 是必须的,用于指定生成的是异步会话
|
|
17
|
+
self.Session = async_sessionmaker(
|
|
18
|
+
bind=self.engine,
|
|
19
|
+
class_=AsyncSession,
|
|
20
|
+
expire_on_commit=False
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
@asynccontextmanager
|
|
24
|
+
async def session(self):
|
|
25
|
+
"""
|
|
26
|
+
异步数据库会话上下文管理器
|
|
27
|
+
自动处理会话的创建、提交、回滚和关闭
|
|
28
|
+
"""
|
|
29
|
+
async with self.Session() as session:
|
|
30
|
+
try:
|
|
31
|
+
yield session
|
|
32
|
+
await session.commit()
|
|
33
|
+
except Exception as e:
|
|
34
|
+
await session.rollback()
|
|
35
|
+
SYLogger.error(f"Database operation failed: {str(e)}")
|
|
36
|
+
raise
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
|
|
2
|
+
from sqlalchemy import text
|
|
3
|
+
|
|
4
|
+
from sycommon.config.Config import SingletonMeta
|
|
5
|
+
from sycommon.config.DatabaseConfig import DatabaseConfig, convert_dict_keys
|
|
6
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
7
|
+
from sycommon.logging.async_sql_logger import AsyncSQLTraceLogger
|
|
8
|
+
from sycommon.synacos.nacos_service import NacosService
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class AsyncDatabaseService(metaclass=SingletonMeta):
|
|
12
|
+
_engine = None
|
|
13
|
+
|
|
14
|
+
@staticmethod
|
|
15
|
+
async def setup_database(config: dict, shareConfigKey: str):
|
|
16
|
+
common = NacosService(config).share_configs.get(shareConfigKey, {})
|
|
17
|
+
if common and common.get('spring', {}).get('datasource', None):
|
|
18
|
+
databaseConfig = common.get('spring', {}).get('datasource', None)
|
|
19
|
+
converted_dict = convert_dict_keys(databaseConfig)
|
|
20
|
+
db_config = DatabaseConfig.model_validate(converted_dict)
|
|
21
|
+
|
|
22
|
+
# 初始化 DatabaseConnector (传入配置)
|
|
23
|
+
connector = AsyncDatabaseConnector(db_config)
|
|
24
|
+
|
|
25
|
+
# 赋值 engine
|
|
26
|
+
AsyncDatabaseService._engine = connector.engine
|
|
27
|
+
|
|
28
|
+
# 执行异步测试连接
|
|
29
|
+
if not await connector.test_connection():
|
|
30
|
+
raise Exception("Database connection test failed")
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def engine():
|
|
34
|
+
return AsyncDatabaseService._engine
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AsyncDatabaseConnector(metaclass=SingletonMeta):
|
|
38
|
+
def __init__(self, db_config: DatabaseConfig):
|
|
39
|
+
# 从 DatabaseConfig 中提取数据库连接信息
|
|
40
|
+
self.db_user = db_config.username
|
|
41
|
+
self.db_password = db_config.password
|
|
42
|
+
|
|
43
|
+
# 提取 URL 中的主机、端口和数据库名
|
|
44
|
+
url_parts = db_config.url.split('//')[1].split('/')
|
|
45
|
+
host_port = url_parts[0].split(':')
|
|
46
|
+
self.db_host = host_port[0]
|
|
47
|
+
self.db_port = host_port[1]
|
|
48
|
+
self.db_name = url_parts[1].split('?')[0]
|
|
49
|
+
|
|
50
|
+
# 提取 URL 中的参数
|
|
51
|
+
params_str = url_parts[1].split('?')[1] if len(
|
|
52
|
+
url_parts[1].split('?')) > 1 else ''
|
|
53
|
+
params = {}
|
|
54
|
+
for param in params_str.split('&'):
|
|
55
|
+
if param:
|
|
56
|
+
key, value = param.split('=')
|
|
57
|
+
params[key] = value
|
|
58
|
+
|
|
59
|
+
# 在params中去掉指定的参数
|
|
60
|
+
for key in ['useUnicode', 'characterEncoding', 'serverTimezone', 'zeroDateTimeBehavior']:
|
|
61
|
+
if key in params:
|
|
62
|
+
del params[key]
|
|
63
|
+
|
|
64
|
+
# 构建数据库连接 URL
|
|
65
|
+
# 注意:这里将 mysqlconnector 替换为 aiomysql 以支持异步
|
|
66
|
+
self.db_url = f'mysql+aiomysql://{self.db_user}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}'
|
|
67
|
+
|
|
68
|
+
SYLogger.info(f"Database URL: {self.db_url}")
|
|
69
|
+
|
|
70
|
+
# 优化连接池配置
|
|
71
|
+
# 使用 create_async_engine 替代 create_engine
|
|
72
|
+
self.engine = create_async_engine(
|
|
73
|
+
self.db_url,
|
|
74
|
+
connect_args=params,
|
|
75
|
+
pool_size=10, # 连接池大小
|
|
76
|
+
max_overflow=20, # 最大溢出连接数
|
|
77
|
+
pool_timeout=30, # 连接超时时间(秒)
|
|
78
|
+
pool_recycle=3600, # 连接回收时间(秒)
|
|
79
|
+
pool_pre_ping=True, # 每次获取连接前检查连接是否有效
|
|
80
|
+
echo=False, # 打印 SQL 语句
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# 注册 SQL 日志拦截器 (注意:SQLTraceLogger 需要支持异步引擎,或者您可能需要调整日志逻辑)
|
|
84
|
+
# 假设 SQLTraceLogger.setup_sql_logging 能够处理 AsyncEngine
|
|
85
|
+
AsyncSQLTraceLogger.setup_sql_logging(self.engine)
|
|
86
|
+
|
|
87
|
+
async def test_connection(self):
|
|
88
|
+
try:
|
|
89
|
+
# 异步上下文管理器
|
|
90
|
+
async with self.engine.connect() as connection:
|
|
91
|
+
# 执行简单查询
|
|
92
|
+
await connection.execute(text("SELECT 1"))
|
|
93
|
+
return True
|
|
94
|
+
except Exception as e:
|
|
95
|
+
SYLogger.error(f"Database connection test failed: {e}")
|
|
96
|
+
return False
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
from sqlalchemy import event
|
|
2
|
+
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
3
|
+
from sycommon.logging.kafka_log import SYLogger
|
|
4
|
+
import time
|
|
5
|
+
from datetime import datetime
|
|
6
|
+
from decimal import Decimal
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class AsyncSQLTraceLogger:
|
|
10
|
+
@staticmethod
|
|
11
|
+
def setup_sql_logging(engine):
|
|
12
|
+
"""
|
|
13
|
+
为 SQLAlchemy 异步引擎注册事件监听器
|
|
14
|
+
注意:必须监听 engine.sync_engine,而不能直接监听 AsyncEngine
|
|
15
|
+
"""
|
|
16
|
+
def serialize_params(params):
|
|
17
|
+
"""处理特殊类型参数的序列化"""
|
|
18
|
+
if isinstance(params, (list, tuple)):
|
|
19
|
+
return [serialize_params(p) for p in params]
|
|
20
|
+
elif isinstance(params, dict):
|
|
21
|
+
return {k: serialize_params(v) for k, v in params.items()}
|
|
22
|
+
elif isinstance(params, datetime):
|
|
23
|
+
return params.isoformat()
|
|
24
|
+
elif isinstance(params, Decimal):
|
|
25
|
+
return float(params)
|
|
26
|
+
else:
|
|
27
|
+
return params
|
|
28
|
+
|
|
29
|
+
# ========== 核心修改 ==========
|
|
30
|
+
# 必须通过 engine.sync_engine 来获取底层的同步引擎进行监听
|
|
31
|
+
target = engine.sync_engine
|
|
32
|
+
|
|
33
|
+
@event.listens_for(target, "after_cursor_execute")
|
|
34
|
+
def after_cursor_execute(
|
|
35
|
+
conn, cursor, statement, parameters, context, executemany
|
|
36
|
+
):
|
|
37
|
+
try:
|
|
38
|
+
# 从连接选项中获取开始时间
|
|
39
|
+
# conn 在这里是同步连接对象
|
|
40
|
+
start_time = conn.info.get('_start_time') or \
|
|
41
|
+
conn._execution_options.get("_start_time", time.time())
|
|
42
|
+
|
|
43
|
+
execution_time = (time.time() - start_time) * 1000
|
|
44
|
+
|
|
45
|
+
sql_log = {
|
|
46
|
+
"type": "SQL",
|
|
47
|
+
"statement": statement,
|
|
48
|
+
"parameters": serialize_params(parameters),
|
|
49
|
+
"execution_time_ms": round(execution_time, 2),
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
# 注意:SYLogger.info 必须是线程安全的或非阻塞的,否则可能影响异步性能
|
|
53
|
+
SYLogger.info(f"SQL执行: {sql_log}")
|
|
54
|
+
except Exception as e:
|
|
55
|
+
SYLogger.error(f"SQL日志处理失败: {str(e)}")
|
|
56
|
+
|
|
57
|
+
@event.listens_for(target, "before_cursor_execute")
|
|
58
|
+
def before_cursor_execute(
|
|
59
|
+
conn, cursor, statement, parameters, context, executemany
|
|
60
|
+
):
|
|
61
|
+
try:
|
|
62
|
+
# 记录开始时间到 execution_options
|
|
63
|
+
conn = conn.execution_options(_start_time=time.time())
|
|
64
|
+
except Exception as e:
|
|
65
|
+
SYLogger.error(f"SQL开始时间记录失败: {str(e)}")
|
{sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/logging/kafka_log.py
RENAMED
|
@@ -114,7 +114,7 @@ class KafkaLogger(metaclass=SingletonMeta):
|
|
|
114
114
|
trace_id = None
|
|
115
115
|
|
|
116
116
|
if not trace_id:
|
|
117
|
-
trace_id = SYLogger.get_trace_id() or Snowflake.
|
|
117
|
+
trace_id = SYLogger.get_trace_id() or Snowflake.id
|
|
118
118
|
|
|
119
119
|
# 获取线程/协程信息
|
|
120
120
|
thread_info = SYLogger._get_execution_context()
|
|
@@ -173,7 +173,7 @@ class KafkaLogger(metaclass=SingletonMeta):
|
|
|
173
173
|
"className": "",
|
|
174
174
|
"sqlCost": 0,
|
|
175
175
|
"size": len(str(message)),
|
|
176
|
-
"uid": int(Snowflake.
|
|
176
|
+
"uid": int(Snowflake.id) # 独立新的id
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
# 智能队列管理
|
|
@@ -212,7 +212,7 @@ class KafkaLogger(metaclass=SingletonMeta):
|
|
|
212
212
|
return
|
|
213
213
|
|
|
214
214
|
# 获取当前的trace_id
|
|
215
|
-
trace_id = SYLogger.get_trace_id() or Snowflake.
|
|
215
|
+
trace_id = SYLogger.get_trace_id() or Snowflake.id
|
|
216
216
|
|
|
217
217
|
# 构建错误日志
|
|
218
218
|
error_log = {
|
|
@@ -459,7 +459,7 @@ class SYLogger:
|
|
|
459
459
|
|
|
460
460
|
@staticmethod
|
|
461
461
|
def _log(msg: any, level: str = "INFO"):
|
|
462
|
-
trace_id = SYLogger.get_trace_id()
|
|
462
|
+
trace_id = SYLogger.get_trace_id() or Snowflake.id
|
|
463
463
|
|
|
464
464
|
if isinstance(msg, dict) or isinstance(msg, list):
|
|
465
465
|
msg_str = json.dumps(msg, ensure_ascii=False)
|
|
@@ -473,7 +473,7 @@ class SYLogger:
|
|
|
473
473
|
request_log = {}
|
|
474
474
|
if level == "ERROR":
|
|
475
475
|
request_log = {
|
|
476
|
-
"trace_id": str(trace_id) if trace_id else Snowflake.
|
|
476
|
+
"trace_id": str(trace_id) if trace_id else Snowflake.id,
|
|
477
477
|
"message": msg_str,
|
|
478
478
|
"traceback": traceback.format_exc(),
|
|
479
479
|
"level": level,
|
|
@@ -481,7 +481,7 @@ class SYLogger:
|
|
|
481
481
|
}
|
|
482
482
|
else:
|
|
483
483
|
request_log = {
|
|
484
|
-
"trace_id": str(trace_id) if trace_id else Snowflake.
|
|
484
|
+
"trace_id": str(trace_id) if trace_id else Snowflake.id,
|
|
485
485
|
"message": msg_str,
|
|
486
486
|
"level": level,
|
|
487
487
|
"threadName": thread_info
|
|
@@ -521,7 +521,7 @@ class SYLogger:
|
|
|
521
521
|
@staticmethod
|
|
522
522
|
def exception(msg: any, *args, **kwargs):
|
|
523
523
|
"""记录异常信息,包括完整堆栈"""
|
|
524
|
-
trace_id = SYLogger.get_trace_id()
|
|
524
|
+
trace_id = SYLogger.get_trace_id() or Snowflake.id
|
|
525
525
|
|
|
526
526
|
if isinstance(msg, dict) or isinstance(msg, list):
|
|
527
527
|
msg_str = json.dumps(msg, ensure_ascii=False)
|
|
@@ -533,7 +533,7 @@ class SYLogger:
|
|
|
533
533
|
|
|
534
534
|
# 构建包含异常堆栈的日志
|
|
535
535
|
request_log = {
|
|
536
|
-
"trace_id": str(trace_id) if trace_id else Snowflake.
|
|
536
|
+
"trace_id": str(trace_id) if trace_id else Snowflake.id,
|
|
537
537
|
"message": msg_str,
|
|
538
538
|
"level": "ERROR",
|
|
539
539
|
"threadName": thread_info
|
{sycommon_python_lib-0.1.52a0 → sycommon_python_lib-0.1.55b1}/src/sycommon/middleware/traceid.py
RENAMED
|
@@ -12,7 +12,7 @@ def setup_trace_id_handler(app):
|
|
|
12
12
|
# 生成或获取 traceId
|
|
13
13
|
trace_id = request.headers.get("x-traceId-header")
|
|
14
14
|
if not trace_id:
|
|
15
|
-
trace_id = Snowflake.
|
|
15
|
+
trace_id = Snowflake.id
|
|
16
16
|
|
|
17
17
|
# 设置 trace_id 上下文
|
|
18
18
|
token = SYLogger.set_trace_id(trace_id)
|
|
@@ -101,9 +101,11 @@ def setup_trace_id_handler(app):
|
|
|
101
101
|
if "application/json" in content_type and not content_disposition.startswith("attachment"):
|
|
102
102
|
try:
|
|
103
103
|
data = json.loads(response_body)
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
data
|
|
104
|
+
new_body = response_body
|
|
105
|
+
if data:
|
|
106
|
+
data["traceId"] = trace_id
|
|
107
|
+
new_body = json.dumps(
|
|
108
|
+
data, ensure_ascii=False).encode()
|
|
107
109
|
|
|
108
110
|
# 创建新响应,确保Content-Length正确
|
|
109
111
|
response = Response(
|
|
@@ -23,6 +23,9 @@ class Services(metaclass=SingletonMeta):
|
|
|
23
23
|
_user_lifespan: Optional[Callable] = None
|
|
24
24
|
_shutdown_lock: asyncio.Lock = asyncio.Lock()
|
|
25
25
|
|
|
26
|
+
# 用于存储待执行的异步数据库初始化任务
|
|
27
|
+
_pending_async_db_setup: List[Tuple[Callable, str]] = []
|
|
28
|
+
|
|
26
29
|
def __init__(self, config: dict, app: FastAPI):
|
|
27
30
|
if not Services._config:
|
|
28
31
|
Services._config = config
|
|
@@ -48,25 +51,24 @@ class Services(metaclass=SingletonMeta):
|
|
|
48
51
|
nacos_service: Optional[Callable[[dict], None]] = None,
|
|
49
52
|
logging_service: Optional[Callable[[dict], None]] = None,
|
|
50
53
|
database_service: Optional[Union[
|
|
51
|
-
Tuple[Callable
|
|
52
|
-
List[Tuple[Callable
|
|
54
|
+
Tuple[Callable, str],
|
|
55
|
+
List[Tuple[Callable, str]]
|
|
53
56
|
]] = None,
|
|
54
57
|
rabbitmq_listeners: Optional[List[RabbitMQListenerConfig]] = None,
|
|
55
58
|
rabbitmq_senders: Optional[List[RabbitMQSendConfig]] = None
|
|
56
59
|
) -> FastAPI:
|
|
57
60
|
load_dotenv()
|
|
58
|
-
# 保存应用实例和配置
|
|
59
61
|
cls._app = app
|
|
60
62
|
cls._config = config
|
|
61
63
|
cls._user_lifespan = app.router.lifespan_context
|
|
62
|
-
|
|
64
|
+
|
|
63
65
|
applications.get_swagger_ui_html = custom_swagger_ui_html
|
|
64
66
|
applications.get_redoc_html = custom_redoc_html
|
|
65
|
-
|
|
67
|
+
|
|
66
68
|
if not cls._config:
|
|
67
69
|
config = yaml.safe_load(open('app.yaml', 'r', encoding='utf-8'))
|
|
68
70
|
cls._config = config
|
|
69
|
-
|
|
71
|
+
|
|
70
72
|
app.state.config = {
|
|
71
73
|
"host": cls._config.get('Host', '0.0.0.0'),
|
|
72
74
|
"port": cls._config.get('Port', 8080),
|
|
@@ -74,7 +76,6 @@ class Services(metaclass=SingletonMeta):
|
|
|
74
76
|
"h11_max_incomplete_event_size": cls._config.get('H11MaxIncompleteEventSize', 1024 * 1024 * 10)
|
|
75
77
|
}
|
|
76
78
|
|
|
77
|
-
# 立即配置非异步服务(在应用启动前)
|
|
78
79
|
if middleware:
|
|
79
80
|
middleware(app, config)
|
|
80
81
|
|
|
@@ -84,25 +85,62 @@ class Services(metaclass=SingletonMeta):
|
|
|
84
85
|
if logging_service:
|
|
85
86
|
logging_service(config)
|
|
86
87
|
|
|
88
|
+
# ========== 处理数据库服务 ==========
|
|
89
|
+
# 清空之前的待执行列表(防止热重载时重复)
|
|
90
|
+
cls._pending_async_db_setup = []
|
|
91
|
+
|
|
87
92
|
if database_service:
|
|
88
|
-
|
|
93
|
+
# 解析配置并区分同步/异步
|
|
94
|
+
items = [database_service] if isinstance(
|
|
95
|
+
database_service, tuple) else database_service
|
|
96
|
+
for item in items:
|
|
97
|
+
db_setup_func, db_name = item
|
|
98
|
+
if asyncio.iscoroutinefunction(db_setup_func):
|
|
99
|
+
# 如果是异步函数,加入待执行列表
|
|
100
|
+
logging.info(f"检测到异步数据库服务: {db_name},将在应用启动时初始化")
|
|
101
|
+
cls._pending_async_db_setup.append(item)
|
|
102
|
+
else:
|
|
103
|
+
# 如果是同步函数,立即执行
|
|
104
|
+
logging.info(f"执行同步数据库服务: {db_name}")
|
|
105
|
+
try:
|
|
106
|
+
db_setup_func(config, db_name)
|
|
107
|
+
except Exception as e:
|
|
108
|
+
logging.error(
|
|
109
|
+
f"同步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
110
|
+
raise
|
|
89
111
|
|
|
90
112
|
# 创建组合生命周期管理器
|
|
91
113
|
@asynccontextmanager
|
|
92
114
|
async def combined_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|
93
115
|
# 1. 执行Services自身的初始化
|
|
94
116
|
instance = cls(config, app)
|
|
95
|
-
|
|
117
|
+
|
|
118
|
+
# ========== 执行挂起的异步数据库初始化 ==========
|
|
119
|
+
if cls._pending_async_db_setup:
|
|
120
|
+
logging.info("开始执行异步数据库初始化...")
|
|
121
|
+
for db_setup_func, db_name in cls._pending_async_db_setup:
|
|
122
|
+
try:
|
|
123
|
+
await db_setup_func(config, db_name)
|
|
124
|
+
logging.info(f"异步数据库服务 {db_name} 初始化成功")
|
|
125
|
+
except Exception as e:
|
|
126
|
+
logging.error(
|
|
127
|
+
f"异步数据库服务 {db_name} 初始化失败: {e}", exc_info=True)
|
|
128
|
+
raise
|
|
129
|
+
|
|
130
|
+
# ========== 初始化 MQ ==========
|
|
131
|
+
has_valid_listeners = bool(
|
|
96
132
|
rabbitmq_listeners and len(rabbitmq_listeners) > 0)
|
|
97
|
-
|
|
133
|
+
has_valid_senders = bool(
|
|
134
|
+
rabbitmq_senders and len(rabbitmq_senders) > 0)
|
|
98
135
|
|
|
99
136
|
try:
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
137
|
+
if has_valid_listeners or has_valid_senders:
|
|
138
|
+
await instance._setup_mq_async(
|
|
139
|
+
rabbitmq_listeners=rabbitmq_listeners if has_valid_listeners else None,
|
|
140
|
+
rabbitmq_senders=rabbitmq_senders if has_valid_senders else None,
|
|
141
|
+
has_listeners=has_valid_listeners,
|
|
142
|
+
has_senders=has_valid_senders
|
|
143
|
+
)
|
|
106
144
|
cls._initialized = True
|
|
107
145
|
logging.info("Services初始化完成")
|
|
108
146
|
except Exception as e:
|
|
@@ -114,28 +152,18 @@ class Services(metaclass=SingletonMeta):
|
|
|
114
152
|
# 2. 执行用户定义的生命周期
|
|
115
153
|
if cls._user_lifespan:
|
|
116
154
|
async with cls._user_lifespan(app):
|
|
117
|
-
yield
|
|
155
|
+
yield
|
|
118
156
|
else:
|
|
119
|
-
yield
|
|
157
|
+
yield
|
|
120
158
|
|
|
121
159
|
# 3. 执行Services的关闭逻辑
|
|
122
160
|
await cls.shutdown()
|
|
123
161
|
logging.info("Services已关闭")
|
|
124
162
|
|
|
125
|
-
# 设置组合生命周期
|
|
126
163
|
app.router.lifespan_context = combined_lifespan
|
|
127
|
-
|
|
128
164
|
return app
|
|
129
165
|
|
|
130
|
-
|
|
131
|
-
def _setup_database_static(database_service, config):
|
|
132
|
-
"""静态方法:设置数据库服务"""
|
|
133
|
-
if isinstance(database_service, tuple):
|
|
134
|
-
db_setup, db_name = database_service
|
|
135
|
-
db_setup(config, db_name)
|
|
136
|
-
elif isinstance(database_service, list):
|
|
137
|
-
for db_setup, db_name in database_service:
|
|
138
|
-
db_setup(config, db_name)
|
|
166
|
+
# 移除了 _setup_database_static,因为逻辑已内联到 plugins 中
|
|
139
167
|
|
|
140
168
|
async def _setup_mq_async(
|
|
141
169
|
self,
|
|
@@ -144,11 +172,13 @@ class Services(metaclass=SingletonMeta):
|
|
|
144
172
|
has_listeners: bool = False,
|
|
145
173
|
has_senders: bool = False,
|
|
146
174
|
):
|
|
147
|
-
"""异步设置MQ
|
|
148
|
-
|
|
175
|
+
"""异步设置MQ相关服务"""
|
|
176
|
+
if not (has_listeners or has_senders):
|
|
177
|
+
logging.info("无RabbitMQ监听器/发送器配置,跳过RabbitMQService初始化")
|
|
178
|
+
return
|
|
179
|
+
|
|
149
180
|
RabbitMQService.init(self._config, has_listeners, has_senders)
|
|
150
181
|
|
|
151
|
-
# 优化:等待连接池“存在且初始化完成”(避免提前执行后续逻辑)
|
|
152
182
|
start_time = asyncio.get_event_loop().time()
|
|
153
183
|
while not (RabbitMQService._connection_pool and RabbitMQService._connection_pool._initialized) and not RabbitMQService._is_shutdown:
|
|
154
184
|
if asyncio.get_event_loop().time() - start_time > 30:
|
|
@@ -156,42 +186,33 @@ class Services(metaclass=SingletonMeta):
|
|
|
156
186
|
logging.info("等待RabbitMQ连接池初始化...")
|
|
157
187
|
await asyncio.sleep(0.5)
|
|
158
188
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
# 判断是否有监听器,如果有遍历监听器列表,队列名一样将prefetch_count属性设置到发送器对象中
|
|
162
|
-
if rabbitmq_listeners:
|
|
189
|
+
if has_senders and rabbitmq_senders:
|
|
190
|
+
if has_listeners and rabbitmq_listeners:
|
|
163
191
|
for sender in rabbitmq_senders:
|
|
164
192
|
for listener in rabbitmq_listeners:
|
|
165
193
|
if sender.queue_name == listener.queue_name:
|
|
166
194
|
sender.prefetch_count = listener.prefetch_count
|
|
167
195
|
await self._setup_senders_async(rabbitmq_senders, has_listeners)
|
|
168
196
|
|
|
169
|
-
|
|
170
|
-
if rabbitmq_listeners:
|
|
197
|
+
if has_listeners and rabbitmq_listeners:
|
|
171
198
|
await self._setup_listeners_async(rabbitmq_listeners, has_senders)
|
|
172
199
|
|
|
173
|
-
# 验证初始化结果
|
|
174
200
|
if has_listeners:
|
|
175
|
-
|
|
176
|
-
listener_count = len(
|
|
177
|
-
RabbitMQService._consumer_tasks)
|
|
201
|
+
listener_count = len(RabbitMQService._consumer_tasks)
|
|
178
202
|
logging.info(f"监听器初始化完成,共启动 {listener_count} 个消费者")
|
|
179
203
|
if listener_count == 0:
|
|
180
204
|
logging.warning("未成功初始化任何监听器,请检查配置或MQ服务状态")
|
|
181
205
|
|
|
182
206
|
async def _setup_senders_async(self, rabbitmq_senders, has_listeners: bool):
|
|
183
|
-
"""
|
|
207
|
+
"""设置发送器"""
|
|
184
208
|
Services._registered_senders = [
|
|
185
209
|
sender.queue_name for sender in rabbitmq_senders]
|
|
186
|
-
|
|
187
|
-
# 将是否有监听器的信息传递给RabbitMQService(异步调用)
|
|
188
210
|
await RabbitMQService.setup_senders(rabbitmq_senders, has_listeners)
|
|
189
|
-
# 更新已注册的发送器(从RabbitMQService获取实际注册的名称)
|
|
190
211
|
Services._registered_senders = RabbitMQService._sender_client_names
|
|
191
212
|
logging.info(f"已注册的RabbitMQ发送器: {Services._registered_senders}")
|
|
192
213
|
|
|
193
214
|
async def _setup_listeners_async(self, rabbitmq_listeners, has_senders: bool):
|
|
194
|
-
"""
|
|
215
|
+
"""设置监听器"""
|
|
195
216
|
await RabbitMQService.setup_listeners(rabbitmq_listeners, has_senders)
|
|
196
217
|
|
|
197
218
|
@classmethod
|
|
@@ -202,7 +223,7 @@ class Services(metaclass=SingletonMeta):
|
|
|
202
223
|
max_retries: int = 3,
|
|
203
224
|
retry_delay: float = 1.0, **kwargs
|
|
204
225
|
) -> None:
|
|
205
|
-
"""
|
|
226
|
+
"""发送消息"""
|
|
206
227
|
if not cls._initialized or not cls._loop:
|
|
207
228
|
logging.error("Services not properly initialized!")
|
|
208
229
|
raise ValueError("服务未正确初始化")
|
|
@@ -213,18 +234,15 @@ class Services(metaclass=SingletonMeta):
|
|
|
213
234
|
|
|
214
235
|
for attempt in range(max_retries):
|
|
215
236
|
try:
|
|
216
|
-
# 验证发送器是否注册
|
|
217
237
|
if queue_name not in cls._registered_senders:
|
|
218
238
|
cls._registered_senders = RabbitMQService._sender_client_names
|
|
219
239
|
if queue_name not in cls._registered_senders:
|
|
220
240
|
raise ValueError(f"发送器 {queue_name} 未注册")
|
|
221
241
|
|
|
222
|
-
# 获取发送器(适配新的异步get_sender方法)
|
|
223
242
|
sender = await RabbitMQService.get_sender(queue_name)
|
|
224
243
|
if not sender:
|
|
225
244
|
raise ValueError(f"发送器 '{queue_name}' 不存在或连接无效")
|
|
226
245
|
|
|
227
|
-
# 发送消息(调用RabbitMQService的异步send_message)
|
|
228
246
|
await RabbitMQService.send_message(data, queue_name, **kwargs)
|
|
229
247
|
logging.info(f"消息发送成功(尝试 {attempt+1}/{max_retries})")
|
|
230
248
|
return
|
|
@@ -234,25 +252,18 @@ class Services(metaclass=SingletonMeta):
|
|
|
234
252
|
logging.error(
|
|
235
253
|
f"消息发送失败(已尝试 {max_retries} 次): {str(e)}", exc_info=True)
|
|
236
254
|
raise
|
|
237
|
-
|
|
238
255
|
logging.warning(
|
|
239
|
-
f"消息发送失败(尝试 {attempt+1}/{max_retries}): {str(e)},"
|
|
240
|
-
f"{retry_delay}秒后重试..."
|
|
241
|
-
)
|
|
256
|
+
f"消息发送失败(尝试 {attempt+1}/{max_retries}): {str(e)},{retry_delay}秒后重试...")
|
|
242
257
|
await asyncio.sleep(retry_delay)
|
|
243
258
|
|
|
244
259
|
@classmethod
|
|
245
260
|
async def shutdown(cls):
|
|
246
|
-
"""
|
|
261
|
+
"""关闭所有服务"""
|
|
247
262
|
async with cls._shutdown_lock:
|
|
248
263
|
if RabbitMQService._is_shutdown:
|
|
249
264
|
logging.info("RabbitMQService已关闭,无需重复操作")
|
|
250
265
|
return
|
|
251
|
-
|
|
252
|
-
# 关闭RabbitMQ服务(异步调用,内部会关闭所有客户端+消费任务)
|
|
253
266
|
await RabbitMQService.shutdown()
|
|
254
|
-
|
|
255
|
-
# 清理全局状态
|
|
256
267
|
cls._initialized = False
|
|
257
268
|
cls._registered_senders.clear()
|
|
258
269
|
logging.info("所有服务已关闭")
|