sycommon-python-lib 0.1.53__py3-none-any.whl → 0.1.55__py3-none-any.whl

sycommon/database/async_base_db_service.py

@@ -0,0 +1,36 @@
+from contextlib import asynccontextmanager
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.async_database_service import AsyncDatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+
+class AsyncBaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装异步会话管理功能"""
+
+    def __init__(self):
+        # 获取异步引擎 (假设 DatabaseService.engine() 返回的是 AsyncEngine)
+        self.engine = AsyncDatabaseService.engine()
+
+        # 创建异步 Session 工厂
+        # class_=AsyncSession 是必须的，用于指定生成的是异步会话
+        self.Session = async_sessionmaker(
+            bind=self.engine,
+            class_=AsyncSession,
+            expire_on_commit=False
+        )
+
+    @asynccontextmanager
+    async def session(self):
+        """
+        异步数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        async with self.Session() as session:
+            try:
+                yield session
+                await session.commit()
+            except Exception as e:
+                await session.rollback()
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise

sycommon/database/async_database_service.py

@@ -0,0 +1,96 @@
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+from sqlalchemy import text
+
+from sycommon.config.Config import SingletonMeta
+from sycommon.config.DatabaseConfig import DatabaseConfig, convert_dict_keys
+from sycommon.logging.kafka_log import SYLogger
+from sycommon.logging.async_sql_logger import AsyncSQLTraceLogger
+from sycommon.synacos.nacos_service import NacosService
+
+
+class AsyncDatabaseService(metaclass=SingletonMeta):
+    _engine = None
+
+    @staticmethod
+    async def setup_database(config: dict, shareConfigKey: str):
+        common = NacosService(config).share_configs.get(shareConfigKey, {})
+        if common and common.get('spring', {}).get('datasource', None):
+            databaseConfig = common.get('spring', {}).get('datasource', None)
+            converted_dict = convert_dict_keys(databaseConfig)
+            db_config = DatabaseConfig.model_validate(converted_dict)
+
+            # 初始化 DatabaseConnector (传入配置)
+            connector = AsyncDatabaseConnector(db_config)
+
+            # 赋值 engine
+            AsyncDatabaseService._engine = connector.engine
+
+            # 执行异步测试连接
+            if not await connector.test_connection():
+                raise Exception("Database connection test failed")
+
+    @staticmethod
+    def engine():
+        return AsyncDatabaseService._engine
+
+
+class AsyncDatabaseConnector(metaclass=SingletonMeta):
+    def __init__(self, db_config: DatabaseConfig):
+        # 从 DatabaseConfig 中提取数据库连接信息
+        self.db_user = db_config.username
+        self.db_password = db_config.password
+
+        # 提取 URL 中的主机、端口和数据库名
+        url_parts = db_config.url.split('//')[1].split('/')
+        host_port = url_parts[0].split(':')
+        self.db_host = host_port[0]
+        self.db_port = host_port[1]
+        self.db_name = url_parts[1].split('?')[0]
+
+        # 提取 URL 中的参数
+        params_str = url_parts[1].split('?')[1] if len(
+            url_parts[1].split('?')) > 1 else ''
+        params = {}
+        for param in params_str.split('&'):
+            if param:
+                key, value = param.split('=')
+                params[key] = value
+
+        # 在params中去掉指定的参数
+        for key in ['useUnicode', 'characterEncoding', 'serverTimezone', 'zeroDateTimeBehavior']:
+            if key in params:
+                del params[key]
+
+        # 构建数据库连接 URL
+        # 注意：这里将 mysqlconnector 替换为 aiomysql 以支持异步
+        self.db_url = f'mysql+aiomysql://{self.db_user}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}'
+
+        SYLogger.info(f"Database URL: {self.db_url}")
+
+        # 优化连接池配置
+        # 使用 create_async_engine 替代 create_engine
+        self.engine = create_async_engine(
+            self.db_url,
+            connect_args=params,
+            pool_size=10,  # 连接池大小
+            max_overflow=20,  # 最大溢出连接数
+            pool_timeout=30,  # 连接超时时间（秒）
+            pool_recycle=3600,  # 连接回收时间（秒）
+            pool_pre_ping=True,  # 每次获取连接前检查连接是否有效
+            echo=False,  # 打印 SQL 语句
+        )
+
+        # 注册 SQL 日志拦截器 (注意：SQLTraceLogger 需要支持异步引擎，或者您可能需要调整日志逻辑)
+        # 假设 SQLTraceLogger.setup_sql_logging 能够处理 AsyncEngine
+        AsyncSQLTraceLogger.setup_sql_logging(self.engine)
+
+    async def test_connection(self):
+        try:
+            # 异步上下文管理器
+            async with self.engine.connect() as connection:
+                # 执行简单查询
+                await connection.execute(text("SELECT 1"))
+                return True
+        except Exception as e:
+            SYLogger.error(f"Database connection test failed: {e}")
+            return False

sycommon/logging/async_sql_logger.py

@@ -0,0 +1,65 @@
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import AsyncEngine
+from sycommon.logging.kafka_log import SYLogger
+import time
+from datetime import datetime
+from decimal import Decimal
+
+
+class AsyncSQLTraceLogger:
+    @staticmethod
+    def setup_sql_logging(engine):
+        """
+        为 SQLAlchemy 异步引擎注册事件监听器
+        注意：必须监听 engine.sync_engine，而不能直接监听 AsyncEngine
+        """
+        def serialize_params(params):
+            """处理特殊类型参数的序列化"""
+            if isinstance(params, (list, tuple)):
+                return [serialize_params(p) for p in params]
+            elif isinstance(params, dict):
+                return {k: serialize_params(v) for k, v in params.items()}
+            elif isinstance(params, datetime):
+                return params.isoformat()
+            elif isinstance(params, Decimal):
+                return float(params)
+            else:
+                return params
+
+        # ========== 核心修改 ==========
+        # 必须通过 engine.sync_engine 来获取底层的同步引擎进行监听
+        target = engine.sync_engine
+
+        @event.listens_for(target, "after_cursor_execute")
+        def after_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                # 从连接选项中获取开始时间
+                # conn 在这里是同步连接对象
+                start_time = conn.info.get('_start_time') or \
+                    conn._execution_options.get("_start_time", time.time())
+
+                execution_time = (time.time() - start_time) * 1000
+
+                sql_log = {
+                    "type": "SQL",
+                    "statement": statement,
+                    "parameters": serialize_params(parameters),
+                    "execution_time_ms": round(execution_time, 2),
+                }
+
+                # 注意：SYLogger.info 必须是线程安全的或非阻塞的，否则可能影响异步性能
+                SYLogger.info(f"SQL执行: {sql_log}")
+            except Exception as e:
+                SYLogger.error(f"SQL日志处理失败: {str(e)}")
+
+        @event.listens_for(target, "before_cursor_execute")
+        def before_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                # 记录开始时间到 execution_options
+                conn = conn.execution_options(_start_time=time.time())
+            except Exception as e:
+                SYLogger.error(f"SQL开始时间记录失败: {str(e)}")

sycommon/logging/kafka_log.py

@@ -15,7 +15,7 @@ from kafka import KafkaProducer
 from loguru import logger
 import loguru
 from sycommon.config.Config import Config, SingletonMeta
-from sycommon.middleware.context import current_trace_id
+from sycommon.middleware.context import current_trace_id, current_headers
 from sycommon.tools.snowflake import Snowflake
 
 # 配置Loguru的颜色方案
@@ -114,7 +114,7 @@ class KafkaLogger(metaclass=SingletonMeta):
                     trace_id = None
 
                 if not trace_id:
-                    trace_id = SYLogger.get_trace_id() or Snowflake.next_id()
+                    trace_id = SYLogger.get_trace_id() or Snowflake.id
 
                 # 获取线程/协程信息
                 thread_info = SYLogger._get_execution_context()
@@ -173,7 +173,7 @@ class KafkaLogger(metaclass=SingletonMeta):
                     "className": "",
                     "sqlCost": 0,
                     "size": len(str(message)),
-                    "uid": int(Snowflake.next_id())  # 独立新的id
+                    "uid": int(Snowflake.id)  # 独立新的id
                 }
 
                 # 智能队列管理
@@ -212,7 +212,7 @@ class KafkaLogger(metaclass=SingletonMeta):
             return
 
         # 获取当前的trace_id
-        trace_id = SYLogger.get_trace_id() or Snowflake.next_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         # 构建错误日志
         error_log = {
@@ -441,6 +441,18 @@ class SYLogger:
         """重置当前的 trace_id"""
         current_trace_id.reset(token)
 
+    @staticmethod
+    def get_headers():
+        return current_headers.get()
+
+    @staticmethod
+    def set_headers(headers: list[tuple[str, str]]):
+        return current_headers.set(headers)
+
+    @staticmethod
+    def reset_headers(token):
+        current_headers.reset(token)
+
     @staticmethod
     def _get_execution_context() -> str:
         """获取当前执行上下文的线程或协程信息，返回格式化字符串"""
@@ -459,7 +471,7 @@ class SYLogger:
 
     @staticmethod
     def _log(msg: any, level: str = "INFO"):
-        trace_id = SYLogger.get_trace_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         if isinstance(msg, dict) or isinstance(msg, list):
             msg_str = json.dumps(msg, ensure_ascii=False)
@@ -473,7 +485,7 @@ class SYLogger:
         request_log = {}
         if level == "ERROR":
             request_log = {
-                "trace_id": str(trace_id) if trace_id else Snowflake.next_id(),
+                "trace_id": str(trace_id) if trace_id else Snowflake.id,
                 "message": msg_str,
                 "traceback": traceback.format_exc(),
                 "level": level,
@@ -481,7 +493,7 @@ class SYLogger:
             }
         else:
             request_log = {
-                "trace_id": str(trace_id) if trace_id else Snowflake.next_id(),
+                "trace_id": str(trace_id) if trace_id else Snowflake.id,
                 "message": msg_str,
                 "level": level,
                 "threadName": thread_info
@@ -521,7 +533,7 @@ class SYLogger:
     @staticmethod
     def exception(msg: any, *args, **kwargs):
         """记录异常信息，包括完整堆栈"""
-        trace_id = SYLogger.get_trace_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         if isinstance(msg, dict) or isinstance(msg, list):
             msg_str = json.dumps(msg, ensure_ascii=False)
@@ -533,7 +545,7 @@ class SYLogger:
 
         # 构建包含异常堆栈的日志
         request_log = {
-            "trace_id": str(trace_id) if trace_id else Snowflake.next_id(),
+            "trace_id": str(trace_id) if trace_id else Snowflake.id,
             "message": msg_str,
             "level": "ERROR",
             "threadName": thread_info

sycommon/logging/logger_levels.py

@@ -0,0 +1,23 @@
+import logging
+
+
+def setup_logger_levels():
+    """配置各模块的日志级别，抑制无关INFO/DEBUG日志"""
+    # Nacos 客户端：仅输出WARNING及以上（屏蔽INFO级的心跳/注册日志）
+    logging.getLogger("nacos.client").setLevel(logging.WARNING)
+
+    # Kafka Python客户端：屏蔽INFO级的连接/版本检测日志
+    logging.getLogger("kafka.conn").setLevel(logging.WARNING)
+    logging.getLogger("kafka.producer").setLevel(logging.WARNING)
+
+    # Uvicorn/FastAPI：屏蔽启动/应用初始化的INFO日志（保留ERROR/WARNING）
+    # logging.getLogger("uvicorn").setLevel(logging.WARNING)
+    # logging.getLogger("uvicorn.access").setLevel(logging.WARNING)  # 屏蔽访问日志
+    # logging.getLogger("uvicorn.error").setLevel(logging.ERROR)     # 仅保留错误
+
+    # 自定义的root日志（如同步数据库/监听器初始化）：屏蔽INFO
+    logging.getLogger("root").setLevel(logging.WARNING)
+
+    # RabbitMQ相关日志（如果有专属日志器）
+    logging.getLogger("pika").setLevel(logging.WARNING)  # 若使用pika客户端
+    logging.getLogger("rabbitmq").setLevel(logging.WARNING)

sycommon/middleware/context.py

@@ -1,3 +1,5 @@
 import contextvars
 
 current_trace_id = contextvars.ContextVar("trace_id", default=None)
+
+current_headers = contextvars.ContextVar("headers", default=None)

sycommon/middleware/traceid.py

@@ -3,33 +3,53 @@ import re
 from typing import Dict, Any
 from fastapi import Request, Response
 from sycommon.logging.kafka_log import SYLogger
+from sycommon.tools.merge_headers import merge_headers
 from sycommon.tools.snowflake import Snowflake
 
 
 def setup_trace_id_handler(app):
     @app.middleware("http")
     async def trace_id_and_log_middleware(request: Request, call_next):
-        # 生成或获取 traceId
-        trace_id = request.headers.get("x-traceId-header")
+        # ========== 1. 请求阶段：确保获取/生成 x-traceId-header ==========
+        # 优先从请求头读取（兼容任意大小写）
+        trace_id = request.headers.get(
+            "x-traceId-header") or request.headers.get("x-traceid-header")
+        # 无则生成雪花ID
         if not trace_id:
-            trace_id = Snowflake.next_id()
+            trace_id = Snowflake.id
 
-        # 设置 trace_id 上下文
+        # 设置 trace_id 到日志上下文
         token = SYLogger.set_trace_id(trace_id)
+        header_token = SYLogger.set_headers(request.headers.raw)
 
         # 获取请求参数
         query_params = dict(request.query_params)
         request_body: Dict[str, Any] = {}
         files_info: Dict[str, str] = {}
 
-        # 检测请求内容类型
+        json_content_types = [
+            "application/json",
+            "text/plain;charset=utf-8",
+            "text/plain"
+        ]
         content_type = request.headers.get("content-type", "").lower()
+        is_json_content = any(ct in content_type for ct in json_content_types)
 
-        if "application/json" in content_type and request.method in ["POST", "PUT", "PATCH"]:
+        if is_json_content and request.method in ["POST", "PUT", "PATCH"]:
             try:
-                request_body = await request.json()
+                # 兼容纯文本格式的 JSON（先读文本再解析）
+                if "text/plain" in content_type:
+                    raw_text = await request.text(encoding="utf-8")
+                    request_body = json.loads(raw_text)
+                else:
+                    # application/json 直接解析
+                    request_body = await request.json()
             except Exception as e:
-                request_body = {"error": f"Failed to parse JSON: {str(e)}"}
+                try:
+                    request_body = await request.json()
+                except Exception as e:
+                    # 精准捕获 JSON 解析错误（而非泛 Exception）
+                    request_body = {"error": f"JSON parse failed: {str(e)}"}
 
         elif "multipart/form-data" in content_type and request.method in ["POST", "PUT"]:
             try:
@@ -62,8 +82,9 @@ def setup_trace_id_handler(app):
                 request_body = {
                     "error": f"Failed to process form data: {str(e)}"}
 
-        # 构建请求日志信息
+        # 构建请求日志（包含 traceId）
         request_message = {
+            "traceId": trace_id,  # 请求日志中加入 traceId
             "method": request.method,
             "url": str(request.url),
             "query_params": query_params,
@@ -77,68 +98,159 @@ def setup_trace_id_handler(app):
             # 处理请求
             response = await call_next(request)
 
-            content_type = response.headers.get("Content-Type", "")
+            # 获取响应Content-Type（统一小写）
+            content_type = response.headers.get("content-type", "").lower()
 
-            # 处理 SSE 响应
+            # ========== 2. SSE 响应：仅设置 x-traceId-header，不修改其他头 ==========
             if "text/event-stream" in content_type:
-                # 流式响应不能有Content-Length，移除它
-                if "Content-Length" in response.headers:
-                    del response.headers["Content-Length"]
-                response.headers["x-traceId-header"] = trace_id
+                try:
+                    # 强制写入 x-traceId-header 到响应头
+                    response.headers["x-traceId-header"] = trace_id
+                    # 确保前端能读取（仅补充暴露头，不覆盖原有值）
+                    expose_headers = response.headers.get(
+                        "access-control-expose-headers", "")
+                    if expose_headers:
+                        if "x-traceId-header" not in expose_headers.lower():
+                            response.headers[
+                                "access-control-expose-headers"] = f"{expose_headers}, x-traceId-header"
+                    else:
+                        response.headers["access-control-expose-headers"] = "x-traceId-header"
+                    # SSE 必须移除 Content-Length（仅这一个额外操作）
+                    headers_lower = {
+                        k.lower(): k for k in response.headers.keys()}
+                    if "content-length" in headers_lower:
+                        del response.headers[headers_lower["content-length"]]
+                except AttributeError:
+                    # 流式响应头只读：初始化时仅加入 traceId 和必要暴露头
+                    new_headers = dict(response.headers) if hasattr(
+                        response.headers, 'items') else {}
+                    new_headers["x-traceId-header"] = trace_id  # 强制加入
+                    # 保留原有暴露头，补充 traceId
+                    if "access-control-expose-headers" in new_headers:
+                        if "x-traceId-header" not in new_headers["access-control-expose-headers"].lower():
+                            new_headers["access-control-expose-headers"] += ", x-traceId-header"
+                    else:
+                        new_headers["access-control-expose-headers"] = "x-traceId-header"
+                    # 移除 Content-Length
+                    new_headers.pop("content-length", None)
+                    response.init_headers(new_headers)
                 return response
 
-            # 处理普通响应
+            # ========== 3. 非 SSE 响应：强制写入 x-traceId-header，保留 CORS ==========
+            # 备份 CORS 头（防止丢失）
+            cors_headers = {}
+            cors_header_keys = [
+                "access-control-allow-origin",
+                "access-control-allow-methods",
+                "access-control-allow-headers",
+                "access-control-expose-headers",
+                "access-control-allow-credentials",
+                "access-control-max-age"
+            ]
+            for key in cors_header_keys:
+                for k in response.headers.keys():
+                    if k.lower() == key:
+                        cors_headers[key] = response.headers[k]
+                        break
+
+            # 合并 headers（非 SSE 场景）
+            merged_headers = merge_headers(
+                source_headers=request.headers,
+                target_headers=response.headers,
+                keep_keys=None,
+                delete_keys={'content-length', 'accept', 'content-type'}
+            )
+
+            # 强制加入 x-traceId-header（优先级最高）
+            merged_headers["x-traceId-header"] = trace_id
+            # 恢复 CORS 头 + 补充 traceId 到暴露头
+            merged_headers.update(cors_headers)
+            expose_headers = merged_headers.get(
+                "access-control-expose-headers", "")
+            if expose_headers:
+                if "x-traceId-header" not in expose_headers.lower():
+                    merged_headers["access-control-expose-headers"] = f"{expose_headers}, x-traceId-header"
+            else:
+                merged_headers["access-control-expose-headers"] = "x-traceId-header"
+
+            # 更新响应头
+            if hasattr(response.headers, 'clear'):
+                response.headers.clear()
+                for k, v in merged_headers.items():
+                    response.headers[k] = v
+            elif hasattr(response, "init_headers"):
+                response.init_headers(merged_headers)
+            else:
+                for k, v in merged_headers.items():
+                    try:
+                        response.headers[k] = v
+                    except (AttributeError, KeyError):
+                        pass
+
+            # 处理普通响应体（JSON 加入 traceId）
             response_body = b""
             try:
-                # 收集所有响应块
                 async for chunk in response.body_iterator:
                     response_body += chunk
 
+                # 获取 Content-Disposition（统一小写）
                 content_disposition = response.headers.get(
-                    "Content-Disposition", "")
+                    "content-disposition", "").lower()
 
-                # 判断是否能添加 trace_id
+                # JSON 响应体加入 traceId
                 if "application/json" in content_type and not content_disposition.startswith("attachment"):
                     try:
                         data = json.loads(response_body)
-                        data["traceId"] = trace_id
-                        new_body = json.dumps(
-                            data, ensure_ascii=False).encode()
+                        new_body = response_body
+                        if data:
+                            data["traceId"] = trace_id  # 响应体也加入
+                            new_body = json.dumps(
+                                data, ensure_ascii=False).encode()
 
-                        # 创建新响应，确保Content-Length正确
+                        # 重建响应，确保 header 包含 x-traceId-header
                         response = Response(
                             content=new_body,
                             status_code=response.status_code,
                             headers=dict(response.headers),
                             media_type=response.media_type
                         )
-                        # 显式设置正确的Content-Length
-                        response.headers["Content-Length"] = str(len(new_body))
+                        response.headers["content-length"] = str(len(new_body))
+                        response.headers["x-traceId-header"] = trace_id  # 再次兜底
+                        # 恢复 CORS 头
+                        for k, v in cors_headers.items():
+                            response.headers[k] = v
                     except json.JSONDecodeError:
-                        # 如果不是JSON，恢复原始响应体并更新长度
+                        # 非 JSON 响应：仅更新长度，强制加入 traceId
                         response = Response(
                             content=response_body,
                             status_code=response.status_code,
                             headers=dict(response.headers),
                             media_type=response.media_type
                         )
-                        response.headers["Content-Length"] = str(
+                        response.headers["content-length"] = str(
                             len(response_body))
+                        response.headers["x-traceId-header"] = trace_id  # 强制加入
+                        for k, v in cors_headers.items():
+                            response.headers[k] = v
                 else:
-                    # 非JSON响应，恢复原始响应体
+                    # 非 JSON 响应：强制加入 traceId
                     response = Response(
                         content=response_body,
                         status_code=response.status_code,
                         headers=dict(response.headers),
                         media_type=response.media_type
                     )
-                    response.headers["Content-Length"] = str(
+                    response.headers["content-length"] = str(
                         len(response_body))
+                    response.headers["x-traceId-header"] = trace_id  # 强制加入
+                    for k, v in cors_headers.items():
+                        response.headers[k] = v
             except StopAsyncIteration:
                 pass
 
-            # 构建响应日志信息
+            # 构建响应日志（包含 traceId）
             response_message = {
+                "traceId": trace_id,  # 响应日志加入 traceId
                 "status_code": response.status_code,
                 "response_body": response_body.decode('utf-8', errors='ignore'),
             }
@@ -146,11 +258,21 @@ def setup_trace_id_handler(app):
                 response_message, ensure_ascii=False)
             SYLogger.info(response_message_str)
 
-            response.headers["x-traceId-header"] = trace_id
+            # ========== 最终兜底：确保响应头必有 x-traceId-header ==========
+            try:
+                response.headers["x-traceId-header"] = trace_id
+            except AttributeError:
+                new_headers = dict(response.headers) if hasattr(
+                    response.headers, 'items') else {}
+                new_headers["x-traceId-header"] = trace_id
+                if hasattr(response, "init_headers"):
+                    response.init_headers(new_headers)
 
             return response
         except Exception as e:
+            # 异常日志也加入 traceId
             error_message = {
+                "traceId": trace_id,
                 "error": str(e),
                 "query_params": query_params,
                 "request_body": request_body,
@@ -160,7 +282,8 @@ def setup_trace_id_handler(app):
             SYLogger.error(error_message_str)
             raise
         finally:
-            # 清理上下文变量，防止泄漏
+            # 清理上下文变量
             SYLogger.reset_trace_id(token)
+            SYLogger.reset_headers(header_token)
 
     return app