sycommon-python-lib 0.1.16__py3-none-any.whl → 0.1.56b1__py3-none-any.whl

sycommon/logging/async_sql_logger.py

@@ -0,0 +1,65 @@
+from sqlalchemy import event
+from sqlalchemy.ext.asyncio import AsyncEngine
+from sycommon.logging.kafka_log import SYLogger
+import time
+from datetime import datetime
+from decimal import Decimal
+
+
+class AsyncSQLTraceLogger:
+    @staticmethod
+    def setup_sql_logging(engine):
+        """
+        为 SQLAlchemy 异步引擎注册事件监听器
+        注意：必须监听 engine.sync_engine，而不能直接监听 AsyncEngine
+        """
+        def serialize_params(params):
+            """处理特殊类型参数的序列化"""
+            if isinstance(params, (list, tuple)):
+                return [serialize_params(p) for p in params]
+            elif isinstance(params, dict):
+                return {k: serialize_params(v) for k, v in params.items()}
+            elif isinstance(params, datetime):
+                return params.isoformat()
+            elif isinstance(params, Decimal):
+                return float(params)
+            else:
+                return params
+
+        # ========== 核心修改 ==========
+        # 必须通过 engine.sync_engine 来获取底层的同步引擎进行监听
+        target = engine.sync_engine
+
+        @event.listens_for(target, "after_cursor_execute")
+        def after_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                # 从连接选项中获取开始时间
+                # conn 在这里是同步连接对象
+                start_time = conn.info.get('_start_time') or \
+                    conn._execution_options.get("_start_time", time.time())
+
+                execution_time = (time.time() - start_time) * 1000
+
+                sql_log = {
+                    "type": "SQL",
+                    "statement": statement,
+                    "parameters": serialize_params(parameters),
+                    "execution_time_ms": round(execution_time, 2),
+                }
+
+                # 注意：SYLogger.info 必须是线程安全的或非阻塞的，否则可能影响异步性能
+                SYLogger.info(f"SQL执行: {sql_log}")
+            except Exception as e:
+                SYLogger.error(f"SQL日志处理失败: {str(e)}")
+
+        @event.listens_for(target, "before_cursor_execute")
+        def before_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                # 记录开始时间到 execution_options
+                conn = conn.execution_options(_start_time=time.time())
+            except Exception as e:
+                SYLogger.error(f"SQL开始时间记录失败: {str(e)}")

sycommon/logging/kafka_log.py

@@ -15,7 +15,7 @@ from kafka import KafkaProducer
 from loguru import logger
 import loguru
 from sycommon.config.Config import Config, SingletonMeta
-from sycommon.middleware.context import current_trace_id
+from sycommon.middleware.context import current_trace_id, current_headers
 from sycommon.tools.snowflake import Snowflake
 
 # 配置Loguru的颜色方案
@@ -114,7 +114,7 @@ class KafkaLogger(metaclass=SingletonMeta):
                     trace_id = None
 
                 if not trace_id:
-                    trace_id = SYLogger.get_trace_id() or Snowflake.next_id()
+                    trace_id = SYLogger.get_trace_id() or Snowflake.id
 
                 # 获取线程/协程信息
                 thread_info = SYLogger._get_execution_context()
@@ -173,7 +173,7 @@ class KafkaLogger(metaclass=SingletonMeta):
                     "className": "",
                     "sqlCost": 0,
                     "size": len(str(message)),
-                    "uid": int(trace_id) if trace_id and trace_id.isdigit() else 0
+                    "uid": int(Snowflake.id)  # 独立新的id
                 }
 
                 # 智能队列管理
@@ -212,7 +212,7 @@ class KafkaLogger(metaclass=SingletonMeta):
             return
 
         # 获取当前的trace_id
-        trace_id = SYLogger.get_trace_id() or Snowflake.next_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         # 构建错误日志
         error_log = {
@@ -441,6 +441,18 @@ class SYLogger:
         """重置当前的 trace_id"""
         current_trace_id.reset(token)
 
+    @staticmethod
+    def get_headers():
+        return current_headers.get()
+
+    @staticmethod
+    def set_headers(headers: list[tuple[str, str]]):
+        return current_headers.set(headers)
+
+    @staticmethod
+    def reset_headers(token):
+        current_headers.reset(token)
+
     @staticmethod
     def _get_execution_context() -> str:
         """获取当前执行上下文的线程或协程信息，返回格式化字符串"""
@@ -459,7 +471,7 @@ class SYLogger:
 
     @staticmethod
     def _log(msg: any, level: str = "INFO"):
-        trace_id = SYLogger.get_trace_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         if isinstance(msg, dict) or isinstance(msg, list):
             msg_str = json.dumps(msg, ensure_ascii=False)
@@ -470,12 +482,22 @@ class SYLogger:
         thread_info = SYLogger._get_execution_context()
 
         # 构建日志结构，添加线程/协程信息到threadName字段
-        request_log = {
-            "trace_id": str(trace_id) if trace_id else Snowflake.next_id(),
-            "message": msg_str,
-            "level": level,
-            "threadName": thread_info
-        }
+        request_log = {}
+        if level == "ERROR":
+            request_log = {
+                "trace_id": str(trace_id) if trace_id else Snowflake.id,
+                "message": msg_str,
+                "traceback": traceback.format_exc(),
+                "level": level,
+                "threadName": thread_info
+            }
+        else:
+            request_log = {
+                "trace_id": str(trace_id) if trace_id else Snowflake.id,
+                "message": msg_str,
+                "level": level,
+                "threadName": thread_info
+            }
 
         # 选择日志级别
         _log = ''
@@ -490,7 +512,7 @@ class SYLogger:
             logger.info(_log)
 
         if os.getenv('DEV-LOG', 'false').lower() == 'true':
-            pprint(_log)
+            pprint.pprint(_log)
 
     @staticmethod
     def info(msg: any, *args, **kwargs):
@@ -511,7 +533,7 @@ class SYLogger:
     @staticmethod
     def exception(msg: any, *args, **kwargs):
         """记录异常信息，包括完整堆栈"""
-        trace_id = SYLogger.get_trace_id()
+        trace_id = SYLogger.get_trace_id() or Snowflake.id
 
         if isinstance(msg, dict) or isinstance(msg, list):
             msg_str = json.dumps(msg, ensure_ascii=False)
@@ -523,7 +545,7 @@ class SYLogger:
 
         # 构建包含异常堆栈的日志
         request_log = {
-            "trace_id": str(trace_id) if trace_id else Snowflake.next_id(),
+            "trace_id": str(trace_id) if trace_id else Snowflake.id,
             "message": msg_str,
             "level": "ERROR",
             "threadName": thread_info

sycommon/logging/logger_levels.py

@@ -0,0 +1,23 @@
+import logging
+
+
+def setup_logger_levels():
+    """配置各模块的日志级别，抑制无关INFO/DEBUG日志"""
+    # Nacos 客户端：仅输出WARNING及以上（屏蔽INFO级的心跳/注册日志）
+    logging.getLogger("nacos.client").setLevel(logging.WARNING)
+
+    # Kafka Python客户端：屏蔽INFO级的连接/版本检测日志
+    logging.getLogger("kafka.conn").setLevel(logging.WARNING)
+    logging.getLogger("kafka.producer").setLevel(logging.WARNING)
+
+    # Uvicorn/FastAPI：屏蔽启动/应用初始化的INFO日志（保留ERROR/WARNING）
+    # logging.getLogger("uvicorn").setLevel(logging.WARNING)
+    # logging.getLogger("uvicorn.access").setLevel(logging.WARNING)  # 屏蔽访问日志
+    # logging.getLogger("uvicorn.error").setLevel(logging.ERROR)     # 仅保留错误
+
+    # 自定义的root日志（如同步数据库/监听器初始化）：屏蔽INFO
+    logging.getLogger("root").setLevel(logging.WARNING)
+
+    # RabbitMQ相关日志（如果有专属日志器）
+    logging.getLogger("pika").setLevel(logging.WARNING)  # 若使用pika客户端
+    logging.getLogger("rabbitmq").setLevel(logging.WARNING)

sycommon/logging/sql_logger.py

@@ -0,0 +1,53 @@
+from sqlalchemy import event
+from sqlalchemy.engine import Engine
+from sycommon.logging.kafka_log import SYLogger
+import time
+from datetime import datetime
+from decimal import Decimal
+
+
+class SQLTraceLogger:
+    @staticmethod
+    def setup_sql_logging(engine: Engine):
+        """为 SQLAlchemy 引擎注册事件监听器"""
+        def serialize_params(params):
+            """处理特殊类型参数的序列化"""
+            if isinstance(params, (list, tuple)):
+                return [serialize_params(p) for p in params]
+            elif isinstance(params, dict):
+                return {k: serialize_params(v) for k, v in params.items()}
+            elif isinstance(params, datetime):
+                return params.isoformat()
+            elif isinstance(params, Decimal):
+                return float(params)
+            else:
+                return params
+
+        @event.listens_for(Engine, "after_cursor_execute")
+        def after_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                start_time = conn._execution_options.get(
+                    "_start_time", time.time())
+                execution_time = (time.time() - start_time) * 1000
+
+                sql_log = {
+                    "type": "SQL",
+                    "statement": statement,
+                    "parameters": serialize_params(parameters),
+                    "execution_time_ms": round(execution_time, 2),
+                }
+
+                SYLogger.info(f"SQL执行: {sql_log}")
+            except Exception as e:
+                SYLogger.error(f"SQL日志处理失败: {str(e)}")
+
+        @event.listens_for(Engine, "before_cursor_execute")
+        def before_cursor_execute(
+            conn, cursor, statement, parameters, context, executemany
+        ):
+            try:
+                conn = conn.execution_options(_start_time=time.time())
+            except Exception as e:
+                SYLogger.error(f"SQL开始时间记录失败: {str(e)}")

sycommon/middleware/context.py

@@ -1,3 +1,5 @@
 import contextvars
 
 current_trace_id = contextvars.ContextVar("trace_id", default=None)
+
+current_headers = contextvars.ContextVar("headers", default=None)

sycommon/middleware/middleware.py

@@ -1,3 +1,4 @@
+from sycommon.health.metrics import setup_metrics_handler
 from sycommon.health.ping import setup_ping_handler
 from sycommon.middleware.cors import setup_cors_handler
 from sycommon.middleware.docs import setup_docs_handler
@@ -34,6 +35,9 @@ class Middleware:
         # ping
         app = setup_ping_handler(app)
 
+        # metrics
+        app = setup_metrics_handler(app)
+
         # 添加mq中间件
         # app = setup_mq_middleware(app)
 

sycommon/middleware/traceid.py

@@ -3,33 +3,53 @@ import re
 from typing import Dict, Any
 from fastapi import Request, Response
 from sycommon.logging.kafka_log import SYLogger
+from sycommon.tools.merge_headers import merge_headers
 from sycommon.tools.snowflake import Snowflake
 
 
 def setup_trace_id_handler(app):
     @app.middleware("http")
     async def trace_id_and_log_middleware(request: Request, call_next):
-        # 生成或获取 traceId
-        trace_id = request.headers.get("x-traceId-header")
+        # ========== 1. 请求阶段：确保获取/生成 x-traceId-header ==========
+        # 优先从请求头读取（兼容任意大小写）
+        trace_id = request.headers.get(
+            "x-traceId-header") or request.headers.get("x-traceid-header")
+        # 无则生成雪花ID
         if not trace_id:
-            trace_id = Snowflake.next_id()
+            trace_id = Snowflake.id
 
-        # 设置 trace_id 上下文
+        # 设置 trace_id 到日志上下文
         token = SYLogger.set_trace_id(trace_id)
+        header_token = SYLogger.set_headers(request.headers.raw)
 
         # 获取请求参数
         query_params = dict(request.query_params)
         request_body: Dict[str, Any] = {}
         files_info: Dict[str, str] = {}
 
-        # 检测请求内容类型
+        json_content_types = [
+            "application/json",
+            "text/plain;charset=utf-8",
+            "text/plain"
+        ]
         content_type = request.headers.get("content-type", "").lower()
+        is_json_content = any(ct in content_type for ct in json_content_types)
 
-        if "application/json" in content_type and request.method in ["POST", "PUT", "PATCH"]:
+        if is_json_content and request.method in ["POST", "PUT", "PATCH"]:
             try:
-                request_body = await request.json()
+                # 兼容纯文本格式的 JSON（先读文本再解析）
+                if "text/plain" in content_type:
+                    raw_text = await request.text(encoding="utf-8")
+                    request_body = json.loads(raw_text)
+                else:
+                    # application/json 直接解析
+                    request_body = await request.json()
             except Exception as e:
-                request_body = {"error": f"Failed to parse JSON: {str(e)}"}
+                try:
+                    request_body = await request.json()
+                except Exception as e:
+                    # 精准捕获 JSON 解析错误（而非泛 Exception）
+                    request_body = {"error": f"JSON parse failed: {str(e)}"}
 
         elif "multipart/form-data" in content_type and request.method in ["POST", "PUT"]:
             try:
@@ -62,8 +82,9 @@ def setup_trace_id_handler(app):
                 request_body = {
                     "error": f"Failed to process form data: {str(e)}"}
 
-        # 构建请求日志信息
+        # 构建请求日志（包含 traceId）
         request_message = {
+            "traceId": trace_id,  # 请求日志中加入 traceId
             "method": request.method,
             "url": str(request.url),
             "query_params": query_params,
@@ -77,68 +98,159 @@ def setup_trace_id_handler(app):
             # 处理请求
             response = await call_next(request)
 
-            content_type = response.headers.get("Content-Type", "")
+            # 获取响应Content-Type（统一小写）
+            content_type = response.headers.get("content-type", "").lower()
 
-            # 处理 SSE 响应 - 关键修复点
+            # ========== 2. SSE 响应：仅设置 x-traceId-header，不修改其他头 ==========
             if "text/event-stream" in content_type:
-                # 流式响应不能有Content-Length，移除它
-                if "Content-Length" in response.headers:
-                    del response.headers["Content-Length"]
-                response.headers["x-traceId-header"] = trace_id
+                try:
+                    # 强制写入 x-traceId-header 到响应头
+                    response.headers["x-traceId-header"] = trace_id
+                    # 确保前端能读取（仅补充暴露头，不覆盖原有值）
+                    expose_headers = response.headers.get(
+                        "access-control-expose-headers", "")
+                    if expose_headers:
+                        if "x-traceId-header" not in expose_headers.lower():
+                            response.headers[
+                                "access-control-expose-headers"] = f"{expose_headers}, x-traceId-header"
+                    else:
+                        response.headers["access-control-expose-headers"] = "x-traceId-header"
+                    # SSE 必须移除 Content-Length（仅这一个额外操作）
+                    headers_lower = {
+                        k.lower(): k for k in response.headers.keys()}
+                    if "content-length" in headers_lower:
+                        del response.headers[headers_lower["content-length"]]
+                except AttributeError:
+                    # 流式响应头只读：初始化时仅加入 traceId 和必要暴露头
+                    new_headers = dict(response.headers) if hasattr(
+                        response.headers, 'items') else {}
+                    new_headers["x-traceId-header"] = trace_id  # 强制加入
+                    # 保留原有暴露头，补充 traceId
+                    if "access-control-expose-headers" in new_headers:
+                        if "x-traceId-header" not in new_headers["access-control-expose-headers"].lower():
+                            new_headers["access-control-expose-headers"] += ", x-traceId-header"
+                    else:
+                        new_headers["access-control-expose-headers"] = "x-traceId-header"
+                    # 移除 Content-Length
+                    new_headers.pop("content-length", None)
+                    response.init_headers(new_headers)
                 return response
 
-            # 处理普通响应
+            # ========== 3. 非 SSE 响应：强制写入 x-traceId-header，保留 CORS ==========
+            # 备份 CORS 头（防止丢失）
+            cors_headers = {}
+            cors_header_keys = [
+                "access-control-allow-origin",
+                "access-control-allow-methods",
+                "access-control-allow-headers",
+                "access-control-expose-headers",
+                "access-control-allow-credentials",
+                "access-control-max-age"
+            ]
+            for key in cors_header_keys:
+                for k in response.headers.keys():
+                    if k.lower() == key:
+                        cors_headers[key] = response.headers[k]
+                        break
+
+            # 合并 headers（非 SSE 场景）
+            merged_headers = merge_headers(
+                source_headers=request.headers,
+                target_headers=response.headers,
+                keep_keys=None,
+                delete_keys={'content-length', 'accept', 'content-type'}
+            )
+
+            # 强制加入 x-traceId-header（优先级最高）
+            merged_headers["x-traceId-header"] = trace_id
+            # 恢复 CORS 头 + 补充 traceId 到暴露头
+            merged_headers.update(cors_headers)
+            expose_headers = merged_headers.get(
+                "access-control-expose-headers", "")
+            if expose_headers:
+                if "x-traceId-header" not in expose_headers.lower():
+                    merged_headers["access-control-expose-headers"] = f"{expose_headers}, x-traceId-header"
+            else:
+                merged_headers["access-control-expose-headers"] = "x-traceId-header"
+
+            # 更新响应头
+            if hasattr(response.headers, 'clear'):
+                response.headers.clear()
+                for k, v in merged_headers.items():
+                    response.headers[k] = v
+            elif hasattr(response, "init_headers"):
+                response.init_headers(merged_headers)
+            else:
+                for k, v in merged_headers.items():
+                    try:
+                        response.headers[k] = v
+                    except (AttributeError, KeyError):
+                        pass
+
+            # 处理普通响应体（JSON 加入 traceId）
             response_body = b""
             try:
-                # 收集所有响应块
                 async for chunk in response.body_iterator:
                     response_body += chunk
 
+                # 获取 Content-Disposition（统一小写）
                 content_disposition = response.headers.get(
-                    "Content-Disposition", "")
+                    "content-disposition", "").lower()
 
-                # 判断是否能添加 trace_id
+                # JSON 响应体加入 traceId
                 if "application/json" in content_type and not content_disposition.startswith("attachment"):
                     try:
                         data = json.loads(response_body)
-                        data["traceId"] = trace_id
-                        new_body = json.dumps(
-                            data, ensure_ascii=False).encode()
+                        new_body = response_body
+                        if data:
+                            data["traceId"] = trace_id  # 响应体也加入
+                            new_body = json.dumps(
+                                data, ensure_ascii=False).encode()
 
-                        # 创建新响应，确保Content-Length正确
+                        # 重建响应，确保 header 包含 x-traceId-header
                         response = Response(
                             content=new_body,
                             status_code=response.status_code,
                             headers=dict(response.headers),
                             media_type=response.media_type
                         )
-                        # 显式设置正确的Content-Length
-                        response.headers["Content-Length"] = str(len(new_body))
+                        response.headers["content-length"] = str(len(new_body))
+                        response.headers["x-traceId-header"] = trace_id  # 再次兜底
+                        # 恢复 CORS 头
+                        for k, v in cors_headers.items():
+                            response.headers[k] = v
                     except json.JSONDecodeError:
-                        # 如果不是JSON，恢复原始响应体并更新长度
+                        # 非 JSON 响应：仅更新长度，强制加入 traceId
                         response = Response(
                             content=response_body,
                             status_code=response.status_code,
                             headers=dict(response.headers),
                             media_type=response.media_type
                         )
-                        response.headers["Content-Length"] = str(
+                        response.headers["content-length"] = str(
                             len(response_body))
+                        response.headers["x-traceId-header"] = trace_id  # 强制加入
+                        for k, v in cors_headers.items():
+                            response.headers[k] = v
                 else:
-                    # 非JSON响应，恢复原始响应体
+                    # 非 JSON 响应：强制加入 traceId
                     response = Response(
                         content=response_body,
                         status_code=response.status_code,
                         headers=dict(response.headers),
                         media_type=response.media_type
                     )
-                    response.headers["Content-Length"] = str(
+                    response.headers["content-length"] = str(
                         len(response_body))
+                    response.headers["x-traceId-header"] = trace_id  # 强制加入
+                    for k, v in cors_headers.items():
+                        response.headers[k] = v
             except StopAsyncIteration:
                 pass
 
-            # 构建响应日志信息
+            # 构建响应日志（包含 traceId）
             response_message = {
+                "traceId": trace_id,  # 响应日志加入 traceId
                 "status_code": response.status_code,
                 "response_body": response_body.decode('utf-8', errors='ignore'),
             }
@@ -146,11 +258,21 @@ def setup_trace_id_handler(app):
                 response_message, ensure_ascii=False)
             SYLogger.info(response_message_str)
 
-            response.headers["x-traceId-header"] = trace_id
+            # ========== 最终兜底：确保响应头必有 x-traceId-header ==========
+            try:
+                response.headers["x-traceId-header"] = trace_id
+            except AttributeError:
+                new_headers = dict(response.headers) if hasattr(
+                    response.headers, 'items') else {}
+                new_headers["x-traceId-header"] = trace_id
+                if hasattr(response, "init_headers"):
+                    response.init_headers(new_headers)
 
             return response
         except Exception as e:
+            # 异常日志也加入 traceId
             error_message = {
+                "traceId": trace_id,
                 "error": str(e),
                 "query_params": query_params,
                 "request_body": request_body,
@@ -160,7 +282,8 @@ def setup_trace_id_handler(app):
             SYLogger.error(error_message_str)
             raise
         finally:
-            # 清理上下文变量，防止泄漏
+            # 清理上下文变量
             SYLogger.reset_trace_id(token)
+            SYLogger.reset_headers(header_token)
 
     return app

sycommon/models/mqlistener_config.py

@@ -31,6 +31,7 @@ class RabbitMQListenerConfig(BaseModel):
     durable: bool = Field(True, description="是否持久化")
     auto_delete: bool = Field(False, description="是否自动删除队列")
     auto_parse_json: bool = Field(True, description="是否自动解析JSON消息")
+    prefetch_count: int = Field(2, description="mq同时消费数量")
 
     class Config:
         """模型配置"""