sycommon-python-lib 0.2.5a25__tar.gz → 0.2.5a26__tar.gz

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a25
+Version: 0.2.5a26
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.5a25"
+version = "0.2.5a26"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

sycommon_python_lib-0.2.5a26/src/sycommon/database/async_base_db_service.py

@@ -0,0 +1,114 @@
+import asyncio
+from contextlib import asynccontextmanager
+from sqlalchemy.exc import OperationalError, InterfaceError, InternalError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.async_database_service import AsyncDatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+# 连接级错误：连接已断，事务必定已丢失，重试安全
+_CONNECTION_ERROR_CODES = {2006, 2013}
+# 事务级错误：死锁、锁等待超时，由调用方决定是否重试
+_RETRYABLE_TX_ERROR_CODES = {1205, 1213, 1206}
+
+
+def _is_connection_error(exc: Exception) -> bool:
+    """判断是否为连接级错误（断开、宕机），可安全重试"""
+    if isinstance(exc, (OperationalError, InterfaceError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _CONNECTION_ERROR_CODES
+    return False
+
+
+def _is_retryable_tx_error(exc: Exception) -> bool:
+    """判断是否为可重试的事务级错误（死锁、锁超时）"""
+    if isinstance(exc, (OperationalError, InternalError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _RETRYABLE_TX_ERROR_CODES
+    return False
+
+
+class AsyncBaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装异步会话管理功能"""
+
+    def __init__(self):
+        self.engine = AsyncDatabaseService.engine()
+        self.Session = async_sessionmaker(
+            bind=self.engine,
+            class_=AsyncSession,
+            expire_on_commit=False
+        )
+
+    @asynccontextmanager
+    async def session(self):
+        """
+        异步数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        async with self.Session() as session:
+            try:
+                yield session
+                await session.commit()
+            except Exception as e:
+                await session.rollback()
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise
+
+    async def execute(self, fn, retry: int = 0):
+        """
+        执行异步数据库操作，支持自动重试
+
+        连接级错误（连接断开 2006/2013）始终自动重试1次。
+        事务级错误（死锁 1213、锁超时 1205）按 retry 参数重试。
+
+        Args:
+            fn: 接收 session 参数的异步可调用对象，例如:
+                async def query(session): return (await session.execute(select(User))).scalars().all()
+            retry: 事务级错误最大重试次数，默认0不重试
+
+        Returns:
+            fn 的返回值
+
+        Usage:
+            result = await self.execute(
+                lambda session: session.execute(select(User)),
+                retry=3
+            )
+        """
+        attempt = 0
+        max_attempts = 1 + max(retry, 0)
+        connection_retried = False
+        last_exc = None
+
+        while True:
+            async with self.Session() as session:
+                try:
+                    result = await fn(session)
+                    await session.commit()
+                    return result
+                except Exception as e:
+                    await session.rollback()
+                    last_exc = e
+
+                    # 连接级错误：始终自动重试1次（不计入 max_attempts）
+                    if _is_connection_error(e) and not connection_retried:
+                        connection_retried = True
+                        SYLogger.warning(
+                            f"DB connection lost (errno={getattr(e.orig, 'errno', None)}), retrying..."
+                        )
+                        attempt += 1
+                        continue
+
+                    # 事务级错误：按 retry 参数重试
+                    if retry > 0 and _is_retryable_tx_error(e) and attempt < max_attempts - 1:
+                        attempt += 1
+                        backoff = min(0.5 * (2 ** (attempt - 1)), 2.0)
+                        SYLogger.warning(
+                            f"DB retryable error (errno={getattr(e.orig, 'errno', None)}), "
+                            f"attempt {attempt}/{max_attempts}, backoff {backoff}s"
+                        )
+                        await asyncio.sleep(backoff)
+                        continue
+
+                    SYLogger.error(f"Database operation failed: {str(e)}")
+                    raise

sycommon_python_lib-0.2.5a26/src/sycommon/database/base_db_service.py

@@ -0,0 +1,114 @@
+import time
+from contextlib import contextmanager
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import OperationalError, InterfaceError, InternalError
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.database_service import DatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+# 连接级错误：连接已断，事务必定已丢失，重试安全
+_CONNECTION_ERROR_CODES = {2006, 2013}
+# 事务级错误：死锁、锁等待超时，由调用方决定是否重试
+_RETRYABLE_TX_ERROR_CODES = {1205, 1213, 1206}
+
+
+def _is_connection_error(exc: Exception) -> bool:
+    """判断是否为连接级错误（断开、宕机），可安全重试"""
+    if isinstance(exc, (OperationalError, InterfaceError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _CONNECTION_ERROR_CODES
+    return False
+
+
+def _is_retryable_tx_error(exc: Exception) -> bool:
+    """判断是否为可重试的事务级错误（死锁、锁超时）"""
+    if isinstance(exc, (OperationalError, InternalError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _RETRYABLE_TX_ERROR_CODES
+    return False
+
+
+class BaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装会话管理功能"""
+
+    def __init__(self):
+        self.engine = DatabaseService.engine()
+        self.Session = sessionmaker(bind=self.engine)
+
+    @contextmanager
+    def session(self):
+        """
+        数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        session = self.Session()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            SYLogger.error(f"Database operation failed: {str(e)}")
+            raise
+        finally:
+            session.close()
+
+    def execute(self, fn, retry: int = 0):
+        """
+        执行数据库操作，支持自动重试
+
+        连接级错误（连接断开 2006/2013）始终自动重试1次。
+        事务级错误（死锁 1213、锁超时 1205）按 retry 参数重试。
+
+        Args:
+            fn: 接收 session 参数的可调用对象，例如:
+                lambda session: session.query(User).all()
+            retry: 事务级错误最大重试次数，默认0不重试
+
+        Returns:
+            fn 的返回值
+
+        Usage:
+            result = self.execute(
+                lambda session: session.query(User).all(),
+                retry=3
+            )
+        """
+        attempt = 0
+        max_attempts = 1 + max(retry, 0)
+        connection_retried = False
+        last_exc = None
+
+        while True:
+            session = self.Session()
+            try:
+                result = fn(session)
+                session.commit()
+                return result
+            except Exception as e:
+                session.rollback()
+                last_exc = e
+
+                # 连接级错误：始终自动重试1次（不计入 max_attempts）
+                if _is_connection_error(e) and not connection_retried:
+                    connection_retried = True
+                    SYLogger.warning(
+                        f"DB connection lost (errno={getattr(e.orig, 'errno', None)}), retrying..."
+                    )
+                    attempt += 1
+                    continue
+
+                # 事务级错误：按 retry 参数重试
+                if retry > 0 and _is_retryable_tx_error(e) and attempt < max_attempts - 1:
+                    attempt += 1
+                    backoff = min(0.5 * (2 ** (attempt - 1)), 2.0)
+                    SYLogger.warning(
+                        f"DB retryable error (errno={getattr(e.orig, 'errno', None)}), "
+                        f"attempt {attempt}/{max_attempts}, backoff {backoff}s"
+                    )
+                    time.sleep(backoff)
+                    continue
+
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise
+            finally:
+                session.close()

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon/logging/kafka_log.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import logging
 import socket
 import threading
 import traceback
@@ -176,6 +177,23 @@ class KafkaSink:
                 pass
 
 
+class _InterceptHandler(logging.Handler):
+    """将 stdlib logging 日志桥接到 Loguru，使 asyncio 等第三方库的日志进入 Kafka 管道"""
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        frame, depth = logging.currentframe(), 2
+        while frame and frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
 class KafkaLogger(metaclass=SingletonMeta):
     _sink_instance = None
 
@@ -206,8 +224,48 @@ class KafkaLogger(metaclass=SingletonMeta):
             diagnose=True
         )
 
+        # 桥接 stdlib logging → Loguru，捕获 asyncio 等第三方库的日志
+        intercept_handler = _InterceptHandler()
+        logging.root.handlers = [intercept_handler]
+        logging.root.setLevel(logging.WARNING)
+
         sys.excepthook = KafkaLogger._handle_exception
 
+        # 注册 asyncio 异常处理器，捕获未处理的 Task 异常
+        KafkaLogger._setup_asyncio_exception_handler()
+
+    @staticmethod
+    def _setup_asyncio_exception_handler():
+        """注册 asyncio 事件循环异常处理器，将未处理的 Task 异常导入 Kafka"""
+        try:
+            loop = asyncio.get_running_loop()
+            loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+        except RuntimeError:
+            pass
+
+    @staticmethod
+    def _asyncio_exception_handler(loop, context):
+        """asyncio 未处理异常回调，通过 SYLogger 记录到 Kafka"""
+        message = context.get("exception", context.get("message", "Unknown asyncio error"))
+        exception = context.get("exception")
+
+        if exception:
+            tb_str = "".join(traceback.format_exception(
+                type(exception), exception, exception.__traceback__
+            ))
+        else:
+            tb_str = str(message)
+
+        trace_id = current_trace_id.get() or str(Snowflake.id)
+        error_msg = json.dumps({
+            "trace_id": trace_id,
+            "message": f"asyncio unhandled: {message}",
+            "level": "ERROR",
+            "detail": tb_str,
+        }, ensure_ascii=False)
+
+        logger.opt(exception=True).error(error_msg)
+
     @staticmethod
     def _handle_exception(exc_type, exc_value, exc_traceback):
         if issubclass(exc_type, KeyboardInterrupt):

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon/models/mqlistener_config.py

@@ -33,7 +33,7 @@ class RabbitMQListenerConfig(BaseModel):
     auto_parse_json: bool = Field(True, description="是否自动解析JSON消息")
     prefetch_count: int = Field(2, description="同时消费并发数（控制MQ预取与handler并发上限）")
     max_priority: Optional[int] = Field(
-        None, description="队列支持的最大优先级（对应 x-max-priority）")
+        10, description="队列支持的最大优先级（对应 x-max-priority）")
 
     class Config:
         """模型配置"""

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon/rabbitmq/rabbitmq_client.py

@@ -42,7 +42,7 @@ class RabbitMQClient:
         auto_delete: bool = False,
         auto_parse_json: bool = True,
         create_if_not_exists: bool = True,
-        max_priority: Optional[int] = None,
+        max_priority: Optional[int] = 10,
         **kwargs,
     ):
         self.connection_pool = connection_pool

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon/services.py

@@ -265,6 +265,14 @@ class Services(metaclass=SingletonMeta):
             sandbox_cleanup_task = None
 
             try:
+                # 注册 asyncio 异常处理器（需在事件循环运行后）
+                try:
+                    from sycommon.logging.kafka_log import KafkaLogger
+                    loop = asyncio.get_running_loop()
+                    loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+                except Exception as e:
+                    logging.debug(f"注册 asyncio 异常处理器失败: {e}")
+
                 # 1. 处理数据库服务
                 if database_service:
                     instance._pending_async_db_setup = []

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a25
+Version: 0.2.5a26
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.5a25 → sycommon_python_lib-0.2.5a26}/src/sycommon_python_lib.egg-info/SOURCES.txt

@@ -286,4 +286,5 @@ src/sycommon_python_lib.egg-info/dependency_links.txt
 src/sycommon_python_lib.egg-info/entry_points.txt
 src/sycommon_python_lib.egg-info/requires.txt
 src/sycommon_python_lib.egg-info/top_level.txt
+tests/test_asyncio_logging.py
 tests/test_sycli.py

sycommon_python_lib-0.2.5a26/tests/test_asyncio_logging.py

@@ -0,0 +1,116 @@
+"""
+验证 asyncio 错误能被 Kafka 日志管道捕获。
+
+覆盖两类场景：
+1. stdlib logging 输出的 asyncio 错误 (如 Unclosed client session)
+2. 未处理的 Task 异常 (Task exception was never retrieved)
+
+直接从已安装的包导入，绕过外部服务依赖。
+"""
+import asyncio
+import logging
+
+import pytest
+from loguru import logger
+
+
+@pytest.fixture(autouse=True)
+def _reset_loguru():
+    logger.remove()
+    yield
+    logger.remove()
+
+
+def _setup_loguru_spy():
+    captured = []
+    logger.add(lambda msg: captured.append(str(msg)), level="DEBUG", format="{message}")
+    return captured
+
+
+# 直接从已安装的 sycommon 包导入（需要 nacos 等服务可用）
+# 这些类不依赖外部连接，可以直接实例化/使用
+from sycommon.logging.kafka_log import _InterceptHandler, KafkaLogger
+
+
+def test_intercept_handler_bridges_stdlib_logging():
+    """_InterceptHandler 能将 stdlib logging 的 ERROR 桥接到 loguru"""
+    captured = _setup_loguru_spy()
+
+    handler = _InterceptHandler()
+    old_handlers = logging.root.handlers[:]
+    old_level = logging.root.level
+    logging.root.handlers = [handler]
+    logging.root.setLevel(logging.WARNING)
+
+    logging.getLogger("asyncio").error("Unclosed client session")
+
+    logging.root.handlers = old_handlers
+    logging.root.setLevel(old_level)
+
+    assert any("Unclosed client session" in msg for msg in captured), \
+        f"InterceptHandler 未捕获 asyncio stdlib 日志, captured: {captured}"
+
+
+def test_asyncio_exception_handler_captures_task_error():
+    """_asyncio_exception_handler 能捕获未处理的 Task 异常"""
+    captured = _setup_loguru_spy()
+
+    loop = asyncio.new_event_loop()
+    loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+
+    async def _fail():
+        raise RuntimeError("boom from task")
+
+    loop.create_task(_fail())
+    loop.run_until_complete(asyncio.sleep(0.1))
+    loop.close()
+
+    assert any("asyncio unhandled" in msg and "boom from task" in msg for msg in captured), \
+        f"asyncio exception handler 未捕获 Task 异常, captured: {captured}"
+
+
+def test_asyncio_exception_handler_captures_non_exception_context():
+    """_asyncio_exception_handler 处理无 exception 的 context"""
+    captured = _setup_loguru_spy()
+
+    loop = asyncio.new_event_loop()
+    loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+    KafkaLogger._asyncio_exception_handler(loop, {"message": "something went wrong"})
+    loop.close()
+
+    assert any("asyncio unhandled" in msg and "something went wrong" in msg for msg in captured), \
+        f"asyncio exception handler 未处理纯 message context, captured: {captured}"
+
+
+def test_full_pipeline_both_error_types():
+    """完整管道：stdlib 和 Task 两类 asyncio 错误都能进入 loguru"""
+    captured = _setup_loguru_spy()
+
+    handler = _InterceptHandler()
+    old_handlers = logging.root.handlers[:]
+    old_level = logging.root.level
+    logging.root.handlers = [handler]
+    logging.root.setLevel(logging.WARNING)
+
+    loop = asyncio.new_event_loop()
+    loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+
+    async def _scenario():
+        # 场景1: stdlib logging 输出的 asyncio 错误
+        logging.getLogger("asyncio").error("Unclosed connector")
+        # 场景2: 未处理的 Task 异常
+        async def _fail():
+            raise AttributeError("'NoneType' object has no attribute 'get'")
+        loop.create_task(_fail())
+        await asyncio.sleep(0.1)
+
+    loop.run_until_complete(_scenario())
+    loop.close()
+
+    logging.root.handlers = old_handlers
+    logging.root.setLevel(old_level)
+
+    assert any("Unclosed connector" in msg for msg in captured), \
+        f"stdlib asyncio 错误未进入 loguru, captured: {captured}"
+    assert any("asyncio unhandled" in msg and "NoneType" in msg for msg in captured), \
+        f"Task 异常未进入 loguru, captured: {captured}"

sycommon_python_lib-0.2.5a25/src/sycommon/database/async_base_db_service.py

@@ -1,36 +0,0 @@
-from contextlib import asynccontextmanager
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from sycommon.config.Config import SingletonMeta
-from sycommon.database.async_database_service import AsyncDatabaseService
-from sycommon.logging.kafka_log import SYLogger
-
-
-class AsyncBaseDBService(metaclass=SingletonMeta):
-    """数据库操作基础服务类，封装异步会话管理功能"""
-
-    def __init__(self):
-        # 获取异步引擎 (假设 DatabaseService.engine() 返回的是 AsyncEngine)
-        self.engine = AsyncDatabaseService.engine()
-
-        # 创建异步 Session 工厂
-        # class_=AsyncSession 是必须的，用于指定生成的是异步会话
-        self.Session = async_sessionmaker(
-            bind=self.engine,
-            class_=AsyncSession,
-            expire_on_commit=False
-        )
-
-    @asynccontextmanager
-    async def session(self):
-        """
-        异步数据库会话上下文管理器
-        自动处理会话的创建、提交、回滚和关闭
-        """
-        async with self.Session() as session:
-            try:
-                yield session
-                await session.commit()
-            except Exception as e:
-                await session.rollback()
-                SYLogger.error(f"Database operation failed: {str(e)}")
-                raise

sycommon_python_lib-0.2.5a25/src/sycommon/database/base_db_service.py

@@ -1,30 +0,0 @@
-from contextlib import contextmanager
-from sqlalchemy.orm import sessionmaker
-from sycommon.config.Config import SingletonMeta
-from sycommon.database.database_service import DatabaseService
-from sycommon.logging.kafka_log import SYLogger
-
-
-class BaseDBService(metaclass=SingletonMeta):
-    """数据库操作基础服务类，封装会话管理功能"""
-
-    def __init__(self):
-        self.engine = DatabaseService.engine()
-        self.Session = sessionmaker(bind=self.engine)
-
-    @contextmanager
-    def session(self):
-        """
-        数据库会话上下文管理器
-        自动处理会话的创建、提交、回滚和关闭
-        """
-        session = self.Session()
-        try:
-            yield session
-            session.commit()
-        except Exception as e:
-            session.rollback()
-            SYLogger.error(f"Database operation failed: {str(e)}")
-            raise
-        finally:
-            session.close()