sycommon-python-lib 0.2.5a24__tar.gz → 0.2.5a26__tar.gz

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a24
+Version: 0.2.5a26
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.2.5a24"
+version = "0.2.5a26"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

sycommon_python_lib-0.2.5a26/src/sycommon/database/async_base_db_service.py

@@ -0,0 +1,114 @@
+import asyncio
+from contextlib import asynccontextmanager
+from sqlalchemy.exc import OperationalError, InterfaceError, InternalError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.async_database_service import AsyncDatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+# 连接级错误：连接已断，事务必定已丢失，重试安全
+_CONNECTION_ERROR_CODES = {2006, 2013}
+# 事务级错误：死锁、锁等待超时，由调用方决定是否重试
+_RETRYABLE_TX_ERROR_CODES = {1205, 1213, 1206}
+
+
+def _is_connection_error(exc: Exception) -> bool:
+    """判断是否为连接级错误（断开、宕机），可安全重试"""
+    if isinstance(exc, (OperationalError, InterfaceError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _CONNECTION_ERROR_CODES
+    return False
+
+
+def _is_retryable_tx_error(exc: Exception) -> bool:
+    """判断是否为可重试的事务级错误（死锁、锁超时）"""
+    if isinstance(exc, (OperationalError, InternalError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _RETRYABLE_TX_ERROR_CODES
+    return False
+
+
+class AsyncBaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装异步会话管理功能"""
+
+    def __init__(self):
+        self.engine = AsyncDatabaseService.engine()
+        self.Session = async_sessionmaker(
+            bind=self.engine,
+            class_=AsyncSession,
+            expire_on_commit=False
+        )
+
+    @asynccontextmanager
+    async def session(self):
+        """
+        异步数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        async with self.Session() as session:
+            try:
+                yield session
+                await session.commit()
+            except Exception as e:
+                await session.rollback()
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise
+
+    async def execute(self, fn, retry: int = 0):
+        """
+        执行异步数据库操作，支持自动重试
+
+        连接级错误（连接断开 2006/2013）始终自动重试1次。
+        事务级错误（死锁 1213、锁超时 1205）按 retry 参数重试。
+
+        Args:
+            fn: 接收 session 参数的异步可调用对象，例如:
+                async def query(session): return (await session.execute(select(User))).scalars().all()
+            retry: 事务级错误最大重试次数，默认0不重试
+
+        Returns:
+            fn 的返回值
+
+        Usage:
+            result = await self.execute(
+                lambda session: session.execute(select(User)),
+                retry=3
+            )
+        """
+        attempt = 0
+        max_attempts = 1 + max(retry, 0)
+        connection_retried = False
+        last_exc = None
+
+        while True:
+            async with self.Session() as session:
+                try:
+                    result = await fn(session)
+                    await session.commit()
+                    return result
+                except Exception as e:
+                    await session.rollback()
+                    last_exc = e
+
+                    # 连接级错误：始终自动重试1次（不计入 max_attempts）
+                    if _is_connection_error(e) and not connection_retried:
+                        connection_retried = True
+                        SYLogger.warning(
+                            f"DB connection lost (errno={getattr(e.orig, 'errno', None)}), retrying..."
+                        )
+                        attempt += 1
+                        continue
+
+                    # 事务级错误：按 retry 参数重试
+                    if retry > 0 and _is_retryable_tx_error(e) and attempt < max_attempts - 1:
+                        attempt += 1
+                        backoff = min(0.5 * (2 ** (attempt - 1)), 2.0)
+                        SYLogger.warning(
+                            f"DB retryable error (errno={getattr(e.orig, 'errno', None)}), "
+                            f"attempt {attempt}/{max_attempts}, backoff {backoff}s"
+                        )
+                        await asyncio.sleep(backoff)
+                        continue
+
+                    SYLogger.error(f"Database operation failed: {str(e)}")
+                    raise

sycommon_python_lib-0.2.5a26/src/sycommon/database/base_db_service.py

@@ -0,0 +1,114 @@
+import time
+from contextlib import contextmanager
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.exc import OperationalError, InterfaceError, InternalError
+from sycommon.config.Config import SingletonMeta
+from sycommon.database.database_service import DatabaseService
+from sycommon.logging.kafka_log import SYLogger
+
+# 连接级错误：连接已断，事务必定已丢失，重试安全
+_CONNECTION_ERROR_CODES = {2006, 2013}
+# 事务级错误：死锁、锁等待超时，由调用方决定是否重试
+_RETRYABLE_TX_ERROR_CODES = {1205, 1213, 1206}
+
+
+def _is_connection_error(exc: Exception) -> bool:
+    """判断是否为连接级错误（断开、宕机），可安全重试"""
+    if isinstance(exc, (OperationalError, InterfaceError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _CONNECTION_ERROR_CODES
+    return False
+
+
+def _is_retryable_tx_error(exc: Exception) -> bool:
+    """判断是否为可重试的事务级错误（死锁、锁超时）"""
+    if isinstance(exc, (OperationalError, InternalError)):
+        if exc.orig and hasattr(exc.orig, 'errno'):
+            return exc.orig.errno in _RETRYABLE_TX_ERROR_CODES
+    return False
+
+
+class BaseDBService(metaclass=SingletonMeta):
+    """数据库操作基础服务类，封装会话管理功能"""
+
+    def __init__(self):
+        self.engine = DatabaseService.engine()
+        self.Session = sessionmaker(bind=self.engine)
+
+    @contextmanager
+    def session(self):
+        """
+        数据库会话上下文管理器
+        自动处理会话的创建、提交、回滚和关闭
+        """
+        session = self.Session()
+        try:
+            yield session
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            SYLogger.error(f"Database operation failed: {str(e)}")
+            raise
+        finally:
+            session.close()
+
+    def execute(self, fn, retry: int = 0):
+        """
+        执行数据库操作，支持自动重试
+
+        连接级错误（连接断开 2006/2013）始终自动重试1次。
+        事务级错误（死锁 1213、锁超时 1205）按 retry 参数重试。
+
+        Args:
+            fn: 接收 session 参数的可调用对象，例如:
+                lambda session: session.query(User).all()
+            retry: 事务级错误最大重试次数，默认0不重试
+
+        Returns:
+            fn 的返回值
+
+        Usage:
+            result = self.execute(
+                lambda session: session.query(User).all(),
+                retry=3
+            )
+        """
+        attempt = 0
+        max_attempts = 1 + max(retry, 0)
+        connection_retried = False
+        last_exc = None
+
+        while True:
+            session = self.Session()
+            try:
+                result = fn(session)
+                session.commit()
+                return result
+            except Exception as e:
+                session.rollback()
+                last_exc = e
+
+                # 连接级错误：始终自动重试1次（不计入 max_attempts）
+                if _is_connection_error(e) and not connection_retried:
+                    connection_retried = True
+                    SYLogger.warning(
+                        f"DB connection lost (errno={getattr(e.orig, 'errno', None)}), retrying..."
+                    )
+                    attempt += 1
+                    continue
+
+                # 事务级错误：按 retry 参数重试
+                if retry > 0 and _is_retryable_tx_error(e) and attempt < max_attempts - 1:
+                    attempt += 1
+                    backoff = min(0.5 * (2 ** (attempt - 1)), 2.0)
+                    SYLogger.warning(
+                        f"DB retryable error (errno={getattr(e.orig, 'errno', None)}), "
+                        f"attempt {attempt}/{max_attempts}, backoff {backoff}s"
+                    )
+                    time.sleep(backoff)
+                    continue
+
+                SYLogger.error(f"Database operation failed: {str(e)}")
+                raise
+            finally:
+                session.close()

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/logging/kafka_log.py

@@ -1,6 +1,7 @@
 import os
 import sys
 import json
+import logging
 import socket
 import threading
 import traceback
@@ -176,6 +177,23 @@ class KafkaSink:
                 pass
 
 
+class _InterceptHandler(logging.Handler):
+    """将 stdlib logging 日志桥接到 Loguru，使 asyncio 等第三方库的日志进入 Kafka 管道"""
+
+    def emit(self, record: logging.LogRecord) -> None:
+        try:
+            level = logger.level(record.levelname).name
+        except ValueError:
+            level = record.levelno
+
+        frame, depth = logging.currentframe(), 2
+        while frame and frame.f_code.co_filename == logging.__file__:
+            frame = frame.f_back
+            depth += 1
+
+        logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
+
+
 class KafkaLogger(metaclass=SingletonMeta):
     _sink_instance = None
 
@@ -206,8 +224,48 @@ class KafkaLogger(metaclass=SingletonMeta):
             diagnose=True
         )
 
+        # 桥接 stdlib logging → Loguru，捕获 asyncio 等第三方库的日志
+        intercept_handler = _InterceptHandler()
+        logging.root.handlers = [intercept_handler]
+        logging.root.setLevel(logging.WARNING)
+
         sys.excepthook = KafkaLogger._handle_exception
 
+        # 注册 asyncio 异常处理器，捕获未处理的 Task 异常
+        KafkaLogger._setup_asyncio_exception_handler()
+
+    @staticmethod
+    def _setup_asyncio_exception_handler():
+        """注册 asyncio 事件循环异常处理器，将未处理的 Task 异常导入 Kafka"""
+        try:
+            loop = asyncio.get_running_loop()
+            loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+        except RuntimeError:
+            pass
+
+    @staticmethod
+    def _asyncio_exception_handler(loop, context):
+        """asyncio 未处理异常回调，通过 SYLogger 记录到 Kafka"""
+        message = context.get("exception", context.get("message", "Unknown asyncio error"))
+        exception = context.get("exception")
+
+        if exception:
+            tb_str = "".join(traceback.format_exception(
+                type(exception), exception, exception.__traceback__
+            ))
+        else:
+            tb_str = str(message)
+
+        trace_id = current_trace_id.get() or str(Snowflake.id)
+        error_msg = json.dumps({
+            "trace_id": trace_id,
+            "message": f"asyncio unhandled: {message}",
+            "level": "ERROR",
+            "detail": tb_str,
+        }, ensure_ascii=False)
+
+        logger.opt(exception=True).error(error_msg)
+
     @staticmethod
     def _handle_exception(exc_type, exc_value, exc_traceback):
         if issubclass(exc_type, KeyboardInterrupt):

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/models/mqlistener_config.py

@@ -32,6 +32,8 @@ class RabbitMQListenerConfig(BaseModel):
     auto_delete: bool = Field(False, description="是否自动删除队列")
     auto_parse_json: bool = Field(True, description="是否自动解析JSON消息")
     prefetch_count: int = Field(2, description="同时消费并发数（控制MQ预取与handler并发上限）")
+    max_priority: Optional[int] = Field(
+        10, description="队列支持的最大优先级（对应 x-max-priority）")
 
     class Config:
         """模型配置"""

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/rabbitmq/rabbitmq_client.py

@@ -42,6 +42,7 @@ class RabbitMQClient:
         auto_delete: bool = False,
         auto_parse_json: bool = True,
         create_if_not_exists: bool = True,
+        max_priority: Optional[int] = 10,
         **kwargs,
     ):
         self.connection_pool = connection_pool
@@ -60,6 +61,7 @@ class RabbitMQClient:
         self.auto_delete = auto_delete
         self.auto_parse_json = auto_parse_json
         self.create_if_not_exists = create_if_not_exists
+        self.max_priority = max_priority
 
         # 资源状态
         self._channel: Optional[AbstractChannel] = None
@@ -91,6 +93,97 @@ class RabbitMQClient:
         except Exception:
             return False
 
+    async def _try_migrate_queue(self, original_error: Exception) -> bool:
+        """尝试迁移队列以支持 x-max-priority。
+
+        当队列已存在但参数不匹配（PRECONDITION_FAILED）时，检查队列是否为空，
+        为空则删除重建，否则跳过迁移。
+
+        Returns:
+            True 迁移成功，False 未迁移（需由调用方处理）
+        """
+        from aiormq.exceptions import ChannelPreconditionFailed
+
+        if not isinstance(original_error, ChannelPreconditionFailed):
+            return False
+
+        if self.max_priority is None:
+            return False
+
+        try:
+            # PRECONDITION_FAILED 会关闭当前 channel，需要用新 channel 检查队列状态
+            channel, _ = await self.connection_pool.acquire_channel()
+            try:
+                queue = await channel.declare_queue(
+                    name=self.queue_name, passive=True
+                )
+                if queue.declaration_result:
+                    message_count = queue.declaration_result.message_count
+                    consumer_count = queue.declaration_result.consumer_count
+                else:
+                    message_count = 0
+                    consumer_count = 0
+            finally:
+                await channel.close()
+
+            if message_count > 0:
+                logger.warning(
+                    f"队列 '{self.queue_name}' 迁移跳过: "
+                    f"存在 {message_count} 条消息"
+                )
+                return False
+
+            logger.info(
+                f"开始迁移队列 '{self.queue_name}'，添加 x-max-priority={self.max_priority}")
+
+            # 删除旧队列
+            channel, _ = await self.connection_pool.acquire_channel()
+            try:
+                await channel.queue_delete(
+                    queue_name=self.queue_name,
+                    if_unused=False,
+                    if_empty=False,
+                )
+            finally:
+                await channel.close()
+
+            # 清理旧 channel（被 PRECONDITION_FAILED 关闭后残留）
+            if self._channel and not self._channel.is_closed:
+                try:
+                    await self._channel.close()
+                except Exception:
+                    pass
+            self._channel = None
+            self._exchange = None
+            self._queue = None
+
+            # 重建 self._channel（旧 channel 已被 PRECONDITION_FAILED 关闭）
+            self._channel, self._channel_conn = await self.connection_pool.acquire_channel()
+
+            # 重新声明交换机
+            self._exchange = await self._channel.declare_exchange(
+                name=self.exchange_name,
+                type=self.exchange_type,
+                durable=self.durable,
+                auto_delete=self.auto_delete,
+            )
+
+            # 重新声明带 priority 的队列
+            self._queue = await self._channel.declare_queue(
+                name=self.queue_name,
+                durable=self.durable,
+                auto_delete=self.auto_delete,
+                arguments={"x-max-priority": self.max_priority},
+            )
+            await self._queue.bind(exchange=self._exchange, routing_key=self.routing_key)
+            logger.info(
+                f"队列 '{self.queue_name}' 迁移成功，已添加 x-max-priority={self.max_priority}")
+            return True
+
+        except Exception as migrate_error:
+            logger.error(f"队列 '{self.queue_name}' 迁移失败: {migrate_error}")
+            return False
+
     async def _rebuild_resources(self) -> None:
         if not self._channel or self._channel.is_closed:
             raise RuntimeError("无有效通道，无法重建资源")
@@ -112,26 +205,32 @@ class RabbitMQClient:
         # 2. 声明队列 (仅消费者需要)
         # 【核心保留逻辑】只有当 queue_name 存在且符合消费者命名规范时，才初始化队列
         if self.queue_name and self.queue_name.endswith(f".{self.app_name}"):
+            queue_args = {}
+            if self.max_priority is not None:
+                queue_args["x-max-priority"] = self.max_priority
+
             try:
                 self._queue = await self._channel.declare_queue(
                     name=self.queue_name,
                     durable=self.durable,
                     auto_delete=self.auto_delete,
                     passive=not self.create_if_not_exists,
+                    arguments=queue_args or None,
                 )
                 await self._queue.bind(exchange=self._exchange, routing_key=self.routing_key)
                 logger.debug(f"队列就绪: {self.queue_name}")
             except Exception as e:
-                # === 关键处理逻辑 ===
-                if self.create_if_not_exists:
-                    # 模式：自动创建。如果失败说明权限不足或服务器限制，属于严重错误
-                    logger.error(f"❌ 无法自动创建队列 '{self.queue_name}': {e}")
-                    raise
-                else:
-                    # 模式：仅检查。队列不存在是预期内的（如生产者启动时队列未创建），跳过
+                if not self.create_if_not_exists:
                     self._queue = None
                     logger.warning(
                         f"⚠️ 队列 '{self.queue_name}' 不存在或无权访问 (Passive模式)，跳过绑定。")
+                    return
+
+                # 尝试自动迁移：参数不匹配时删除重建
+                migrated = await self._try_migrate_queue(e)
+                if not migrated:
+                    # 迁移失败（非参数不匹配 / 队列非空 / 有消费者），抛出原始异常
+                    raise
         else:
             # 不符合命名规范或无队列名，清空队列引用
             self._queue = None
@@ -145,7 +244,8 @@ class RabbitMQClient:
                     t.cancel()
             try:
                 await asyncio.wait_for(
-                    asyncio.gather(*self._running_tasks, return_exceptions=True),
+                    asyncio.gather(*self._running_tasks,
+                                   return_exceptions=True),
                     timeout=5.0
                 )
             except asyncio.TimeoutError:
@@ -350,16 +450,21 @@ class RabbitMQClient:
         if not self._queue and self.queue_name and self.queue_name.endswith(f".{self.app_name}"):
             logger.warning(f"队列对象在消费前为空，尝试强制初始化: {self.queue_name}")
             try:
+                queue_args = {}
+                if self.max_priority is not None:
+                    queue_args["x-max-priority"] = self.max_priority
                 self._queue = await self._channel.declare_queue(
                     name=self.queue_name,
                     durable=self.durable,
                     auto_delete=self.auto_delete,
-                    passive=False,  # 强制确保队列存在
+                    passive=False,
+                    arguments=queue_args or None,
                 )
                 await self._queue.bind(exchange=self._exchange, routing_key=self.routing_key)
             except Exception as e:
-                logger.error(f"强制初始化队列失败: {e}")
-                raise RuntimeError(f"无法初始化队列 {self.queue_name}")
+                migrated = await self._try_migrate_queue(e)
+                if not migrated:
+                    raise RuntimeError(f"无法初始化队列 {self.queue_name}")
 
         if not self._queue:
             raise RuntimeError("未配置队列名或队列初始化失败")
@@ -396,6 +501,7 @@ class RabbitMQClient:
         content_type: str = "application/json",
         delivery_mode: DeliveryMode = DeliveryMode.PERSISTENT,
         retry_count: int = 3,
+        priority: Optional[int] = None,
     ) -> None:
         if self._closed:
             raise RuntimeError("客户端已关闭，无法发布消息")
@@ -415,8 +521,11 @@ class RabbitMQClient:
             logger.error(f"消息体序列化失败: {e}")
             raise
 
-        message = Message(body=body, headers=headers or {},
+        msg_kwargs = dict(body=body, headers=headers or {},
                           content_type=content_type, delivery_mode=delivery_mode)
+        if priority is not None:
+            msg_kwargs["priority"] = priority
+        message = Message(**msg_kwargs)
         last_exception = None
 
         for attempt in range(retry_count):
@@ -471,11 +580,13 @@ class RabbitMQClient:
             logger.info(f"等待 {len(self._running_tasks)} 个后台任务完成...")
             try:
                 await asyncio.wait_for(
-                    asyncio.gather(*self._running_tasks, return_exceptions=True),
+                    asyncio.gather(*self._running_tasks,
+                                   return_exceptions=True),
                     timeout=15.0
                 )
             except asyncio.TimeoutError:
-                logger.warning(f"等待后台任务超时，剩余 {len(self._running_tasks)} 个任务将被丢弃")
+                logger.warning(
+                    f"等待后台任务超时，剩余 {len(self._running_tasks)} 个任务将被丢弃")
             self._running_tasks.clear()
 
         if self._channel and not self._channel.is_closed:

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/rabbitmq/rabbitmq_service_client_manager.py

@@ -183,6 +183,7 @@ class RabbitMQClientManager(RabbitMQCoreService):
             auto_parse_json=kwargs.get('auto_parse_json', True),
             create_if_not_exists=create_if_not_exists,
             prefetch_count=kwargs.get('prefetch_count', 2),
+            max_priority=kwargs.get('max_priority'),
         )
 
         # 4. 连接客户端
@@ -238,6 +239,9 @@ class RabbitMQClientManager(RabbitMQCoreService):
                             client.queue_name = input_qn
 
                 if await client.is_connected:
+                    # 更新 max_priority（发送器先创建客户端时没有此参数）
+                    if client_type == "listener" and 'max_priority' in kwargs:
+                        client.max_priority = kwargs['max_priority']
                     return client
                 else:
                     logger.info(f"客户端 '{client_name}' 连接已断开，重新创建")

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/rabbitmq/rabbitmq_service_producer_manager.py

@@ -218,7 +218,8 @@ class RabbitMQProducerManager(RabbitMQClientManager):
             await sender.publish(
                 message_body=mq_message.model_dump_json(),
                 headers=mq_header,
-                content_type="application/json"
+                content_type="application/json",
+                priority=kwargs.get('priority')
             )
             logger.info(f"消息发送成功 (队列: {queue_name})")
         except Exception as e:

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon/services.py

@@ -265,6 +265,14 @@ class Services(metaclass=SingletonMeta):
             sandbox_cleanup_task = None
 
             try:
+                # 注册 asyncio 异常处理器（需在事件循环运行后）
+                try:
+                    from sycommon.logging.kafka_log import KafkaLogger
+                    loop = asyncio.get_running_loop()
+                    loop.set_exception_handler(KafkaLogger._asyncio_exception_handler)
+                except Exception as e:
+                    logging.debug(f"注册 asyncio 异常处理器失败: {e}")
+
                 # 1. 处理数据库服务
                 if database_service:
                     instance._pending_async_db_setup = []

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon_python_lib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.5a24
+Version: 0.2.5a26
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.5a24 → sycommon_python_lib-0.2.5a26}/src/sycommon_python_lib.egg-info/SOURCES.txt

@@ -286,4 +286,5 @@ src/sycommon_python_lib.egg-info/dependency_links.txt
 src/sycommon_python_lib.egg-info/entry_points.txt
 src/sycommon_python_lib.egg-info/requires.txt
 src/sycommon_python_lib.egg-info/top_level.txt
+tests/test_asyncio_logging.py
 tests/test_sycli.py