sycommon-python-lib 0.1.56b4__tar.gz → 0.1.56b6__tar.gz

{sycommon_python_lib-0.1.56b4 → sycommon_python_lib-0.1.56b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.1.56b4
+Version: 0.1.56b6
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.1.56b4 → sycommon_python_lib-0.1.56b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sycommon-python-lib"
-version = "0.1.56-beta4"
+version = "0.1.56-beta6"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.11"

{sycommon_python_lib-0.1.56b4 → sycommon_python_lib-0.1.56b6}/src/sycommon/rabbitmq/rabbitmq_client.py

@@ -145,10 +145,15 @@ class RabbitMQClient:
             raise RuntimeError("客户端已关闭，无法重新连接")
 
         async with self._connect_lock:
-            # 释放旧资源（回调+通道，单通道无需归还，仅清理状态）
-            if self._conn_close_callback and self._channel_conn:
-                self._channel_conn.close_callbacks.discard(
-                    self._conn_close_callback)
+            # 1. 清理旧连接回调（防止内存泄漏）
+            if self._channel_conn and self._conn_close_callback:
+                try:
+                    self._channel_conn.close_callbacks.discard(
+                        self._conn_close_callback)
+                except Exception:
+                    pass
+
+            # 2. 清理状态
             self._channel = None
             self._channel_conn = None
             self._exchange = None
@@ -156,18 +161,14 @@ class RabbitMQClient:
             self._conn_close_callback = None
 
             try:
-                # 从单通道池获取通道+连接（连接池自动确保通道有效）
+                # 3. 获取新通道
                 self._channel, self._channel_conn = await self.connection_pool.acquire_channel()
 
+                # 4. 设置新连接回调（使用 weakref）
                 def on_conn_closed(conn: AbstractRobustConnection, exc: Optional[BaseException]):
-                    """连接关闭回调：触发固定间隔重连"""
-                    logger.warning(
-                        f"客户端连接关闭: {conn!r}，原因: {exc}", exc_info=exc)
-                    self._reconnect_fail_count += 1
-                    # 超过阈值告警
-                    if self._reconnect_fail_count >= self._reconnect_alert_threshold:
-                        logger.error(
-                            f"连接失败次数已达阈值({self._reconnect_alert_threshold})，请检查MQ服务状态")
+                    # 注意：这里需要访问外部的 self，使用闭包或 weakref
+                    # 简单起见，这里用闭包，但务必在 self.close 或 self.connect 时清理回调
+                    logger.warning(f"检测到连接关闭: {exc}")
                     if not self._closed:
                         asyncio.create_task(self._safe_reconnect())
 
@@ -176,20 +177,26 @@ class RabbitMQClient:
                     self._channel_conn.close_callbacks.add(
                         self._conn_close_callback)
 
-                # 重建交换机/队列资源
+                # 5. 重建资源
                 await self._rebuild_resources()
 
-                # 重连成功，重置失败计数器
+                # 重置计数
                 self._reconnect_fail_count = 0
                 logger.info("客户端连接初始化完成")
             except Exception as e:
                 logger.error(f"客户端连接失败: {str(e)}", exc_info=True)
-                # 清理异常状态
-                if self._conn_close_callback and self._channel_conn:
-                    self._channel_conn.close_callbacks.discard(
-                        self._conn_close_callback)
+                # 失败时也要清理可能产生的残留引用
+                if self._channel_conn and self._conn_close_callback:
+                    try:
+                        self._channel_conn.close_callbacks.discard(
+                            self._conn_close_callback)
+                    except Exception:
+                        pass
+                # 清空状态
                 self._channel = None
                 self._channel_conn = None
+                self._conn_close_callback = None
+
                 # 触发重连
                 if not self._closed:
                     asyncio.create_task(self._safe_reconnect())
@@ -238,7 +245,7 @@ class RabbitMQClient:
             logger.info("消息处理器设置成功")
 
     async def start_consuming(self) -> Optional[ConsumerTag]:
-        """启动消息消费（支持自动重连）"""
+        """启动消息消费（支持自动重连 + Header 重试计数限制）"""
         if self._closed:
             raise RuntimeError("客户端已关闭，无法启动消费")
 
@@ -251,10 +258,11 @@ class RabbitMQClient:
             if not self._queue:
                 raise RuntimeError("未配置队列名或队列未创建，无法启动消费")
 
-            # 2. 定义消费回调（包含异常处理和重连逻辑）
+            # 2. 定义消费回调
             async def consume_callback(message: AbstractIncomingMessage):
                 try:
                     # 解析消息体
+                    msg_obj: MQMsgModel
                     if self.auto_parse_json:
                         try:
                             body_dict = json.loads(
@@ -263,19 +271,26 @@ class RabbitMQClient:
                         except json.JSONDecodeError as e:
                             logger.error(
                                 f"JSON消息解析失败: {str(e)}，消息体: {message.body[:100]}...")
-                            await message.nack(requeue=False)  # 解析失败，不重入队
+                            # 解析失败通常无法重试，直接丢弃
+                            await message.nack(requeue=False)
                             return
                     else:
                         msg_obj = MQMsgModel(
                             body=message.body.decode("utf-8"),
                             routing_key=message.routing_key,
                             delivery_tag=message.delivery_tag,
+                            traceId=message.headers.get("trace-id", None),
+                            headers=message.headers
                         )
 
+                    # 统一追踪ID
+                    SYLogger.set_trace_id(
+                        message.headers.get("trace-id", None))
+
                     # 调用消息处理器
                     await self._message_handler(msg_obj, message)
 
-                    # 手动ACK
+                    # 处理成功，手动ACK
                     await message.ack()
                     logger.debug(
                         f"消息处理成功，delivery_tag: {message.delivery_tag}")
@@ -285,19 +300,61 @@ class RabbitMQClient:
                         f"消息处理失败，delivery_tag: {message.delivery_tag}",
                         exc_info=True
                     )
-                    # 处理失败逻辑：首次失败重入队，再次失败丢弃
-                    if message.redelivered:
+
+                    # 1. 获取当前重试次数，默认为 0
+                    current_retry_count = 0
+                    if message.headers:
+                        current_retry_count = int(
+                            message.headers.get("x-retry-count", 0))
+
+                    # 2. 检查是否超过最大重试次数 (3次)
+                    MAX_RETRY = 3
+                    if current_retry_count >= MAX_RETRY:
                         logger.warning(
-                            f"消息已重入队过，本次拒绝入队: {message.delivery_tag}")
+                            f"消息重试次数已达上限({MAX_RETRY})，丢弃消息。"
+                            f"delivery_tag: {message.delivery_tag}, routing_key: {message.routing_key}"
+                        )
+                        # 丢弃消息（不重新入队）
                         await message.reject(requeue=False)
-                    else:
-                        logger.warning(f"消息重入队: {message.delivery_tag}")
-                        await message.nack(requeue=True)
+                        return
 
-                    # 连接失效则触发重连
-                    if not await self.is_connected:
-                        logger.warning("连接已失效，触发客户端重连")
-                        asyncio.create_task(self._safe_reconnect())
+                    # 3. 次数未满，准备重入队
+                    # 注意：AbstractIncomingMessage.headers 是只读的 (frozendict)，
+                    # 不能直接 message.headers["x-retry-count"] = ...
+                    # 必须重新构造 headers 字典。
+                    new_headers = dict(
+                        message.headers) if message.headers else {}
+                    new_headers["x-retry-count"] = current_retry_count + 1
+
+                    logger.warning(
+                        f"消息处理失败，准备第 {current_retry_count + 1} 次重入队。"
+                        f"delivery_tag: {message.delivery_tag}"
+                    )
+
+                    try:
+                        new_msg = Message(
+                            body=message.body,
+                            headers=new_headers,
+                            content_type=message.content_type,
+                            delivery_mode=message.delivery_mode
+                        )
+
+                        # 发布到原 Exchange 和 RoutingKey
+                        # 注意：这里可能会抛出异常，如果抛异常，消息可能会丢失（因为已经 Nack 掉了）
+                        await self._exchange.publish(
+                            new_msg,
+                            routing_key=message.routing_key
+                        )
+
+                        # 发布成功后，Ack 掉旧消息
+                        await message.ack()
+                        logger.info(
+                            f"消息已重新发布 (重试次数: {new_headers['x-retry-count']})")
+
+                    except Exception as publish_err:
+                        logger.error(f"重新发布消息失败(消息丢失): {publish_err}")
+                        # 重新发布失败，只能丢弃或者 Nack(False)
+                        await message.reject(requeue=False)
 
             # 3. 启动消费（单通道消费，避免阻塞发布需确保业务回调非阻塞）
             self._consumer_tag = await self._queue.consume(consume_callback)

sycommon_python_lib-0.1.56b6/src/sycommon/rabbitmq/rabbitmq_pool.py

@@ -0,0 +1,370 @@
+import asyncio
+import random
+from typing import Optional, List, Dict, Callable, Tuple
+from aio_pika import connect_robust, RobustChannel, Message
+from aio_pika.abc import (
+    AbstractRobustConnection, AbstractQueue, AbstractExchange, AbstractMessage
+)
+from sycommon.logging.kafka_log import SYLogger
+
+logger = SYLogger
+
+
+class AsyncProperty:
+    """实现 await obj.attr 的支持"""
+
+    def __init__(self, method):
+        self.method = method
+
+    def __get__(self, obj, objtype=None):
+        if obj is None:
+            return self
+        # 关键：当访问 obj.attr 时，直接返回协程对象，而不是方法本身
+        return self.method(obj)
+
+
+class RabbitMQConnectionPool:
+    """单连接单通道RabbitMQ客户端 (增强版日志)"""
+
+    def __init__(
+        self,
+        hosts: List[str],
+        port: int,
+        username: str,
+        password: str,
+        virtualhost: str = "/",
+        heartbeat: int = 30,
+        app_name: str = "",
+        connection_timeout: int = 30,
+        reconnect_interval: int = 5,
+        prefetch_count: int = 2,
+    ):
+        self.hosts = [host.strip() for host in hosts if host.strip()]
+        if not self.hosts:
+            raise ValueError("至少需要提供一个RabbitMQ主机地址")
+
+        self.port = port
+        self.username = username
+        self.password = password
+        self.virtualhost = virtualhost
+        self.app_name = app_name or "rabbitmq-client"
+        self.heartbeat = heartbeat
+        self.connection_timeout = connection_timeout
+        self.reconnect_interval = reconnect_interval
+        self.prefetch_count = prefetch_count
+
+        self._current_host: str = random.choice(self.hosts)
+        logger.info(f"[INIT] 随机选择RabbitMQ主机: {self._current_host}")
+
+        # 核心资源
+        self._connection: Optional[AbstractRobustConnection] = None
+        self._channel: Optional[RobustChannel] = None
+        self._consumer_channels: Dict[str, RobustChannel] = {}
+
+        # 状态控制
+        self._lock = asyncio.Lock()
+        self._initialized = False
+        self._is_shutdown = False
+
+    @AsyncProperty
+    async def is_alive(self) -> bool:
+        """对外暴露的连接存活状态（原子化判断）"""
+        async with self._lock:
+            if self._is_shutdown:
+                return False
+
+            if not self._initialized:
+                return False
+
+            if self._connection is None or self._connection.is_closed:
+                return False
+
+            # 可选：检查主通道是否存活
+            if self._channel is None or self._channel.is_closed:
+                # 如果你认为通道断了连接也算死，就保留这行；否则删除
+                return False
+
+            return True
+
+    async def _create_connection_impl(self) -> AbstractRobustConnection:
+        """
+        连接创建入口
+        """
+        conn_url = (
+            f"amqp://{self.username}:{self.password}@{self._current_host}:{self.port}/"
+            f"{self.virtualhost}?name={self.app_name}&heartbeat={self.heartbeat}"
+            f"&reconnect_interval={self.reconnect_interval}&fail_fast=1"
+        )
+        logger.info(
+            f"🔌 [CONNECT_START] 尝试创建连接 -> {self._current_host}:{self.port}")
+        try:
+            conn = await connect_robust(conn_url, timeout=self.connection_timeout)
+            # 注意：connect_robust 返回时，底层 TCP 可能还在握手，但对象已创建
+            logger.info(f"✅ [CONNECT_OK] 连接对象创建成功: {id(conn)}")
+            return conn
+        except Exception as e:
+            logger.error(f"❌ [CONNECT_FAIL] 连接创建失败: {str(e)}")
+            raise ConnectionError(f"无法连接RabbitMQ {self._current_host}") from e
+
+    async def _create_channel_impl(self, connection: AbstractRobustConnection) -> RobustChannel:
+        """创建通道"""
+        try:
+            channel = await connection.channel()
+            await channel.set_qos(prefetch_count=self.prefetch_count)
+            logger.debug(f"✅ [CHANNEL_OK] 通道创建成功: {id(channel)}")
+            return channel
+        except Exception as e:
+            logger.error(f"❌ [CHANNEL_FAIL] 通道创建失败: {str(e)}")
+            raise
+
+    async def _ensure_main_channel(self) -> RobustChannel:
+        """确保主通道有效 (原子操作 + 自动轮询重试 + 失败清理)"""
+        async with self._lock:
+            if self._is_shutdown:
+                raise RuntimeError("客户端已关闭")
+
+            # 如果连接对象不存在或已关闭，进入重连流程
+            if self._connection is None or self._connection.is_closed:
+                retry_hosts = self.hosts.copy()  # 复制一份列表用于重试
+                last_error = None
+                temp_conn = None  # 临时变量，用于引用本次尝试创建的连接
+
+                # 轮询尝试列表中的每一个 Host
+                while retry_hosts:
+                    host = random.choice(retry_hosts)
+                    retry_hosts.remove(host)  # 从待尝试列表中移除
+
+                    logger.info(f"⚠️ [RECONNECT] 尝试连接节点: {host}")
+                    self._current_host = host
+                    temp_conn = None  # 重置临时引用
+
+                    try:
+                        conn_url = (
+                            f"amqp://{self.username}:{self.password}@{host}:{self.port}/"
+                            f"{self.virtualhost}?name={self.app_name}&heartbeat={self.heartbeat}"
+                            f"&reconnect_interval={self.reconnect_interval}&fail_fast=1"
+                        )
+
+                        # 尝试连接
+                        temp_conn = await connect_robust(conn_url, timeout=self.connection_timeout)
+
+                        # --- 关键点：连接成功 ---
+                        self._connection = temp_conn
+                        temp_conn = None  # 清空临时引用，因为所有权已经移交给 self._connection
+
+                        logger.info(f"🔗 [RECONNECT_OK] 成功连接到节点: {host}")
+                        last_error = None
+                        break  # 成功，跳出循环
+
+                    except Exception as e:
+                        logger.warning(
+                            f"❌ [RECONNECT_FAIL] 节点 {host} 不可用: {str(e)}")
+
+                        # 【核心修复】清理失败的连接对象
+                        if temp_conn is not None:
+                            try:
+                                logger.debug(
+                                    f"🧹 [CLEANUP] 正在关闭失败的连接对象: {id(temp_conn)}")
+                                # 即使连接对象处于异常状态，close() 通常也是安全的
+                                await temp_conn.close()
+                                logger.debug(f"✅ [CLEANUP] 失败连接已关闭")
+                            except Exception as close_err:
+                                logger.warning(
+                                    f"⚠️ [CLEANUP_ERR] 关闭失败连接时出错: {str(close_err)}")
+
+                        last_error = e
+                        asyncio.sleep(self.reconnect_interval)
+                        continue  # 继续试下一个
+
+                # 如果所有节点都试完了还是失败
+                if last_error:
+                    logger.error("💥 [RECONNECT_FATAL] 所有 RabbitMQ 节点均不可用")
+                    raise ConnectionError("所有 RabbitMQ 节点连接失败") from last_error
+
+            # 2. 确保主通道存在
+            if self._channel is None or self._channel.is_closed:
+                logger.info("⚠️ [RECOVER_CHANNEL] 检测到主通道不存在或已关闭，开始恢复...")
+                self._channel = await self._create_channel_impl(self._connection)
+
+            return self._channel
+
+    async def init_pools(self):
+        """
+        初始化入口与异常处理 (修复泄漏的关键)
+        """
+        async with self._lock:
+            if self._is_shutdown:
+                raise RuntimeError("客户端已关闭")
+            if self._initialized:
+                return
+
+        conn_created_in_this_try = None
+        try:
+            # 步骤 A: 创建连接 (在锁外进行，避免阻塞其他操作)
+            conn = await self._create_connection_impl()
+            conn_created_in_this_try = conn  # 记录本次创建的对象，用于失败回滚
+
+            # 步骤 B: 更新状态和初始化通道 (在锁内进行，保证原子性)
+            async with self._lock:
+                if self._is_shutdown:
+                    # 如果在创建连接期间，外部调用了 close，则必须立即清理刚创建的连接
+                    logger.warning("⚠️ [ABORT] 检测到关闭信号，放弃初始化并清理资源")
+                    raise RuntimeError("客户端已关闭")
+
+                self._connection = conn
+                self._channel = await self._create_channel_impl(conn)
+                self._initialized = True
+                logger.info(
+                    f"🚀 [INIT_SUCCESS] 客户端初始化完成. ConnID: {id(self._connection)}")
+
+        except Exception as e:
+            logger.error(f"💥 [INIT_ERROR] 初始化流程异常: {str(e)}", exc_info=True)
+            # 如果步骤A成功但步骤B失败（例如通道创建失败），或者步骤B中出错，
+            # 必须显式关闭在步骤A中创建的连接，否则它会变成“游离连接”。
+            if conn_created_in_this_try:
+                logger.warning(
+                    f"🧹 [LEAK_PREVENTION] 检测到初始化失败，正在显式关闭刚创建的连接: {id(conn_created_in_this_try)}")
+                try:
+                    await conn_created_in_this_try.close()
+                    logger.info(
+                        f"✅ [CLOSE_OK] 泄漏连接已关闭: {id(conn_created_in_this_try)}")
+                except Exception as close_err:
+                    logger.error(f"❌ [CLOSE_ERR] 关闭泄漏连接时出错: {str(close_err)}")
+
+            # 如果是因为中途关闭导致的错误，不需要再次调用全局 close，否则调用
+            if not self._is_shutdown:
+                await self.close()
+            raise
+
+    async def acquire_channel(self) -> Tuple[RobustChannel, AbstractRobustConnection]:
+        """获取主通道"""
+        if not self._initialized and not self._is_shutdown:
+            await self.init_pools()
+        return await self._ensure_main_channel(), self._connection
+
+    async def publish_message(self, routing_key: str, message_body: bytes, exchange_name: str = "", **kwargs):
+        """发布消息"""
+        channel, _ = await self.acquire_channel()
+        try:
+            exchange = channel.default_exchange if not exchange_name else await channel.get_exchange(exchange_name)
+            message = Message(body=message_body, **kwargs)
+            await exchange.publish(message, routing_key=routing_key)
+            logger.debug(f"📤 [PUBLISH] 消息发布成功 - RK: {routing_key}")
+        except Exception as e:
+            logger.error(f"❌ [PUBLISH_FAIL] 发布失败: {str(e)}")
+            raise
+
+    async def consume_queue(self, queue_name: str, callback: Callable[[AbstractMessage], asyncio.Future], auto_ack: bool = False, **kwargs):
+        """消费队列"""
+        if not self._initialized:
+            await self.init_pools()
+
+        async with self._lock:
+            if self._is_shutdown:
+                raise RuntimeError("客户端已关闭")
+            if queue_name in self._consumer_channels:
+                logger.warning(f"⚠️ [CONSUMER_EXISTS] 队列 {queue_name} 已在消费中")
+                return
+            if not self._connection or self._connection.is_closed:
+                raise RuntimeError("连接不可用，无法启动消费")
+
+        await self.declare_queue(queue_name, **kwargs)
+
+        try:
+            # 获取原始连接对象创建新通道
+            conn = self._connection
+            consumer_channel = await conn.channel()
+            await consumer_channel.set_qos(prefetch_count=self.prefetch_count)
+            logger.info(
+                f"✅ [CONSUMER_CHANNEL_OK] 消费者通道创建: {id(consumer_channel)}")
+
+            async with self._lock:
+                if self._is_shutdown:
+                    await consumer_channel.close()
+                    return
+                self._consumer_channels[queue_name] = consumer_channel
+
+            async def consume_callback_wrapper(message: AbstractMessage):
+                try:
+                    await callback(message)
+                    if not auto_ack:
+                        await message.ack()
+                except Exception as e:
+                    logger.error(
+                        f"❌ [CALLBACK_ERR] 消费回调异常 {queue_name}: {str(e)}")
+                    if not auto_ack:
+                        await message.nack(requeue=True)
+
+            await consumer_channel.basic_consume(
+                queue_name, consumer_callback=consume_callback_wrapper, auto_ack=auto_ack, **kwargs
+            )
+            logger.info(f"🎧 [CONSUME_START] 开始消费队列: {queue_name}")
+
+        except Exception as e:
+            logger.error(f"💥 [CONSUME_ERR] 启动消费失败 {queue_name}: {str(e)}")
+            async with self._lock:
+                if queue_name in self._consumer_channels:
+                    del self._consumer_channels[queue_name]
+            raise
+
+    async def close(self):
+        """
+        资源销毁入口
+        """
+        async with self._lock:
+            if self._is_shutdown:
+                return
+            self._is_shutdown = True
+            self._initialized = False
+            # 记录即将关闭的连接ID
+            conn_to_close_id = id(
+                self._connection) if self._connection else None
+
+        logger.info(f"🛑 [CLOSE_START] 开始关闭客户端... (准备关闭连接: {conn_to_close_id})")
+
+        # 1. 关闭消费者通道
+        channels_to_close = []
+        async with self._lock:
+            channels_to_close = list(self._consumer_channels.values())
+            self._consumer_channels.clear()
+
+        for ch in channels_to_close:
+            try:
+                if not ch.is_closed:
+                    await ch.close()
+                    logger.debug(f"✅ [CLOSE_CHANNEL] 消费者通道已关闭")
+            except Exception as e:
+                logger.warning(f"❌ [CLOSE_CHANNEL_ERR] 关闭消费者通道失败: {str(e)}")
+
+        # 2. 关闭主通道
+        if self._channel:
+            try:
+                if not self._channel.is_closed:
+                    await self._channel.close()
+                    logger.info(f"✅ [CLOSE_CHANNEL] 主通道已关闭")
+            except Exception:
+                pass
+            self._channel = None
+
+        # 3. 关闭连接
+        # 确保在 finally 或显式 close 中调用 connection.close()
+        if self._connection:
+            try:
+                # 打印关闭操作
+                logger.info(f"🔌 [CLOSE_CONN] 正在关闭连接: {id(self._connection)}")
+                await self._connection.close()
+                logger.info(f"✅ [CLOSE_OK] 连接已成功关闭: {id(self._connection)}")
+            except Exception as e:
+                logger.warning(f"❌ [CLOSE_ERR] 关闭连接失败: {str(e)}")
+            self._connection = None
+
+        logger.info("🏁 [CLOSE_DONE] RabbitMQ客户端已完全关闭")
+
+    # --- 辅助方法省略 (declare_queue 等) ---
+    async def declare_queue(self, queue_name: str, **kwargs) -> AbstractQueue:
+        channel, _ = await self.acquire_channel()
+        return await channel.declare_queue(queue_name, **kwargs)
+
+    async def declare_exchange(self, exchange_name: str, exchange_type: str = "direct", **kwargs) -> AbstractExchange:
+        channel, _ = await self.acquire_channel()
+        return await channel.declare_exchange(exchange_name, exchange_type, **kwargs)