sycommon-python-lib 0.2.8a12__py3-none-any.whl → 0.2.8a13__py3-none-any.whl

sycommon/rabbitmq/rabbitmq_service_connection_monitor.py

@@ -36,103 +36,121 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
     @classmethod
     async def _monitor_connections(cls):
         """连接监控任务：定期检查所有客户端连接状态和消费者健康
-        信任 aio-pika Robust 自动恢复，Monitor 只做兜底检测+安全重启。"""
+        信任 aio-pika Robust 自动恢复，Monitor 只做兜底检测+安全重启。
+
+        🔒 监控自保护（2026-08-24 线上实锤）：循环里任何一步 await 卡死
+        （broker 半开/网络黑洞/Kafka 日志阻塞）都会让整个监控永久停摆，
+        消费者丢失再无人检测——线上表现为"MQ 相关日志完全消失"。
+        现在每轮整体加 wait_for 看门狗，超时即放弃本轮进下一轮，
+        单轮卡死不再扩散成永久失明。"""
         logger.info("RabbitMQ连接监控任务启动")
         while not cls._is_shutdown:
             try:
                 await asyncio.sleep(cls.RECONNECT_INTERVAL)
+                try:
+                    await asyncio.wait_for(
+                        cls._monitor_one_round(), timeout=cls.MONITOR_ROUND_TIMEOUT)
+                except asyncio.TimeoutError:
+                    logger.warning(
+                        f"⚠️ 监控单轮超时({cls.MONITOR_ROUND_TIMEOUT}s)，"
+                        f"疑似 await 卡死，放弃本轮继续（防监控停摆）")
+            except Exception as e:
+                logger.error("RabbitMQ连接监控任务异常", exc_info=True)
+                await asyncio.sleep(cls.RECONNECT_INTERVAL)
+
+        logger.info("RabbitMQ连接监控任务停止")
 
-                # 跳过未初始化的连接池（使用公共方法）
-                if not cls.is_pool_initialized():
+    # 单轮监控超时（秒）：池探活+逐 client 检查的总预算
+    MONITOR_ROUND_TIMEOUT = 120
+
+    @classmethod
+    async def _monitor_one_round(cls) -> None:
+        """单轮监控（从 _monitor_connections 拆出，供看门狗 wait_for 包裹）。"""
+        # 跳过未初始化的连接池（使用公共方法）
+        if not cls.is_pool_initialized():
+            return
+
+        # 检查连接池本身状态
+        pool_alive = await cls._connection_pool.is_alive()
+        if not pool_alive:
+            # 🔒 池死亡主动重建（2026-08-21）：连续 N 轮不活即触发
+            # force_reconnect（丢掉旧连接走单飞重建）。不能只等
+            # aio-pika 原生恢复——robust 连接存在 transport=None
+            # 的卡死中间态（is_closed=False 但已断），原生恢复链
+            # 不会推进；纯消费者服务没有发送流量，acquire_channel
+            # 永远不会被触发，池就永久死了。
+            cls._pool_dead_rounds += 1
+            logger.error(
+                f"RabbitMQ连接池已断开 "
+                f"(连续 {cls._pool_dead_rounds}/"
+                f"{cls._POOL_DEAD_REBUILD_THRESHOLD} 轮)，等待原生自动重连")
+            if cls._pool_dead_rounds >= cls._POOL_DEAD_REBUILD_THRESHOLD:
+                cls._pool_dead_rounds = 0
+                try:
+                    asyncio.create_task(
+                        cls._connection_pool.force_reconnect())
+                    logger.warning(
+                        "🔄 连接池死亡超阈值，已触发主动 force_reconnect")
+                except Exception as e:
+                    logger.error(f"触发连接池重建失败: {e}", exc_info=True)
+            return
+        cls._pool_dead_rounds = 0
+
+        # 清理已完成的重连任务
+        completed_clients = [
+            name for name, task in cls._reconnect_tasks.items()
+            if task.done()
+        ]
+        for name in completed_clients:
+            del cls._reconnect_tasks[name]
+
+        # 检查所有客户端连接
+        for client_name, client in list(cls._clients.items()):
+            try:
+                # 跳过正在重连的客户端
+                if client_name in cls._reconnect_tasks:
                     continue
 
-                # 检查连接池本身状态
-                pool_alive = await cls._connection_pool.is_alive()
-                if not pool_alive:
-                    # 🔒 池死亡主动重建（2026-08-21）：连续 N 轮不活即触发
-                    # force_reconnect（丢掉旧连接走单飞重建）。不能只等
-                    # aio-pika 原生恢复——robust 连接存在 transport=None
-                    # 的卡死中间态（is_closed=False 但已断），原生恢复链
-                    # 不会推进；纯消费者服务没有发送流量，acquire_channel
-                    # 永远不会被触发，池就永久死了。
-                    cls._pool_dead_rounds += 1
-                    logger.error(
-                        f"RabbitMQ连接池已断开 "
-                        f"(连续 {cls._pool_dead_rounds}/"
-                        f"{cls._POOL_DEAD_REBUILD_THRESHOLD} 轮)，等待原生自动重连")
-                    if cls._pool_dead_rounds >= cls._POOL_DEAD_REBUILD_THRESHOLD:
-                        cls._pool_dead_rounds = 0
-                        try:
-                            asyncio.create_task(
-                                cls._connection_pool.force_reconnect())
-                            logger.warning(
-                                "🔄 连接池死亡超阈值，已触发主动 force_reconnect")
-                        except Exception as e:
-                            logger.error(f"触发连接池重建失败: {e}", exc_info=True)
+                client_connected = await client.is_connected
+
+                # 连接断开 → 尝试重连（不做破坏性清理）
+                if not client_connected:
+                    logger.warning(
+                        f"客户端 '{client_name}' 连接异常，尝试重连")
+                    task = asyncio.create_task(
+                        cls._reconnect_client_with_cleanup(client_name, client))
+                    cls._reconnect_tasks[client_name] = task
                     continue
-                cls._pool_dead_rounds = 0
 
-                # 清理已完成的重连任务
-                completed_clients = [
-                    name for name, task in cls._reconnect_tasks.items()
-                    if task.done()
-                ]
-                for name in completed_clients:
-                    del cls._reconnect_tasks[name]
-
-                # 检查所有客户端连接
-                for client_name, client in list(cls._clients.items()):
-                    try:
-                        # 跳过正在重连的客户端
-                        if client_name in cls._reconnect_tasks:
-                            continue
-
-                        client_connected = await client.is_connected
-
-                        # 连接断开 → 尝试重连（不做破坏性清理）
-                        if not client_connected:
-                            logger.warning(
-                                f"客户端 '{client_name}' 连接异常，尝试重连")
-                            task = asyncio.create_task(
-                                cls._reconnect_client_with_cleanup(client_name, client))
-                            cls._reconnect_tasks[client_name] = task
-                            continue
-
-                        # 消费者健康检查（本地状态层，快速路径）
-                        if client._message_handler and client._consumer_tag is None:
-                            logger.warning(
-                                f"客户端 '{client_name}' 消费者丢失(handler存在但无consumer_tag)，重新注册消费")
-                            await cls._restart_consumer_if_needed(client_name)
-                            continue
-
-                        # 消费者健康检查（本地注册表层，broker Cancel 感知）
-                        # broker 主动取消消费者（队列重建/节点切换）会发
-                        # Basic.Cancel，aiormq 收到后从 channel.consumers
-                        # pop 掉 tag——本地 tag 引用还在但注册表已空。
-                        if client._message_handler and client._consumer_tag \
-                                and not client.local_consumer_alive():
-                            logger.warning(
-                                f"客户端 '{client_name}' 本地消费者注册表已失效"
-                                f"(疑似 broker Basic.Cancel)，强制重启消费")
-                            if await client.force_restart_consuming():
-                                cls._sync_manager_consumer_tag(client_name, client)
-                            continue
-
-                        # 消费者健康检查（broker 真实状态层，兜底路径）
-                        # 根因修复：aio-pika OneShotCallback 恢复链断裂后，本地
-                        # task/tag/handler 全残留（三层全绿），但 broker 侧消费者
-                        # 已消失。唯一可信判据是 broker 的 consumer_count。
-                        if client._message_handler and client.queue_name:
-                            await cls._check_consumer_on_broker(client_name, client)
-                    except Exception as e:
-                        logger.error(
-                            f"监控客户端 '{client_name}' 连接状态失败: {str(e)}", exc_info=True)
+                # 消费者健康检查（本地状态层，快速路径）
+                if client._message_handler and client._consumer_tag is None:
+                    logger.warning(
+                        f"客户端 '{client_name}' 消费者丢失(handler存在但无consumer_tag)，重新注册消费")
+                    await cls._restart_consumer_if_needed(client_name)
+                    continue
 
-            except Exception as e:
-                logger.error("RabbitMQ连接监控任务异常", exc_info=True)
-                await asyncio.sleep(cls.RECONNECT_INTERVAL)
+                # 消费者健康检查（本地注册表层，broker Cancel 感知）
+                # broker 主动取消消费者（队列重建/节点切换）会发
+                # Basic.Cancel，aiormq 收到后从 channel.consumers
+                # pop 掉 tag——本地 tag 引用还在但注册表已空。
+                if client._message_handler and client._consumer_tag \
+                        and not client.local_consumer_alive():
+                    logger.warning(
+                        f"客户端 '{client_name}' 本地消费者注册表已失效"
+                        f"(疑似 broker Basic.Cancel)，强制重启消费")
+                    if await client.force_restart_consuming():
+                        cls._sync_manager_consumer_tag(client_name, client)
+                    continue
 
-        logger.info("RabbitMQ连接监控任务停止")
+                # 消费者健康检查（broker 真实状态层，兜底路径）
+                # 根因修复：aio-pika OneShotCallback 恢复链断裂后，本地
+                # task/tag/handler 全残留（三层全绿），但 broker 侧消费者
+                # 已消失。唯一可信判据是 broker 的 consumer_count。
+                if client._message_handler and client.queue_name:
+                    await cls._check_consumer_on_broker(client_name, client)
+            except Exception as e:
+                logger.error(
+                    f"监控客户端 '{client_name}' 连接状态失败: {str(e)}", exc_info=True)
 
     @classmethod
     async def _restart_consumer_if_needed(cls, client_name: str):

sycommon/rabbitmq/rabbitmq_service_consumer_manager.py

@@ -128,6 +128,49 @@ class RabbitMQConsumerManager(RabbitMQClientManager):
             name for name in required_clients if name not in running_clients and not cls._is_shutdown]
         if failed_clients:
             logger.error(f"以下消费者启动失败: {', '.join(failed_clients)}")
+            # 🔒 启动失败兜底重试（2026-08-24 线上实锤）：broker 瞬时 reset
+            # （Server connection reset 104）会让 setup 阶段的 consume 启动失败，
+            # 旧实现只打日志就返回——服务从此裸奔到下次重启（HTTP 正常、MQ 零消费，
+            # 监控循环也因 client 未初始化而无事可做）。改为后台持续重试直到成功。
+            asyncio.create_task(cls._retry_failed_consumers(failed_clients))
+
+    @classmethod
+    async def _retry_failed_consumers(cls, failed: List[str], max_minutes: int = 30) -> None:
+        """对启动失败的消费者做周期重试（指数退避 5s→60s，持续 max_minutes）。
+
+        成功标志：task 存活且拿到 consumer_tag。任何一轮部分成功则缩小重试名单。"""
+        import time as _time
+        pending = list(failed)
+        backoff = 5.0
+        deadline = _time.time() + max_minutes * 60
+        while pending and not cls._is_shutdown and _time.time() < deadline:
+            await asyncio.sleep(backoff)
+            if cls._is_shutdown:
+                return
+            still = []
+            for name in pending:
+                task = cls._consumer_tasks.get(name)
+                if (task is not None and not task.done()
+                        and name in cls._consumer_tags):
+                    logger.info(f"✅ 消费者 '{name}' 重试后已恢复消费")
+                    continue
+                try:
+                    logger.warning(
+                        f"消费者 '{name}' 启动失败重试中(退避{backoff:.0f}s)...")
+                    await cls.start_consumer(name)
+                except Exception as e:
+                    logger.warning(f"消费者 '{name}' 重试异常: {e}")
+                # 重试后再查：可能本轮已成功
+                t2 = cls._consumer_tasks.get(name)
+                if t2 is not None and not t2.done() and name in cls._consumer_tags:
+                    logger.info(f"✅ 消费者 '{name}' 重试成功")
+                else:
+                    still.append(name)
+            pending = still
+            backoff = min(backoff * 2, 60)
+        if pending:
+            logger.error(
+                f"消费者重试超时({max_minutes}min)仍未成功: {pending}，需人工介入")
 
     @classmethod
     async def start_all_consumers(cls) -> None:

{sycommon_python_lib-0.2.8a12.dist-info → sycommon_python_lib-0.2.8a13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.8a12
+Version: 0.2.8a13
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.8a12.dist-info → sycommon_python_lib-0.2.8a13.dist-info}/RECORD

@@ -264,8 +264,8 @@ sycommon/rabbitmq/rabbitmq_client.py,sha256=G36mcyFdrMSHO5FeMcDaVPEN_d8R2u18GCRH
 sycommon/rabbitmq/rabbitmq_pool.py,sha256=alZ4vDZAgulSaDbkGJy04nxh239wAVxiYvm2DTLXSBE,18163
 sycommon/rabbitmq/rabbitmq_service.py,sha256=EBepnotvzZvj34t40HEDup_GAtJI_i1xb795n3nh244,10043
 sycommon/rabbitmq/rabbitmq_service_client_manager.py,sha256=UZgpD6n0mdo5u3gaGW-jhUuPV50bRNdXFESzL9ABsR4,10623
-sycommon/rabbitmq/rabbitmq_service_connection_monitor.py,sha256=EbRmvkNuOSZ_SKHtMFn-TDMlOWuCfhhRybvTOyqX0_o,22685
-sycommon/rabbitmq/rabbitmq_service_consumer_manager.py,sha256=GeiS1GEtny5MxopIdKOCfeRB1AzkHZKPXKq9_jwg1KI,10274
+sycommon/rabbitmq/rabbitmq_service_connection_monitor.py,sha256=NtGKkwE_UdSSdFaEjEaLZdrYgUKCgalXrNmOdOVD8cA,23165
+sycommon/rabbitmq/rabbitmq_service_consumer_manager.py,sha256=6lsAq8X6xBAl8WzlVws6NSwkoD0F5P6WIB96upOh8w0,12601
 sycommon/rabbitmq/rabbitmq_service_core.py,sha256=_t8LQSUB5PyA16uHkjGgh8LvNVYg9DB7ICL0GYD3nlg,10600
 sycommon/rabbitmq/rabbitmq_service_producer_manager.py,sha256=60xryGoRDRS57cFIVQQj1CPxECXOQRBKXN4OaAknRQA,15132
 sycommon/sentry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -334,8 +334,8 @@ sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
 sycommon/tools/user_id.py,sha256=o-zsubGSR5rSSZf8Pk8B7AhE21ahDTTIW5JZmJO6Flw,4761
 sycommon/xxljob/__init__.py,sha256=7eoBlQxv-B39IfRSCY2bkqdGYs1QRe1umAWd88VMEEM,86
 sycommon/xxljob/xxljob_service.py,sha256=1yifwIBNGsCIxLnQjHKiBlbsigc_zvPH-dMTZcNxe-Q,7649
-sycommon_python_lib-0.2.8a12.dist-info/METADATA,sha256=AuGxUAW8tC-CIh63rj0YZf0PjwNRnry3-KpD4cyZJFU,8033
-sycommon_python_lib-0.2.8a12.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-sycommon_python_lib-0.2.8a12.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
-sycommon_python_lib-0.2.8a12.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
-sycommon_python_lib-0.2.8a12.dist-info/RECORD,,
+sycommon_python_lib-0.2.8a13.dist-info/METADATA,sha256=5hqjCXuLwT_Xod6rUQqUIuZF1tjiMJqh2akzY0LDCWM,8033
+sycommon_python_lib-0.2.8a13.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sycommon_python_lib-0.2.8a13.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
+sycommon_python_lib-0.2.8a13.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
+sycommon_python_lib-0.2.8a13.dist-info/RECORD,,