sycommon-python-lib 0.2.8a10__py3-none-any.whl → 0.2.8a12__py3-none-any.whl

sycommon/rabbitmq/rabbitmq_client.py

@@ -4,6 +4,7 @@ import json
 import os
 from typing import Optional, Callable, Dict, Any, Union
 from aio_pika import Message, DeliveryMode, ExchangeType
+from aiormq.exceptions import ChannelNotFoundEntity
 from aio_pika.abc import (
     AbstractChannel,
     AbstractExchange,
@@ -140,6 +141,14 @@ class RabbitMQClient:
         try:
             state = await self._probe_queue_state()
             return state[1] if state else 0
+        except ChannelNotFoundEntity:
+            # 🔒 队列已不存在（被动声明 404）≠ 无法探测。队列被删（迁移/清理）
+            # 时 broker 已发 Basic.Cancel，消费者必然为 0；返回 0 让监控的
+            # 连续确认机制触发 force_restart_consuming 重建队列
+            # （create_if_not_exists=True 会重新 declare）。
+            logger.warning(
+                f"队列 '{self.queue_name}' 在 broker 上不存在，消费者必然为 0")
+            return 0
         except Exception as e:
             logger.debug(f"探测队列 '{self.queue_name}' 消费者数失败: {e}")
             return None
@@ -163,11 +172,19 @@ class RabbitMQClient:
         if channel is None:
             return False
         try:
+            # 🔒 真实结构（2026-08-21 真实MQ连接实证修正）：
+            # aio_pika.Channel._channel 是 UnderlayChannel 包装器(abc.py:509)，
+            # 其 .channel 属性才是 aiormq 的 Channel，consumers 注册表挂在
+            # 那上面。旧的单路径 getattr(underlay,"consumers") 在真实环境
+            # 恒取到 None → 判定退化为恒 True。双路径兼容两种形态：
+            #   1) _channel = UnderlayChannel → .channel.consumers（aio-pika 10.x 实际）
+            #   2) _channel = aiormq Channel → .consumers（未来版本直挂/测试直挂）
             underlay = channel._channel
-            if underlay is None or underlay.channel is None:
+            if underlay is None:
                 return False
+            target = getattr(underlay, "channel", None) or underlay
             # aiormq Channel.consumers: broker Cancel 时会被 pop
-            consumers = getattr(underlay.channel, "consumers", None)
+            consumers = getattr(target, "consumers", None)
             if isinstance(consumers, dict):
                 if self._consumer_tag not in consumers:
                     logger.warning(
@@ -628,6 +645,13 @@ class RabbitMQClient:
             logger.warning(f"强制重启前停止消费异常(忽略): {e}")
             self._consumer_tag = None
 
+        # 🔒 清 stale 队列引用：stop_consuming 失败（队列已删/通道已死）时
+        # self._queue 仍是旧对象，start_consuming 见 _queue 非空就跳过重新
+        # 声明、直接在死队列上 consume() → 必然失败 → 退避后重试 → 无限
+        # 失败循环。置空强制走 _ensure_consumer_queue 重新声明
+        # （队列被删场景由 create_if_not_exists=True 重建）。
+        self._queue = None
+
         try:
             if not await self.is_connected:
                 await self.connect()

sycommon/rabbitmq/rabbitmq_service_connection_monitor.py

@@ -15,8 +15,18 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
     _reconnect_tasks: Dict[str, asyncio.Task] = {}
     # 消费者强制重启的退避记录：client_name -> (下次允许重启的时间戳, 连续失败数)
     _consumer_restart_backoff: Dict[str, tuple] = {}
-    # 连续探测失败次数（broker 不可达时不做重启判定）
-    _probe_fail_count: int = 0
+    # 🔒 P2-1 根治（2026-08-21）：探测失败计数按 client 维度隔离。
+    # 旧实现是类级单一 int —— 多 client 并存时互相污染：A 探测失败 +1，
+    # B 探测成功清零，A 的失败史被抹掉（或反向累加误触发）。
+    # 现改为 Dict[client_name, count]，语义不变：仅用于诊断日志。
+    _probe_fail_counts: Dict[str, int] = {}
+    # 🔒 池死亡连续轮数：is_alive 连续 False 达阈值后主动 force_reconnect。
+    # 旧实现只打日志 continue —— 池的重建入口仅有 acquire_channel（有人
+    # 发消息时）与 publish 失败两条路径，纯消费者服务（不发消息）踩到
+    # robust 连接"transport=None 卡死"中间态后，没有任何代码会去重建池，
+    # 监控永久跳过、消费者丢失无人检测无人恢复。
+    _pool_dead_rounds: int = 0
+    _POOL_DEAD_REBUILD_THRESHOLD = 3  # 连续 3 轮(约45s)判定池需主动重建
 
     # 连续 N 个周期确认消费者丢失后才强制重启（防 manager API 瞬时抖动误判）
     _CONSUMER_LOSS_CONFIRM_ROUNDS = 2
@@ -39,8 +49,28 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
                 # 检查连接池本身状态
                 pool_alive = await cls._connection_pool.is_alive()
                 if not pool_alive:
-                    logger.error("RabbitMQ连接池已断开，等待原生自动重连")
+                    # 🔒 池死亡主动重建（2026-08-21）：连续 N 轮不活即触发
+                    # force_reconnect（丢掉旧连接走单飞重建）。不能只等
+                    # aio-pika 原生恢复——robust 连接存在 transport=None
+                    # 的卡死中间态（is_closed=False 但已断），原生恢复链
+                    # 不会推进；纯消费者服务没有发送流量，acquire_channel
+                    # 永远不会被触发，池就永久死了。
+                    cls._pool_dead_rounds += 1
+                    logger.error(
+                        f"RabbitMQ连接池已断开 "
+                        f"(连续 {cls._pool_dead_rounds}/"
+                        f"{cls._POOL_DEAD_REBUILD_THRESHOLD} 轮)，等待原生自动重连")
+                    if cls._pool_dead_rounds >= cls._POOL_DEAD_REBUILD_THRESHOLD:
+                        cls._pool_dead_rounds = 0
+                        try:
+                            asyncio.create_task(
+                                cls._connection_pool.force_reconnect())
+                            logger.warning(
+                                "🔄 连接池死亡超阈值，已触发主动 force_reconnect")
+                        except Exception as e:
+                            logger.error(f"触发连接池重建失败: {e}", exc_info=True)
                     continue
+                cls._pool_dead_rounds = 0
 
                 # 清理已完成的重连任务
                 completed_clients = [
@@ -117,7 +147,18 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
             # 检查消费任务状态
             task_exists = client_name in RabbitMQConsumerManager._consumer_tasks
             task = RabbitMQConsumerManager._consumer_tasks.get(client_name)
-            has_tag = client_name in RabbitMQConsumerManager._consumer_tags
+            # 🔒 tag 残留修复（2026-08-22 线上实锤）：manager._consumer_tags 只在
+            # consume_task 的 finally 里清理，而 task 正常时永远挂在
+            # stop_event.wait() 不退出 → 一旦 client._consumer_tag 被清
+            # （通道关闭钩子/恢复失败置None），manager 侧记录残留 has_tag=True，
+            # 假死判定分支被整体跳过（"task运行中且有tag→正常"），监控每 15s
+            # 打"重新注册消费"却永远不做任何事——线上消费者丢失 70+ 分钟
+            # 未自愈即此死锁。has_tag 必须以 client 真实状态为准。
+            client_obj = RabbitMQConsumerManager._clients.get(client_name)
+            client_tag_alive = bool(
+                client_obj and getattr(client_obj, "_consumer_tag", None))
+            has_tag = (client_name in RabbitMQConsumerManager._consumer_tags) \
+                and client_tag_alive
 
             # 【兜底】有 handler 但 task 不存在 → 消费者根本没起来，直接重启
             # （例如 start_consumer 抛异常、或 reload/异常路径把 task 清掉但 handler 还在）
@@ -198,11 +239,12 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
             broker_consumers = None
             logger.debug(f"探测 '{client_name}' broker 消费者数异常: {e}")
 
-        # 探测不可用：不动状态、不重启，等下个周期
+        # 探测不可用：不动状态、不重启，等下个周期（计数按 client 隔离）
         if broker_consumers is None:
-            cls._probe_fail_count += 1
+            cls._probe_fail_counts[client_name] = (
+                cls._probe_fail_counts.get(client_name, 0) + 1)
             return
-        cls._probe_fail_count = 0
+        cls._probe_fail_counts.pop(client_name, None)
 
         if broker_consumers > 0:
             # broker 侧有消费者。单实例场景下即本实例，健康；
@@ -265,16 +307,31 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
     # 管理 API 客户端（懒初始化，None = 未启用）
     _mgmt_client = None
     _mgmt_client_initialized = False
+    # 🔒 P2-2：构建 mgmt_client 时的配置指纹（host|user|mgmt端口），
+    # 变化即重建客户端，reload 换集群后管理 API 跟随切换。
+    _mgmt_fingerprint: str = ""
 
     @classmethod
     def _get_mgmt_client(cls):
-        """懒构建管理 API 客户端（进程级单例；未配置/构建失败返回 None）。
+        """懒构建管理 API 客户端；未配置/构建失败返回 None。
 
         多实例精确判定依赖 mq.yml 的 spring.rabbitmq.management 段；
-        未配置时返回 None，调用方退化为队列级 count 判定。"""
-        if cls._mgmt_client_initialized:
+        未配置时返回 None，调用方退化为队列级 count 判定。
+
+        🔒 P2-2 根治（2026-08-21）：以「构建时的 host 指纹」检测配置变更。
+        reload 换池会更新 cls._config（host/账号可能变），旧实现一旦初始化
+        就永不重建 → 管理 API 客户端永远指向旧集群。现比对当前配置指纹，
+        变化即重建（未配置 management 段时指纹为空，行为与旧版一致）。"""
+        current_fingerprint = ""
+        try:
+            if cls._config:
+                current_fingerprint = f"{cls._config.get('host','')}|{cls._config.get('username','')}|{(cls._config.get('management') or {}).get('port', 15672)}"
+        except Exception:
+            current_fingerprint = ""
+        if cls._mgmt_client_initialized and current_fingerprint == cls._mgmt_fingerprint:
             return cls._mgmt_client
         cls._mgmt_client_initialized = True
+        cls._mgmt_fingerprint = current_fingerprint
         try:
             if not cls._config:
                 return None
@@ -376,5 +433,8 @@ class RabbitMQConnectionMonitor(RabbitMQClientManager):
         cls._connection_monitor_task = None
         cls._consumer_restart_backoff.clear()
         cls._consumer_loss_rounds.clear()
+        cls._pool_dead_rounds = 0
+        cls._probe_fail_counts.clear()
         cls._mgmt_client = None
         cls._mgmt_client_initialized = False
+        cls._mgmt_fingerprint = ""

sycommon/tests/test_consumer_broker_health.py

@@ -289,13 +289,16 @@ def test_force_restart_failure_clears_tag():
 # ------------------------------------------------------ 多副本/多Worker 场景
 
 def _make_channel_stack(consumers: dict):
-    """构造真实层级：aio_pika Channel → UnderlayChannel → aiormq Channel.
+    """构造真实层级（2026-08-21 真实MQ连接实证）：
+    aio_pika Channel._channel = UnderlayChannel(abc.py:509 包装器)，
+    其 .channel 才是 aiormq Channel，consumers 注册表挂在后者上。
 
-    local_consumer_alive 内省链：client._channel(_aio_pika Channel)
-    ._channel(UnderlayChannel).channel(aiormq Channel).consumers(dict)"""
+    local_consumer_alive 双路径兼容：
+    - UnderlayChannel 形态 → .channel.consumers（aio-pika 10.x 实际）
+    - aiormq Channel 直挂形态 → .consumers（未来版本/测试直挂）"""
     aiormq_ch = MagicMock()
     aiormq_ch.consumers = consumers
-    underlay = MagicMock()
+    underlay = MagicMock(spec=["channel", "close", "close_callback"])
     underlay.channel = aiormq_ch
     aio_pika_ch = MagicMock()
     aio_pika_ch._channel = underlay

{sycommon_python_lib-0.2.8a10.dist-info → sycommon_python_lib-0.2.8a12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.2.8a10
+Version: 0.2.8a12
 Summary: Add your description here
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.2.8a10.dist-info → sycommon_python_lib-0.2.8a12.dist-info}/RECORD

@@ -260,11 +260,11 @@ sycommon/notice/uvicorn_monitor.py,sha256=_SlLB91N7g-Jt_xP_AX1IWFgA7_1lCqsQFwm_Y
 sycommon/notice/wecom_message.py,sha256=_oA6hg0Ask7NkFbvUpL78WOmyUb6uW7KcY71GhNKiJY,12363
 sycommon/rabbitmq/process_pool_consumer.py,sha256=EXdNypgathjTfPqkccoAW0HiVB9HuZLGhPbWUcYh0_Q,29750
 sycommon/rabbitmq/rabbit_management_client.py,sha256=cf7GSbvdzOA6vFV4jC3IL6wh8yu8IEDgTPHUqRV2_uE,4021
-sycommon/rabbitmq/rabbitmq_client.py,sha256=9g5iKY1tbJ-Vq0gSqUuYyBvOdDhEWhV1fqdJHFdRjlk,35652
+sycommon/rabbitmq/rabbitmq_client.py,sha256=G36mcyFdrMSHO5FeMcDaVPEN_d8R2u18GCRHG_v8eE8,37343
 sycommon/rabbitmq/rabbitmq_pool.py,sha256=alZ4vDZAgulSaDbkGJy04nxh239wAVxiYvm2DTLXSBE,18163
 sycommon/rabbitmq/rabbitmq_service.py,sha256=EBepnotvzZvj34t40HEDup_GAtJI_i1xb795n3nh244,10043
 sycommon/rabbitmq/rabbitmq_service_client_manager.py,sha256=UZgpD6n0mdo5u3gaGW-jhUuPV50bRNdXFESzL9ABsR4,10623
-sycommon/rabbitmq/rabbitmq_service_connection_monitor.py,sha256=runtHGs9ArOnm8FpQ3A4Q5nzcpvTb_KzdVmZnxP4x18,18421
+sycommon/rabbitmq/rabbitmq_service_connection_monitor.py,sha256=EbRmvkNuOSZ_SKHtMFn-TDMlOWuCfhhRybvTOyqX0_o,22685
 sycommon/rabbitmq/rabbitmq_service_consumer_manager.py,sha256=GeiS1GEtny5MxopIdKOCfeRB1AzkHZKPXKq9_jwg1KI,10274
 sycommon/rabbitmq/rabbitmq_service_core.py,sha256=_t8LQSUB5PyA16uHkjGgh8LvNVYg9DB7ICL0GYD3nlg,10600
 sycommon/rabbitmq/rabbitmq_service_producer_manager.py,sha256=60xryGoRDRS57cFIVQQj1CPxECXOQRBKXN4OaAknRQA,15132
@@ -289,7 +289,7 @@ sycommon/tests/deep_agent_server.py,sha256=fYr3IUTywRwSEoz0ImTz_0uXHnl9JBy6McNbA
 sycommon/tests/test_backend_factory_compat.py,sha256=L8TV5EU6x2SNwoQGbi4ytHn3IDNhMefNuDFWBo7UrLI,7136
 sycommon/tests/test_consistent_hash.py,sha256=sOHM88LeQCqpXtSjFecv1ylKZ-PsTKAqFTm_V7pHvFQ,10015
 sycommon/tests/test_consume_dedup.py,sha256=3O9JTrbq9JiBQeE4MvXHkoOESq4Cp-cPif4fk2l4oDg,7446
-sycommon/tests/test_consumer_broker_health.py,sha256=VF8JCEINidU8W5-fA1u74mZuuRoyBavB7Dx3eXSEIHo,18573
+sycommon/tests/test_consumer_broker_health.py,sha256=2w_j5AHxHADYQ3YBYDh-Mxj88NxlRIM9UJj2t7jeE0s,18808
 sycommon/tests/test_consumer_loss_rootcause.py,sha256=rP_XdbGbzOM9RB36vk0gU4kqipXG9iHqrCN8rKyd2Aw,16294
 sycommon/tests/test_consumer_recovery.py,sha256=pMMtOwHIijNTZxg3IVBYSGFYgezMX0PUeylks8xzMM8,10364
 sycommon/tests/test_consumer_recovery_fix.py,sha256=q2OW0-KEsTqHpelNIDr15G8p-ZOC49rXGxq2wqBmFZM,12857
@@ -334,8 +334,8 @@ sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
 sycommon/tools/user_id.py,sha256=o-zsubGSR5rSSZf8Pk8B7AhE21ahDTTIW5JZmJO6Flw,4761
 sycommon/xxljob/__init__.py,sha256=7eoBlQxv-B39IfRSCY2bkqdGYs1QRe1umAWd88VMEEM,86
 sycommon/xxljob/xxljob_service.py,sha256=1yifwIBNGsCIxLnQjHKiBlbsigc_zvPH-dMTZcNxe-Q,7649
-sycommon_python_lib-0.2.8a10.dist-info/METADATA,sha256=p9DVU8Q5F4H4N-O2sIleQ_5cAq0fUMPS8uxSclEsZ4w,8033
-sycommon_python_lib-0.2.8a10.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-sycommon_python_lib-0.2.8a10.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
-sycommon_python_lib-0.2.8a10.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
-sycommon_python_lib-0.2.8a10.dist-info/RECORD,,
+sycommon_python_lib-0.2.8a12.dist-info/METADATA,sha256=AuGxUAW8tC-CIh63rj0YZf0PjwNRnry3-KpD4cyZJFU,8033
+sycommon_python_lib-0.2.8a12.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+sycommon_python_lib-0.2.8a12.dist-info/entry_points.txt,sha256=gsR4SssKxDWjRU8ggidzNcdMXDPRSKRS7UaGyNP84Qg,92
+sycommon_python_lib-0.2.8a12.dist-info/top_level.txt,sha256=RgphKrg7nJyZ7irJqbxFr-5H2LUYTvI7ivoWZH2hcD0,29
+sycommon_python_lib-0.2.8a12.dist-info/RECORD,,