sycommon-python-lib 0.1.38__py3-none-any.whl → 0.1.40__py3-none-any.whl

sycommon/rabbitmq/rabbitmq_pool.py

@@ -1,7 +1,7 @@
-from typing import Optional, List
+import asyncio
+from typing import List, Set
 from aio_pika import connect_robust, Channel
 from aio_pika.abc import AbstractRobustConnection
-from aio_pika.pool import Pool
 
 from sycommon.logging.kafka_log import SYLogger
 
@@ -9,7 +9,7 @@ logger = SYLogger
 
 
 class RabbitMQConnectionPool:
-    """RabbitMQ连接池管理，负责创建和管理连接池与通道池"""
+    """RabbitMQ连接池管理（简化实现，避免上下文管理器冲突）"""
 
     def __init__(
         self,
@@ -33,72 +33,148 @@ class RabbitMQConnectionPool:
         self.app_name = app_name or "rabbitmq-client"
         self.heartbeat = heartbeat
 
-        # 连接池和通道池
-        self.connection_pool: Optional[Pool] = None
-        self.channel_pool: Optional[Pool] = None
+        # 连接池配置
         self.connection_pool_size = connection_pool_size
         self.channel_pool_size = channel_pool_size
 
+        # 实际存储的连接和通道
+        self._connections: List[AbstractRobustConnection] = []
+        self._free_channels: List[Channel] = []
+        self._used_channels: Set[Channel] = set()
+
+        # 锁用于线程安全
+        self._conn_lock = asyncio.Lock()
+        self._chan_lock = asyncio.Lock()
+
+        # 连接状态
+        self._initialized = False
+
     async def init_pools(self):
-        """初始化连接池和通道池"""
-        # 连接创建函数（支持集群节点轮询）
-        async def create_connection() -> AbstractRobustConnection:
-            # 轮询选择主机（简单负载均衡）
-            hosts = self.hosts.copy()
-            while hosts:
-                host = hosts.pop(0)
+        """初始化连接池（创建指定数量的连接）"""
+        if self._initialized:
+            logger.warning("连接池已初始化，无需重复调用")
+            return
+
+        try:
+            # 创建核心连接（数量=connection_pool_size）
+            for i in range(self.connection_pool_size):
+                conn = await self._create_connection()
+                self._connections.append(conn)
+                # 为每个连接创建初始通道（数量=channel_pool_size//connection_pool_size）
+                chan_count_per_conn = self.channel_pool_size // self.connection_pool_size
+                for _ in range(chan_count_per_conn):
+                    chan = await conn.channel()
+                    self._free_channels.append(chan)
+
+            self._initialized = True
+            logger.info(
+                f"RabbitMQ连接池初始化成功 - 连接数: {len(self._connections)}, "
+                f"空闲通道数: {len(self._free_channels)}, 集群节点: {self.hosts}"
+            )
+        except Exception as e:
+            logger.error(f"连接池初始化失败: {str(e)}", exc_info=True)
+            # 清理异常状态
+            await self.close()
+            raise
+
+    async def _create_connection(self) -> AbstractRobustConnection:
+        """创建单个RabbitMQ连接（支持集群节点轮询）"""
+        hosts = self.hosts.copy()
+        while hosts:
+            host = hosts.pop(0)
+            try:
+                connection = await connect_robust(
+                    host=host,
+                    port=self.port,
+                    login=self.username,
+                    password=self.password,
+                    virtualhost=self.virtualhost,
+                    heartbeat=self.heartbeat,
+                    client_properties={
+                        "connection_name": f"{self.app_name}@{host}"
+                    },
+                    reconnect_interval=2  # aio_pika 内置重连间隔
+                )
+                logger.info(f"成功连接到 RabbitMQ 节点: {host}:{self.port}")
+                return connection
+            except Exception as e:
+                logger.warning(
+                    f"连接主机 {host}:{self.port} 失败，尝试下一个节点: {str(e)}")
+                if not hosts:
+                    raise  # 所有节点失败时抛出异常
+
+    async def acquire_channel(self) -> Channel:
+        """获取通道（从空闲通道池获取，无则创建新通道）"""
+        if not self._initialized:
+            raise RuntimeError("连接池未初始化，请先调用 init_pools()")
+
+        async with self._chan_lock:
+            # 优先从空闲通道池获取
+            if self._free_channels:
+                channel = self._free_channels.pop()
+                # 检查通道是否有效
+                if not channel.is_closed:
+                    self._used_channels.add(channel)
+                    return channel
+                else:
+                    logger.warning("发现无效通道，已自动清理")
+
+            # 空闲通道不足，创建新通道（不超过最大限制）
+            if len(self._used_channels) < self.channel_pool_size:
+                # 选择一个空闲连接创建通道
+                async with self._conn_lock:
+                    for conn in self._connections:
+                        if not conn.is_closed:
+                            try:
+                                channel = await conn.channel()
+                                self._used_channels.add(channel)
+                                logger.info(
+                                    f"创建新通道，当前通道数: {len(self._used_channels)}/{self.channel_pool_size}")
+                                return channel
+                            except Exception as e:
+                                logger.warning(f"使用连接创建通道失败: {str(e)}")
+                    # 所有连接都无效，尝试重新创建连接
+                    conn = await self._create_connection()
+                    self._connections.append(conn)
+                    channel = await conn.channel()
+                    self._used_channels.add(channel)
+                    return channel
+            else:
+                raise RuntimeError(f"通道池已达最大限制: {self.channel_pool_size}")
+
+    async def release_channel(self, channel: Channel):
+        """释放通道（归还到空闲通道池）"""
+        async with self._chan_lock:
+            if channel in self._used_channels:
+                self._used_channels.remove(channel)
+                # 通道有效则归还，无效则丢弃
+                if not channel.is_closed:
+                    self._free_channels.append(channel)
+                else:
+                    logger.warning("释放无效通道，已自动丢弃")
+
+    async def close(self):
+        """关闭连接池（释放所有连接和通道）"""
+        # 释放所有通道
+        async with self._chan_lock:
+            for channel in self._free_channels + list(self._used_channels):
                 try:
-                    return await connect_robust(
-                        host=host,
-                        port=self.port,
-                        login=self.username,
-                        password=self.password,
-                        virtualhost=self.virtualhost,
-                        heartbeat=self.heartbeat,
-                        client_properties={
-                            "connection_name": f"{self.app_name}@{host}"
-                        }
-                    )
+                    if not channel.is_closed:
+                        await channel.close()
                 except Exception as e:
-                    logger.warning(
-                        f"连接主机 {host}:{self.port} 失败，尝试下一个节点: {str(e)}")
-                    if not hosts:
-                        raise  # 所有节点都失败时抛出异常
-
-        # 初始化连接池
-        self.connection_pool = Pool(
-            create_connection,
-            max_size=self.connection_pool_size
-        )
-
-        # 通道创建函数
-        async def create_channel() -> Channel:
-            async with self.connection_pool.acquire() as connection:
-                channel = await connection.channel()
-                return channel
-
-        # 初始化通道池
-        self.channel_pool = Pool(
-            create_channel,
-            max_size=self.channel_pool_size
-        )
-
-        logger.info(
-            f"RabbitMQ连接池初始化完成 - 连接池大小: {self.connection_pool_size}, "
-            f"通道池大小: {self.channel_pool_size}, 集群节点: {self.hosts}"
-        )
+                    logger.warning(f"关闭通道失败: {str(e)}")
+            self._free_channels.clear()
+            self._used_channels.clear()
 
-    async def close(self):
-        """关闭连接池和通道池"""
-        if self.channel_pool:
-            await self.channel_pool.close()
-        if self.connection_pool:
-            await self.connection_pool.close()
-        logger.info("RabbitMQ连接池已关闭")
-
-    async def __aenter__(self):
-        await self.init_pools()
-        return self
-
-    async def __aexit__(self, exc_type, exc, tb):
-        await self.close()
+        # 关闭所有连接
+        async with self._conn_lock:
+            for conn in self._connections:
+                try:
+                    if not conn.is_closed:
+                        await conn.close()
+                except Exception as e:
+                    logger.warning(f"关闭连接失败: {str(e)}")
+            self._connections.clear()
+
+        self._initialized = False
+        logger.info("RabbitMQ连接池已完全关闭")

sycommon/rabbitmq/rabbitmq_service.py

@@ -179,18 +179,14 @@ class RabbitMQService:
             auto_delete=kwargs.get('auto_delete', False),
             auto_parse_json=kwargs.get('auto_parse_json', True),
             create_if_not_exists=create_if_not_exists,
-            connection_timeout=kwargs.get('connection_timeout', 10),
             rpc_timeout=kwargs.get('rpc_timeout', 10),
-            reconnection_delay=kwargs.get('reconnection_delay', 1),
-            max_reconnection_attempts=kwargs.get(
-                'max_reconnection_attempts', 5),
             prefetch_count=kwargs.get('prefetch_count', 2),
             consumption_stall_threshold=kwargs.get(
                 'consumption_stall_threshold', 60),  # 延长停滞阈值
         )
 
-        # 使用declare_queue控制是否声明队列（发送器不声明，监听器声明）
-        await client.connect(declare_queue=not is_sender)
+        # 新客户端connect无declare_queue参数，自动根据create_if_not_exists处理
+        await client.connect()
 
         # 监听器客户端连接后延迟1秒，确保消费状态就绪（仅首次启动）
         if not is_sender and create_if_not_exists:
@@ -245,13 +241,13 @@ class RabbitMQService:
                         if not queue:
                             logger.info(f"客户端 '{client_name}' 队列未初始化，重新连接")
                             client.create_if_not_exists = True
-                            await client.connect(force_reconnect=True, declare_queue=True)
+                            await client.connect()  # 移除force_reconnect和declare_queue
                     return client
                 else:
                     logger.info(f"客户端 '{client_name}' 连接已关闭，重新连接")
                     if not is_sender:
                         client.create_if_not_exists = True
-                    await client.connect(declare_queue=not is_sender)
+                    await client.connect()  # 移除force_reconnect和declare_queue
                     return client
 
             # 创建新客户端
@@ -267,7 +263,7 @@ class RabbitMQService:
                     app_name=cls._config.get("APP_NAME", ""),
                     **kwargs
                 )
-                await client.connect(declare_queue=False)
+                await client.connect()  # 移除declare_queue=False
                 cls._clients[client_name] = client
                 return client
 
@@ -280,7 +276,7 @@ class RabbitMQService:
                 client = await cls._create_client(
                     initial_queue_name, ** kwargs
                 )
-                await client.connect(declare_queue=True)
+                await client.connect()  # 移除declare_queue=True
                 cls._clients[client_name] = client
                 return client
 
@@ -292,14 +288,14 @@ class RabbitMQService:
             )
 
             client.create_if_not_exists = True
-            await client.connect(declare_queue=True)
+            await client.connect()  # 移除declare_queue=True
 
             # 验证队列是否创建成功（异步获取队列）
             _, _, queue = await client._get_connection_resources()
             if not queue:
                 logger.error(f"队列 '{initial_queue_name}' 创建失败，尝试重新创建")
                 client.create_if_not_exists = True
-                await client.connect(force_reconnect=True, declare_queue=True)
+                await client.connect()
                 _, _, queue = await client._get_connection_resources()
                 if not queue:
                     raise Exception(f"无法创建队列 '{initial_queue_name}'")
@@ -348,7 +344,7 @@ class RabbitMQService:
                 if normalized_name in cls._clients:
                     client = cls._clients[normalized_name]
                     if not await client.is_connected:
-                        await client.connect(declare_queue=False)
+                        await client.connect()  # 移除declare_queue=False
                 else:
                     client = await cls.get_client(
                         client_name=normalized_name,
@@ -535,15 +531,19 @@ class RabbitMQService:
                 while attempt < max_attempts and not stop_event.is_set() and not cls._is_shutdown:
                     try:
                         # 启动消费前再次校验连接和队列状态
-                        if not await client.is_connected:
+                        if not await client.is_connected:  # 确保这里没有括号
                             logger.info(f"消费者 '{client_name}' 连接断开，尝试重连")
-                            await client.connect(force_reconnect=True, declare_queue=True)
+                            await client.connect()
 
                         # 确保队列已就绪
                         _, _, queue = await client._get_connection_resources()
                         if not queue:
                             raise Exception("队列未初始化完成")
 
+                        # 确保消息处理器已设置
+                        if not hasattr(client, '_message_handler') or not client._message_handler:
+                            raise Exception("消息处理器未设置")
+
                         consumer_tag = await client.start_consuming()
                         if consumer_tag:
                             break
@@ -552,7 +552,7 @@ class RabbitMQService:
                         logger.warning(
                             f"启动消费者尝试 {attempt}/{max_attempts} 失败: {str(e)}")
                         if attempt < max_attempts:
-                            await asyncio.sleep(1)
+                            await asyncio.sleep(2)  # 延长重试间隔
 
                 if cls._is_shutdown:
                     logger.info("服务关闭中，消费者启动中止")
@@ -644,7 +644,7 @@ class RabbitMQService:
             else:
                 logger.info(f"发送器 '{queue_name}' 连接已断开，尝试重连")
                 try:
-                    await client.connect(declare_queue=False)
+                    await client.connect()  # 移除declare_queue=False
                     if await client.is_connected:
                         return client
                 except Exception as e:
@@ -662,7 +662,7 @@ class RabbitMQService:
                 else:
                     logger.info(f"发送器 '{suffixed_name}' 连接已断开，尝试重连")
                     try:
-                        await client.connect(declare_queue=False)
+                        await client.connect()  # 移除declare_queue=False
                         if await client.is_connected:
                             return client
                     except Exception as e:
@@ -698,7 +698,7 @@ class RabbitMQService:
             while retry_count < max_retry and not cls._is_shutdown:
                 try:
                     # 尝试重连，每次重试间隔1秒
-                    await sender.connect(force_reconnect=True, declare_queue=False)
+                    await sender.connect()  # 移除force_reconnect和declare_queue
                     if await sender.is_connected:
                         logger.info(
                             f"发送器 '{queue_name}' 第 {retry_count + 1} 次重连成功")

{sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.1.38
+Version: 0.1.40
 Summary: Add your description here
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.1.38.dist-info → sycommon_python_lib-0.1.40.dist-info}/RECORD

@@ -35,9 +35,9 @@ sycommon/models/mqlistener_config.py,sha256=vXp2uMmd0XQ5B9noSRXWHewTy-juQ2y7IsWt
 sycommon/models/mqmsg_model.py,sha256=cxn0M5b0utQK6crMYmL-1waeGYHvK3AlGaRy23clqTE,277
 sycommon/models/mqsend_config.py,sha256=NQX9dc8PpuquMG36GCVhJe8omAW1KVXXqr6lSRU6D7I,268
 sycommon/models/sso_user.py,sha256=i1WAN6k5sPcPApQEdtjpWDy7VrzWLpOrOQewGLGoGIw,2702
-sycommon/rabbitmq/rabbitmq_client.py,sha256=eLfYuLSZuhMJ0Y-gn8dG1BBVvvKPrQLipAf1-Fub63I,42660
-sycommon/rabbitmq/rabbitmq_pool.py,sha256=_NMOO4CZy-I_anMqpzfYinz-8373_rg5FM9eqzdjGyU,3598
-sycommon/rabbitmq/rabbitmq_service.py,sha256=fylrwIBYJfR2D7o2k_WxAbK5ZNNFV__vMcLBrmXe8-k,36589
+sycommon/rabbitmq/rabbitmq_client.py,sha256=vyyfIWJ0b7tXZSK1m8fKERW6D-LQCrDvmjKmsPvOG6M,12141
+sycommon/rabbitmq/rabbitmq_pool.py,sha256=YmvvFO8xP2pNnwuyZ0vp28XnfTzPScIPnBMih34f7Wc,7390
+sycommon/rabbitmq/rabbitmq_service.py,sha256=LXoL7zm-1_iTts6S9CreEMJ66ExwRm9y4zUmcOqI1YM,36597
 sycommon/sse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sycommon/sse/event.py,sha256=k_rBJy23R7crtzQeetT0Q73D8o5-5p-eESGSs_BPOj0,2797
 sycommon/sse/sse.py,sha256=__CfWEcYxOxQ-HpLor4LTZ5hLWqw9-2X7CngqbVHsfw,10128
@@ -52,8 +52,8 @@ sycommon/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sycommon/tools/docs.py,sha256=OPj2ETheuWjXLyaXtaZPbwmJKfJaYXV5s4XMVAUNrms,1607
 sycommon/tools/snowflake.py,sha256=DdEj3T5r5OEvikp3puxqmmmz6BrggxomoSlnsRFb5dM,1174
 sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
-sycommon_python_lib-0.1.38.dist-info/METADATA,sha256=ttyk1WD6XP7AfWf4usymgNcm0ILuA_T5gijUTf3FCJ4,7037
-sycommon_python_lib-0.1.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sycommon_python_lib-0.1.38.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
-sycommon_python_lib-0.1.38.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
-sycommon_python_lib-0.1.38.dist-info/RECORD,,
+sycommon_python_lib-0.1.40.dist-info/METADATA,sha256=-9GpbSobbSHJj66au2M-0ThPFN5ynZEALO69lvLOClY,7037
+sycommon_python_lib-0.1.40.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sycommon_python_lib-0.1.40.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
+sycommon_python_lib-0.1.40.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
+sycommon_python_lib-0.1.40.dist-info/RECORD,,