sycommon-python-lib 0.1.16__py3-none-any.whl → 0.1.17__py3-none-any.whl

sycommon/rabbitmq/rabbitmq_pool.py

@@ -0,0 +1,104 @@
+from typing import Optional, List
+from aio_pika import connect_robust, Channel
+from aio_pika.abc import AbstractRobustConnection
+from aio_pika.pool import Pool
+
+from sycommon.logging.kafka_log import SYLogger
+
+logger = SYLogger
+
+
+class RabbitMQConnectionPool:
+    """RabbitMQ连接池管理，负责创建和管理连接池与通道池"""
+
+    def __init__(
+        self,
+        hosts: List[str],
+        port: int,
+        username: str,
+        password: str,
+        virtualhost: str = "/",
+        connection_pool_size: int = 2,
+        channel_pool_size: int = 10,
+        heartbeat: int = 10,
+        app_name: str = ""
+    ):
+        self.hosts = [host.strip() for host in hosts if host.strip()]
+        if not self.hosts:
+            raise ValueError("至少需要提供一个RabbitMQ主机地址")
+        self.port = port
+        self.username = username
+        self.password = password
+        self.virtualhost = virtualhost
+        self.app_name = app_name or "rabbitmq-client"
+        self.heartbeat = heartbeat
+
+        # 连接池和通道池
+        self.connection_pool: Optional[Pool] = None
+        self.channel_pool: Optional[Pool] = None
+        self.connection_pool_size = connection_pool_size
+        self.channel_pool_size = channel_pool_size
+
+    async def init_pools(self):
+        """初始化连接池和通道池"""
+        # 连接创建函数（支持集群节点轮询）
+        async def create_connection() -> AbstractRobustConnection:
+            # 轮询选择主机（简单负载均衡）
+            hosts = self.hosts.copy()
+            while hosts:
+                host = hosts.pop(0)
+                try:
+                    return await connect_robust(
+                        host=host,
+                        port=self.port,
+                        login=self.username,
+                        password=self.password,
+                        virtualhost=self.virtualhost,
+                        heartbeat=self.heartbeat,
+                        client_properties={
+                            "connection_name": f"{self.app_name}@{host}"
+                        }
+                    )
+                except Exception as e:
+                    logger.warning(
+                        f"连接主机 {host}:{self.port} 失败，尝试下一个节点: {str(e)}")
+                    if not hosts:
+                        raise  # 所有节点都失败时抛出异常
+
+        # 初始化连接池
+        self.connection_pool = Pool(
+            create_connection,
+            max_size=self.connection_pool_size
+        )
+
+        # 通道创建函数
+        async def create_channel() -> Channel:
+            async with self.connection_pool.acquire() as connection:
+                channel = await connection.channel()
+                return channel
+
+        # 初始化通道池
+        self.channel_pool = Pool(
+            create_channel,
+            max_size=self.channel_pool_size
+        )
+
+        logger.info(
+            f"RabbitMQ连接池初始化完成 - 连接池大小: {self.connection_pool_size}, "
+            f"通道池大小: {self.channel_pool_size}, 集群节点: {self.hosts}"
+        )
+
+    async def close(self):
+        """关闭连接池和通道池"""
+        if self.channel_pool:
+            await self.channel_pool.close()
+        if self.connection_pool:
+            await self.connection_pool.close()
+        logger.info("RabbitMQ连接池已关闭")
+
+    async def __aenter__(self):
+        await self.init_pools()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.close()

sycommon/rabbitmq/rabbitmq_service.py

@@ -1,5 +1,4 @@
 import asyncio
-import logging
 from typing import (
     Callable, Coroutine, Dict, List, Optional, Type, Union, Any, Set
 )
@@ -11,15 +10,14 @@ from sycommon.models.mqlistener_config import RabbitMQListenerConfig
 from sycommon.models.mqsend_config import RabbitMQSendConfig
 from sycommon.models.sso_user import SsoUser
 from sycommon.logging.kafka_log import SYLogger
-from .rabbitmq_client import RabbitMQClient
+from sycommon.rabbitmq.rabbitmq_client import RabbitMQClient, RabbitMQConnectionPool
 
 logger = SYLogger
 
 
 class RabbitMQService:
     """
-    RabbitMQ服务封装，管理多个客户端实例，提供发送和接收消息的高级接口
-    负责客户端的创建、配置、生命周期管理和错误处理
+    RabbitMQ服务封装，管理多个客户端实例，基于连接池实现资源复用
     """
 
     # 保存多个客户端实例
@@ -45,11 +43,13 @@ class RabbitMQService:
     _has_senders: bool = False
     # 消费启动超时设置
     CONSUMER_START_TIMEOUT = 30  # 30秒超时
+    # 连接池实例
+    _connection_pool: Optional[RabbitMQConnectionPool] = None
 
     @classmethod
     def init(cls, config: dict, has_listeners: bool = False, has_senders: bool = False) -> Type['RabbitMQService']:
         """
-        初始化RabbitMQ服务（支持集群配置）
+        初始化RabbitMQ服务（支持集群配置），同时创建连接池
         """
         from sycommon.synacos.nacos_service import NacosService
 
@@ -70,13 +70,63 @@ class RabbitMQService:
         cls._has_listeners = has_listeners
         cls._has_senders = has_senders
 
+        # 初始化连接池（在单独的异步方法中启动）
+        asyncio.create_task(cls._init_connection_pool())
+
         return cls
 
     @classmethod
-    async def _create_client(cls, mq_config: dict, queue_name: str, **kwargs) -> RabbitMQClient:
+    async def _init_connection_pool(cls):
+        """初始化连接池（异步操作）"""
+        if cls._connection_pool or not cls._config:
+            return
+
+        try:
+            # 解析集群节点
+            hosts_str = cls._config.get('host', "")
+            hosts_list = [host.strip()
+                          for host in hosts_str.split(',') if host.strip()]
+            if not hosts_list:
+                raise ValueError("RabbitMQ集群配置为空，请检查host参数")
+
+            # 创建连接池
+            cls._connection_pool = RabbitMQConnectionPool(
+                hosts=hosts_list,
+                port=cls._config.get('port', 5672),
+                username=cls._config.get('username', ""),
+                password=cls._config.get('password', ""),
+                virtualhost=cls._config.get('virtual-host', "/"),
+                connection_pool_size=cls._config.get(
+                    'connection_pool_size', 2),  # 连接池大小
+                channel_pool_size=cls._config.get(
+                    'channel_pool_size', 10),       # 通道池大小
+                heartbeat=cls._config.get('heartbeat', 10),
+                app_name=cls._config.get("APP_NAME", "")
+            )
+
+            # 初始化连接池
+            await cls._connection_pool.init_pools()
+            logger.info("RabbitMQ连接池初始化成功")
+
+        except Exception as e:
+            logger.error(f"RabbitMQ连接池初始化失败: {str(e)}", exc_info=True)
+            # 连接池初始化失败时重试
+            await asyncio.sleep(1)
+            asyncio.create_task(cls._init_connection_pool())
+
+    @classmethod
+    async def _create_client(cls, queue_name: str, **kwargs) -> RabbitMQClient:
         """
-        创建RabbitMQ客户端实例（支持集群节点列表）
+        创建RabbitMQ客户端实例（基于连接池）
         """
+        if not cls._connection_pool:
+            # 等待连接池初始化
+            start_time = asyncio.get_event_loop().time()
+            while not cls._connection_pool:
+                if asyncio.get_event_loop().time() - start_time > 30:
+                    raise TimeoutError("等待连接池初始化超时")
+                await asyncio.sleep(1)
+
         app_name = kwargs.get('app_name', cls._config.get(
             "APP_NAME", "")) if cls._config else ""
 
@@ -100,20 +150,9 @@ class RabbitMQService:
             f"允许创建: {create_if_not_exists}"
         )
 
-        # 将逗号分隔的host字符串拆分为集群节点列表
-        hosts_str = mq_config.get('host', "")
-        hosts_list = [host.strip()
-                      for host in hosts_str.split(',') if host.strip()]
-        if not hosts_list:
-            raise ValueError("RabbitMQ集群配置为空，请检查host参数")
-
         return RabbitMQClient(
-            hosts=hosts_list,
-            port=mq_config.get('port', 5672),
-            username=mq_config.get('username', ""),
-            password=mq_config.get('password', ""),
-            virtualhost=mq_config.get('virtual-host', "/"),
-            exchange_name=mq_config.get(
+            connection_pool=cls._connection_pool,  # 传入连接池实例
+            exchange_name=cls._config.get(
                 'exchange_name', "system.topic.exchange"),
             exchange_type=kwargs.get('exchange_type', "topic"),
             queue_name=processed_queue_name,
@@ -125,11 +164,9 @@ class RabbitMQService:
             create_if_not_exists=create_if_not_exists,
             connection_timeout=kwargs.get('connection_timeout', 10),
             rpc_timeout=kwargs.get('rpc_timeout', 5),
-            app_name=app_name,
             reconnection_delay=kwargs.get('reconnection_delay', 1),
             max_reconnection_attempts=kwargs.get(
                 'max_reconnection_attempts', 5),
-            heartbeat=kwargs.get('heartbeat', 10),
             prefetch_count=kwargs.get('prefetch_count', 2),
             consumption_stall_threshold=kwargs.get(
                 'consumption_stall_threshold', 10)
@@ -141,15 +178,19 @@ class RabbitMQService:
         client_name: str = "default", ** kwargs
     ) -> RabbitMQClient:
         """
-        获取或创建RabbitMQ客户端
-
-        :param client_name: 客户端名称
-        :param kwargs: 客户端参数
-        :return: RabbitMQClient实例
+        获取或创建RabbitMQ客户端（基于连接池）
         """
         if not cls._config:
             raise ValueError("RabbitMQService尚未初始化，请先调用init方法")
 
+        # 等待连接池就绪
+        if not cls._connection_pool:
+            start_time = asyncio.get_event_loop().time()
+            while not cls._connection_pool:
+                if asyncio.get_event_loop().time() - start_time > 30:
+                    raise TimeoutError("等待连接池初始化超时")
+                await asyncio.sleep(1)
+
         # 确保锁存在
         if client_name not in cls._init_locks:
             cls._init_locks[client_name] = asyncio.Lock()
@@ -184,7 +225,6 @@ class RabbitMQService:
             if is_sender:
                 kwargs['create_if_not_exists'] = False
                 client = await cls._create_client(
-                    cls._config,
                     initial_queue_name,
                     app_name=cls._config.get("APP_NAME", ""),
                     **kwargs
@@ -200,7 +240,6 @@ class RabbitMQService:
             if initial_queue_name in cls._initialized_queues:
                 logger.debug(f"队列 '{initial_queue_name}' 已初始化，直接创建客户端")
                 client = await cls._create_client(
-                    cls._config,
                     initial_queue_name, ** kwargs
                 )
                 await client.connect(declare_queue=True)
@@ -209,7 +248,6 @@ class RabbitMQService:
 
             # 创建并连接客户端
             client = await cls._create_client(
-                cls._config,
                 initial_queue_name,
                 app_name=cls._config.get("APP_NAME", ""),
                 **kwargs
@@ -234,16 +272,11 @@ class RabbitMQService:
             cls._clients[client_name] = client
             return client
 
+    # 以下方法逻辑与原有保持一致（无需修改）
     @classmethod
     async def setup_senders(cls, senders: List[RabbitMQSendConfig], has_listeners: bool = False) -> None:
-        """
-        设置消息发送器
-
-        :param senders: 发送器配置列表
-        :param has_listeners: 是否同时存在监听器
-        """
+        """设置消息发送器"""
         cls._has_senders = True
-        # 保存监听器存在状态
         cls._has_listeners = has_listeners
         logger.info(f"开始设置 {len(senders)} 个消息发送器")
 
@@ -300,14 +333,8 @@ class RabbitMQService:
 
     @classmethod
     async def setup_listeners(cls, listeners: List[RabbitMQListenerConfig], has_senders: bool = False) -> None:
-        """
-        设置消息监听器
-
-        :param listeners: 监听器配置列表
-        :param has_senders: 是否同时存在发送器
-        """
+        """设置消息监听器"""
         cls._has_listeners = True
-        # 保存发送器存在状态
         cls._has_senders = has_senders
         logger.info(f"开始设置 {len(listeners)} 个消息监听器")
 
@@ -337,16 +364,11 @@ class RabbitMQService:
 
     @classmethod
     async def _verify_consumers_started(cls, timeout: int = 30) -> None:
-        """
-        验证消费者是否成功启动
-
-        :param timeout: 超时时间(秒)
-        """
+        """验证消费者是否成功启动"""
         start_time = asyncio.get_event_loop().time()
         required_clients = list(cls._message_handlers.keys())
         running_clients = []
 
-        # 等待所有消费者启动或超时
         while len(running_clients) < len(required_clients) and \
                 (asyncio.get_event_loop().time() - start_time) < timeout:
 
@@ -357,14 +379,12 @@ class RabbitMQService:
 
             logger.info(
                 f"消费者启动验证: {len(running_clients)}/{len(required_clients)} 已启动")
-            await asyncio.sleep(2)
+            await asyncio.sleep(1)
 
-        # 检查未成功启动的消费者
         failed_clients = [
             name for name in required_clients if name not in running_clients]
         if failed_clients:
             logger.error(f"以下消费者启动失败: {', '.join(failed_clients)}")
-            # 尝试重新启动失败的消费者
             for client_name in failed_clients:
                 logger.info(f"尝试重新启动消费者: {client_name}")
                 asyncio.create_task(cls.start_consumer(client_name))
@@ -375,13 +395,7 @@ class RabbitMQService:
         queue_name: str,
         handler: Callable[[MQMsgModel, AbstractIncomingMessage], Coroutine[Any, Any, None]], ** kwargs
     ) -> None:
-        """
-        添加消息监听器
-
-        :param queue_name: 队列名称
-        :param handler: 消息处理函数
-        :param kwargs: 其他参数
-        """
+        """添加消息监听器"""
         if not cls._config:
             raise ValueError("RabbitMQService尚未初始化，请先调用init方法")
 
@@ -408,11 +422,7 @@ class RabbitMQService:
 
     @classmethod
     async def start_consumer(cls, client_name: str) -> None:
-        """
-        启动指定客户端的消费者
-
-        :param client_name: 客户端名称
-        """
+        """启动指定客户端的消费者"""
         if client_name in cls._consumer_tasks and not cls._consumer_tasks[client_name].done():
             logger.debug(f"消费者 '{client_name}' 已在运行中，无需重复启动")
             return
@@ -461,7 +471,7 @@ class RabbitMQService:
                         logger.warning(
                             f"启动消费者尝试 {attempt}/{max_attempts} 失败: {str(e)}")
                         if attempt < max_attempts:
-                            await asyncio.sleep(2)
+                            await asyncio.sleep(1)
 
                 if not consumer_tag:
                     raise Exception(f"经过 {max_attempts} 次尝试仍无法启动消费者")
@@ -528,12 +538,7 @@ class RabbitMQService:
 
     @classmethod
     def get_sender(cls, queue_name: str) -> Optional[RabbitMQClient]:
-        """
-        获取发送客户端
-
-        :param queue_name: 队列名称
-        :return: RabbitMQClient实例或None
-        """
+        """获取发送客户端"""
         if not queue_name:
             logger.warning("发送器名称不能为空")
             return None
@@ -556,13 +561,7 @@ class RabbitMQService:
         data: Union[BaseModel, str, Dict[str, Any], None],
         queue_name: str, ** kwargs
     ) -> None:
-        """
-        发送消息到指定队列
-
-        :param data: 消息数据
-        :param queue_name: 队列名称
-        :param kwargs: 其他参数
-        """
+        """发送消息到指定队列"""
         # 获取发送客户端
         sender = cls.get_sender(queue_name)
         if not sender:
@@ -628,11 +627,7 @@ class RabbitMQService:
 
     @classmethod
     async def shutdown(cls, timeout: float = 10.0) -> None:
-        """
-        优雅关闭所有资源
-
-        :param timeout: 超时时间(秒)
-        """
+        """优雅关闭所有资源（新增连接池关闭逻辑）"""
         start_time = asyncio.get_event_loop().time()
         logger.info("开始关闭RabbitMQ服务...")
 
@@ -646,7 +641,6 @@ class RabbitMQService:
             0.0, timeout - (asyncio.get_event_loop().time() - start_time))
         if remaining_time > 0 and cls._consumer_tasks:
             try:
-                # 等待所有消费者任务完成或超时
                 done, pending = await asyncio.wait(
                     list(cls._consumer_tasks.values()),
                     timeout=remaining_time,
@@ -680,6 +674,14 @@ class RabbitMQService:
                     logger.warning(f"关闭客户端 '{name}' 时出错: {str(e)}")
                 logger.info(f"客户端 '{name}' 已关闭")
 
+        # 关闭连接池
+        if cls._connection_pool:
+            try:
+                await cls._connection_pool.close()
+                logger.info("RabbitMQ连接池已关闭")
+            except Exception as e:
+                logger.warning(f"关闭连接池时出错: {str(e)}")
+
         # 清理所有状态
         cls._clients.clear()
         cls._consumer_tasks.clear()
@@ -691,5 +693,6 @@ class RabbitMQService:
         cls._init_locks.clear()
         cls._has_listeners = False
         cls._has_senders = False
+        cls._connection_pool = None
 
         logger.info("RabbitMQ服务已完全关闭")

{sycommon_python_lib-0.1.16.dist-info → sycommon_python_lib-0.1.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sycommon-python-lib
-Version: 0.1.16
+Version: 0.1.17
 Summary: Add your description here
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{sycommon_python_lib-0.1.16.dist-info → sycommon_python_lib-0.1.17.dist-info}/RECORD

@@ -33,8 +33,9 @@ sycommon/models/mqlistener_config.py,sha256=PPwhAVJ2AWvVAvNox_1t0fuBKTyRH3Ui9cuu
 sycommon/models/mqmsg_model.py,sha256=cxn0M5b0utQK6crMYmL-1waeGYHvK3AlGaRy23clqTE,277
 sycommon/models/mqsend_config.py,sha256=NQX9dc8PpuquMG36GCVhJe8omAW1KVXXqr6lSRU6D7I,268
 sycommon/models/sso_user.py,sha256=i1WAN6k5sPcPApQEdtjpWDy7VrzWLpOrOQewGLGoGIw,2702
-sycommon/rabbitmq/rabbitmq_client.py,sha256=BJa5_CkBzI7LdZL7ozs4zc2qVgIVV5Oqg3AxVg3qiNM,37165
-sycommon/rabbitmq/rabbitmq_service.py,sha256=pBApkWWcLyvwuk7av_8owtJVXjsJ28Mb9et8nzkxYGQ,27794
+sycommon/rabbitmq/rabbitmq_client.py,sha256=1mnccWeqBHHTPy__8kGEBX77UtAPL3_ffVoAZbbyPf4,26429
+sycommon/rabbitmq/rabbitmq_pool.py,sha256=_NMOO4CZy-I_anMqpzfYinz-8373_rg5FM9eqzdjGyU,3598
+sycommon/rabbitmq/rabbitmq_service.py,sha256=lpYSavAwl-UOmz5gTgS3yj2ulfF-s7boL3lsMibN-JM,28881
 sycommon/sse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sycommon/sse/event.py,sha256=k_rBJy23R7crtzQeetT0Q73D8o5-5p-eESGSs_BPOj0,2797
 sycommon/sse/sse.py,sha256=__CfWEcYxOxQ-HpLor4LTZ5hLWqw9-2X7CngqbVHsfw,10128
@@ -45,8 +46,8 @@ sycommon/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sycommon/tools/docs.py,sha256=OPj2ETheuWjXLyaXtaZPbwmJKfJaYXV5s4XMVAUNrms,1607
 sycommon/tools/snowflake.py,sha256=rc-VUjBMMpdAvbnHroVwfVt1xzApJbTCthUy9mglAuw,237
 sycommon/tools/timing.py,sha256=OiiE7P07lRoMzX9kzb8sZU9cDb0zNnqIlY5pWqHcnkY,2064
-sycommon_python_lib-0.1.16.dist-info/METADATA,sha256=Ng_7vxoqZThJVeSOmS2AMsNbgtfKJTvIcn2qqbUvYzc,7005
-sycommon_python_lib-0.1.16.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sycommon_python_lib-0.1.16.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
-sycommon_python_lib-0.1.16.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
-sycommon_python_lib-0.1.16.dist-info/RECORD,,
+sycommon_python_lib-0.1.17.dist-info/METADATA,sha256=Yz-MTXIYKGwCFWX3SRSsNQpSiPXZ3vAhsSLig9fl890,7005
+sycommon_python_lib-0.1.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sycommon_python_lib-0.1.17.dist-info/entry_points.txt,sha256=q_h2nbvhhmdnsOUZEIwpuoDjaNfBF9XqppDEmQn9d_A,46
+sycommon_python_lib-0.1.17.dist-info/top_level.txt,sha256=98CJ-cyM2WIKxLz-Pf0AitWLhJyrfXvyY8slwjTXNuc,17
+sycommon_python_lib-0.1.17.dist-info/RECORD,,