aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/db/__init__.py

@@ -1,3 +1,14 @@
+"""
+Database management module for aioscrapy.
+aioscrapy的数据库管理模块。
+
+This module provides a unified interface for managing different types of database
+connections in aioscrapy. It supports Redis, MySQL, RabbitMQ, MongoDB, and PostgreSQL
+through their respective async libraries.
+此模块为aioscrapy中的不同类型数据库连接提供了统一的接口。它通过各自的异步库
+支持Redis、MySQL、RabbitMQ、MongoDB和PostgreSQL。
+"""
+
 from importlib import import_module
 from typing import Any
 
@@ -7,8 +18,12 @@ from aioscrapy.db.aioredis import redis_manager
 from aioscrapy.utils.log import logger
 from aioscrapy.utils.misc import load_object
 
+# Public API
+# 公共API
 __all__ = ['db_manager', 'get_pool', 'get_manager']
 
+# Mapping of database module names to their manager keys and class paths
+# 数据库模块名称到其管理器键和类路径的映射
 DB_MODULE_MAP = {
     'redis': ('redis', 'aioscrapy.db.aioredis.redis_manager'),
     'aiomysql': ('mysql', 'aioscrapy.db.aiomysql.mysql_manager'),
@@ -17,56 +32,190 @@ DB_MODULE_MAP = {
     'asyncpg': ('pg', 'aioscrapy.db.aiopg.pg_manager'),
 }
 
+# Dictionary to store available database managers
+# 存储可用数据库管理器的字典
 db_manager_map = {}
 
+# Dynamically load available database managers based on installed packages
+# 根据已安装的包动态加载可用的数据库管理器
 for module_name, (manager_key, class_path) in DB_MODULE_MAP.items():
     try:
+        # Try to import the module to check if it's available
+        # 尝试导入模块以检查它是否可用
         import_module(module_name)
     except ImportError:
+        # Skip if the module is not installed
+        # 如果模块未安装，则跳过
         pass
     else:
+        # Load the manager class and add it to the map
+        # 加载管理器类并将其添加到映射中
         db_manager_map[manager_key] = load_object(class_path)
 
 
 class DBManager:
+    """
+    Central manager for database connections in aioscrapy.
+    aioscrapy中数据库连接的中央管理器。
+
+    This class provides a unified interface for accessing different types of database
+    connections. It manages connection pools for various database types and provides
+    methods to initialize connections from settings or configuration dictionaries.
+    此类为访问不同类型的数据库连接提供了统一的接口。它管理各种数据库类型的连接池，
+    并提供从设置或配置字典初始化连接的方法。
+    """
 
     @staticmethod
     def get_manager(db_type: str) -> AbsDBPoolManager:
+        """
+        Get the database manager for a specific database type.
+        获取特定数据库类型的数据库管理器。
+
+        Args:
+            db_type: The type of database ('redis', 'mysql', 'rabbitmq', 'mongo', 'pg').
+                    数据库类型（'redis'、'mysql'、'rabbitmq'、'mongo'、'pg'）。
+
+        Returns:
+            AbsDBPoolManager: The database manager instance.
+                             数据库管理器实例。
+
+        Raises:
+            AssertionError: If the specified database type is not supported.
+                           如果不支持指定的数据库类型。
+        """
         manager = db_manager_map.get(db_type)
         assert manager is not None, f"Not support db type：{db_type}"
         return manager
 
     def get_pool(self, db_type: str, alias='default') -> Any:
+        """
+        Get a connection pool for a specific database type.
+        获取特定数据库类型的连接池。
+
+        Args:
+            db_type: The type of database ('redis', 'mysql', 'rabbitmq', 'mongo', 'pg').
+                    数据库类型（'redis'、'mysql'、'rabbitmq'、'mongo'、'pg'）。
+            alias: The alias of the connection pool, defaults to 'default'.
+                  连接池的别名，默认为'default'。
+
+        Returns:
+            Any: The connection pool instance.
+                连接池实例。
+        """
         manager = self.get_manager(db_type)
         return manager.get_pool(alias)
 
     @staticmethod
     async def close_all() -> None:
+        """
+        Close all database connections.
+        关闭所有数据库连接。
+
+        This method closes all connection pools for all database types.
+        此方法关闭所有数据库类型的所有连接池。
+
+        Returns:
+            None
+        """
         for manager in db_manager_map.values():
             await manager.close_all()
 
     @staticmethod
     async def from_dict(db_args: dict) -> None:
+        """
+        Initialize database connections from a configuration dictionary.
+        从配置字典初始化数据库连接。
+
+        Args:
+            db_args: A dictionary mapping database types to their configuration.
+                    将数据库类型映射到其配置的字典。
+                    Example:
+                    {
+                        'redis': {'default': {'host': 'localhost', 'port': 6379}},
+                        'mysql': {'default': {'host': 'localhost', 'user': 'root'}}
+                    }
+
+        Returns:
+            None
+        """
         for db_type, args in db_args.items():
             manager = db_manager_map.get(db_type)
             if manager is None:
                 logger.warning(f'Not support db type: {db_type}; Only {", ".join(db_manager_map.keys())} supported')
+                continue
             await manager.from_dict(args)
 
     @staticmethod
     async def from_settings(settings: aioscrapy.Settings) -> None:
+        """
+        Initialize database connections from aioscrapy settings.
+        从aioscrapy设置初始化数据库连接。
+
+        This method initializes all available database managers with the provided settings.
+        Each manager will extract its relevant settings from the settings object.
+        此方法使用提供的设置初始化所有可用的数据库管理器。
+        每个管理器将从设置对象中提取其相关设置。
+
+        Args:
+            settings: The aioscrapy settings object.
+                     aioscrapy设置对象。
+
+        Returns:
+            None
+        """
         for manager in db_manager_map.values():
             await manager.from_settings(settings)
 
     async def from_crawler(self, crawler: "aioscrapy.Crawler") -> None:
+        """
+        Initialize database connections from a crawler.
+        从爬虫初始化数据库连接。
+
+        This is a convenience method that extracts settings from the crawler
+        and calls from_settings.
+        这是一个便捷方法，它从爬虫中提取设置并调用from_settings。
+
+        Args:
+            crawler: The aioscrapy crawler instance.
+                    aioscrapy爬虫实例。
+
+        Returns:
+            None
+        """
         return await self.from_settings(crawler.settings)
 
     def __getattr__(self, db_type: str) -> Any:
+        """
+        Access a database manager directly as an attribute.
+        直接将数据库管理器作为属性访问。
+
+        This method allows accessing database managers using attribute syntax:
+        db_manager.redis, db_manager.mysql, etc.
+        此方法允许使用属性语法访问数据库管理器：
+        db_manager.redis、db_manager.mysql等。
+
+        Args:
+            db_type: The type of database ('redis', 'mysql', 'rabbitmq', 'mongo', 'pg').
+                    数据库类型（'redis'、'mysql'、'rabbitmq'、'mongo'、'pg'）。
+
+        Returns:
+            AbsDBPoolManager: The database manager instance.
+                             数据库管理器实例。
+
+        Raises:
+            AttributeError: If the specified database type is not supported.
+                           如果不支持指定的数据库类型。
+        """
         if db_type not in db_manager_map:
             raise AttributeError(f'Not support db type: {db_type}')
         return db_manager_map[db_type]
 
 
+# Singleton instance of DBManager
+# DBManager的单例实例
 db_manager = DBManager()
+
+# Convenience functions for accessing the singleton
+# 用于访问单例的便捷函数
 get_manager = db_manager.get_manager
 get_pool = db_manager.get_pool

aioscrapy/db/absmanager.py

@@ -1,47 +1,253 @@
+"""
+Abstract database pool manager for aioscrapy.
+aioscrapy的抽象数据库池管理器。
+
+This module defines the abstract base class for all database pool managers
+in aioscrapy. It specifies the interface that all database managers must implement.
+此模块定义了aioscrapy中所有数据库池管理器的抽象基类。
+它指定了所有数据库管理器必须实现的接口。
+"""
+
 from abc import ABCMeta, abstractmethod
 
 import aioscrapy
 
 
 class AbsDBPoolManager(object, metaclass=ABCMeta):
+    """
+    Abstract base class for database connection pool managers.
+    数据库连接池管理器的抽象基类。
+
+    This class defines the interface that all database pool managers must implement.
+    It provides methods for creating, accessing, and closing connection pools,
+    as well as initializing pools from settings or configuration dictionaries.
+    此类定义了所有数据库池管理器必须实现的接口。
+    它提供了创建、访问和关闭连接池的方法，以及从设置或配置字典初始化池的方法。
+    """
 
     @property
     @abstractmethod
     def _clients(self) -> dict:
+        """
+        Dictionary of connection pools managed by this manager.
+        此管理器管理的连接池字典。
+
+        This property should return a dictionary mapping aliases to connection pools.
+        Subclasses must implement this property.
+        此属性应返回将别名映射到连接池的字典。
+        子类必须实现此属性。
+
+        Returns:
+            dict: A dictionary mapping aliases to connection pools.
+                 将别名映射到连接池的字典。
+        """
         return {}
 
     @abstractmethod
     async def create(self, alias: str, params: dict):
-        """Create pool of connection"""
+        """
+        Create a new connection pool.
+        创建新的连接池。
+
+        This method creates a new connection pool with the given alias and parameters.
+        Subclasses must implement this method.
+        此方法使用给定的别名和参数创建新的连接池。
+        子类必须实现此方法。
+
+        Args:
+            alias: The alias for the new connection pool.
+                  新连接池的别名。
+            params: The parameters for creating the connection pool.
+                   创建连接池的参数。
+
+        Returns:
+            The created connection pool.
+            创建的连接池。
+        """
 
     @abstractmethod
     def get_pool(self, alias: str):
-        """Get pool named `alias`"""
+        """
+        Get a connection pool by its alias.
+        通过别名获取连接池。
+
+        This method retrieves an existing connection pool with the given alias.
+        Subclasses must implement this method.
+        此方法检索具有给定别名的现有连接池。
+        子类必须实现此方法。
+
+        Args:
+            alias: The alias of the connection pool to retrieve.
+                  要检索的连接池的别名。
+
+        Returns:
+            The connection pool with the given alias.
+            具有给定别名的连接池。
+
+        Raises:
+            KeyError: If no connection pool exists with the given alias.
+                     如果不存在具有给定别名的连接池。
+        """
 
     @abstractmethod
     async def close_all(self):
-        """Close all pool"""
+        """
+        Close all connection pools.
+        关闭所有连接池。
+
+        This method closes all connection pools managed by this manager.
+        Subclasses must implement this method.
+        此方法关闭此管理器管理的所有连接池。
+        子类必须实现此方法。
+
+        Returns:
+            None
+        """
 
     @abstractmethod
     async def close(self, alias: str):
-        """Close pool named `alias`"""
+        """
+        Close a specific connection pool.
+        关闭特定的连接池。
+
+        This method closes the connection pool with the given alias.
+        Subclasses must implement this method.
+        此方法关闭具有给定别名的连接池。
+        子类必须实现此方法。
+
+        Args:
+            alias: The alias of the connection pool to close.
+                  要关闭的连接池的别名。
+
+        Returns:
+            None
+
+        Raises:
+            KeyError: If no connection pool exists with the given alias.
+                     如果不存在具有给定别名的连接池。
+        """
 
     @abstractmethod
     async def from_dict(self, db_args: dict):
-        """Create pool with dict"""
+        """
+        Initialize connection pools from a configuration dictionary.
+        从配置字典初始化连接池。
+
+        This method creates connection pools based on the configuration in db_args.
+        Subclasses must implement this method.
+        此方法根据db_args中的配置创建连接池。
+        子类必须实现此方法。
+
+        Args:
+            db_args: A dictionary mapping aliases to connection parameters.
+                    将别名映射到连接参数的字典。
+                    Example:
+                    {
+                        'default': {'host': 'localhost', 'port': 6379},
+                        'cache': {'host': 'cache.example.com', 'port': 6379}
+                    }
+
+        Returns:
+            None
+        """
 
     @abstractmethod
     async def from_settings(self, settings: aioscrapy.Settings):
-        """Create pool with settings"""
+        """
+        Initialize connection pools from aioscrapy settings.
+        从aioscrapy设置初始化连接池。
+
+        This method creates connection pools based on the configuration in settings.
+        Subclasses must implement this method.
+        此方法根据settings中的配置创建连接池。
+        子类必须实现此方法。
+
+        Args:
+            settings: The aioscrapy settings object.
+                     aioscrapy设置对象。
+
+        Returns:
+            None
+        """
 
     async def from_crawler(self, crawler: "aioscrapy.Crawler"):
+        """
+        Initialize connection pools from a crawler.
+        从爬虫初始化连接池。
+
+        This is a convenience method that extracts settings from the crawler
+        and calls from_settings.
+        这是一个便捷方法，它从爬虫中提取设置并调用from_settings。
+
+        Args:
+            crawler: The aioscrapy crawler instance.
+                    aioscrapy爬虫实例。
+
+        Returns:
+            None
+        """
         return await self.from_settings(crawler.settings)
 
     def __call__(self, alias: str):
+        """
+        Make the manager callable to get a connection pool.
+        使管理器可调用以获取连接池。
+
+        This method allows using the manager as a function to get a connection pool:
+        manager('default')
+        此方法允许将管理器用作函数以获取连接池：
+        manager('default')
+
+        Args:
+            alias: The alias of the connection pool to retrieve.
+                  要检索的连接池的别名。
+
+        Returns:
+            The connection pool with the given alias.
+            具有给定别名的连接池。
+        """
         return self.get_pool(alias)
 
     def __getitem__(self, alias: str):
+        """
+        Allow dictionary-style access to connection pools.
+        允许字典样式访问连接池。
+
+        This method allows accessing connection pools using dictionary syntax:
+        manager['default']
+        此方法允许使用字典语法访问连接池：
+        manager['default']
+
+        Args:
+            alias: The alias of the connection pool to retrieve.
+                  要检索的连接池的别名。
+
+        Returns:
+            The connection pool with the given alias.
+            具有给定别名的连接池。
+        """
         return self.get_pool(alias)
 
     def __getattr__(self, alias: str):
+        """
+        Allow attribute-style access to connection pools.
+        允许属性样式访问连接池。
+
+        This method allows accessing connection pools using attribute syntax:
+        manager.default
+        此方法允许使用属性语法访问连接池：
+        manager.default
+
+        Args:
+            alias: The alias of the connection pool to retrieve.
+                  要检索的连接池的别名。
+
+        Returns:
+            The connection pool with the given alias.
+            具有给定别名的连接池。
+
+        Raises:
+            AttributeError: If no connection pool exists with the given alias.
+                           如果不存在具有给定别名的连接池。
+        """
         return self.get_pool(alias)