aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/db/aioredis.py

@@ -1,3 +1,14 @@
+"""
+Redis connection pool manager for aioscrapy.
+aioscrapy的Redis连接池管理器。
+
+This module provides classes for managing Redis connection pools in aioscrapy.
+It includes a pool manager for creating and managing Redis clients, and an executor
+for convenient access to Redis commands.
+此模块提供了在aioscrapy中管理Redis连接池的类。
+它包括一个用于创建和管理Redis客户端的池管理器，以及一个用于方便访问Redis命令的执行器。
+"""
+
 from redis.asyncio import BlockingConnectionPool, Redis
 
 import aioscrapy
@@ -5,90 +16,328 @@ from aioscrapy.db.absmanager import AbsDBPoolManager
 
 
 class RedisExecutor:
+    """
+    Executor for Redis commands.
+    Redis命令的执行器。
+
+    This class provides a convenient way to execute Redis commands on a specific
+    Redis client. It dynamically forwards command calls to the underlying Redis client.
+    此类提供了一种在特定Redis客户端上执行Redis命令的便捷方式。
+    它动态地将命令调用转发到底层Redis客户端。
+    """
 
     def __init__(self, alias: str, pool_manager: "AioRedisPoolManager"):
+        """
+        Initialize a RedisExecutor.
+        初始化RedisExecutor。
+
+        Args:
+            alias: The alias of the Redis client to use.
+                  要使用的Redis客户端的别名。
+            pool_manager: The Redis pool manager that manages the Redis client.
+                         管理Redis客户端的Redis池管理器。
+        """
         self.alias = alias
         self.pool_manager = pool_manager
 
     def __getattr__(self, command: str):
+        """
+        Dynamically forward command calls to the Redis client.
+        动态地将命令调用转发到Redis客户端。
+
+        This method allows calling Redis commands directly on the executor:
+        executor.get('key'), executor.set('key', 'value'), etc.
+        此方法允许直接在执行器上调用Redis命令：
+        executor.get('key')、executor.set('key', 'value')等。
+
+        Args:
+            command: The Redis command to execute.
+                    要执行的Redis命令。
+
+        Returns:
+            The method of the Redis client corresponding to the command.
+            对应于命令的Redis客户端的方法。
+        """
         redis_pool: Redis = self.pool_manager.get_pool(self.alias)
         return getattr(redis_pool, command)
 
 
 class AioRedisPoolManager(AbsDBPoolManager):
+    """
+    Pool manager for Redis connections.
+    Redis连接的池管理器。
+
+    This class manages Redis connection pools and clients. It implements the
+    AbsDBPoolManager interface for Redis connections, providing methods for
+    creating, accessing, and closing Redis clients.
+    此类管理Redis连接池和客户端。它为Redis连接实现了AbsDBPoolManager接口，
+    提供了创建、访问和关闭Redis客户端的方法。
+    """
+
+    # Dictionary to store Redis clients by alias
+    # 按别名存储Redis客户端的字典
     _clients = {}
 
     async def create(self, alias: str, params: dict) -> Redis:
-        """Create Redis client"""
+        """
+        Create a new Redis client.
+        创建新的Redis客户端。
+
+        This method creates a new Redis client with the given alias and parameters.
+        If a client with the given alias already exists, it returns the existing client.
+        此方法使用给定的别名和参数创建新的Redis客户端。
+        如果具有给定别名的客户端已经存在，则返回现有客户端。
+
+        Args:
+            alias: The alias for the new Redis client.
+                  新Redis客户端的别名。
+            params: The parameters for creating the Redis client. Can include:
+                   创建Redis客户端的参数。可以包括：
+                   - url: Redis connection URL (e.g., 'redis://user:password@host:port/db')
+                          Redis连接URL（例如，'redis://user:password@host:port/db'）
+                   - host: Redis server host
+                           Redis服务器主机
+                   - port: Redis server port
+                           Redis服务器端口
+                   - db: Redis database number
+                         Redis数据库编号
+                   - password: Redis server password
+                              Redis服务器密码
+                   - socket_connect_timeout: Connection timeout in seconds
+                                           连接超时（秒）
+                   - and other parameters accepted by BlockingConnectionPool
+                     以及BlockingConnectionPool接受的其他参数
+
+        Returns:
+            Redis: The created or existing Redis client.
+                  创建的或现有的Redis客户端。
+        """
+        # Return existing client if it exists
+        # 如果客户端已存在，则返回现有客户端
         if alias in self._clients:
             return self._clients[alias]
 
+        # Make a copy of params to avoid modifying the original
+        # 复制params以避免修改原始参数
         params = params.copy()
+
+        # Extract URL if provided
+        # 如果提供了URL，则提取它
         url = params.pop('url', None)
+
+        # Set default connection timeout
+        # 设置默认连接超时
         params.setdefault('socket_connect_timeout', 30)
+
+        # Create connection pool from URL or parameters
+        # 从URL或参数创建连接池
         if url:
             connection_pool = BlockingConnectionPool.from_url(url, **params)
         else:
             connection_pool = BlockingConnectionPool(**params)
+
+        # Create Redis client with the connection pool
+        # 使用连接池创建Redis客户端
         redis = Redis(connection_pool=connection_pool)
+
+        # Store and return the client
+        # 存储并返回客户端
         return self._clients.setdefault(alias, redis)
 
-    def get_pool(self, alias: str):
-        """Get redis client"""
+    def get_pool(self, alias: str) -> Redis:
+        """
+        Get a Redis client by its alias.
+        通过别名获取Redis客户端。
+
+        This method retrieves an existing Redis client with the given alias.
+        此方法检索具有给定别名的现有Redis客户端。
+
+        Args:
+            alias: The alias of the Redis client to retrieve.
+                  要检索的Redis客户端的别名。
+
+        Returns:
+            Redis: The Redis client with the given alias.
+                  具有给定别名的Redis客户端。
+
+        Raises:
+            AssertionError: If no Redis client exists with the given alias.
+                           如果不存在具有给定别名的Redis客户端。
+        """
         redis_pool: Redis = self._clients.get(alias)
         assert redis_pool is not None, f"Dont create the redis client named {alias}"
         return redis_pool
 
     def executor(self, alias: str) -> RedisExecutor:
-        """Get RedisExecutor"""
+        """
+        Get a RedisExecutor for a specific Redis client.
+        获取特定Redis客户端的RedisExecutor。
+
+        This method creates a RedisExecutor that provides a convenient way to
+        execute commands on the Redis client with the given alias.
+        此方法创建一个RedisExecutor，提供了一种在具有给定别名的Redis客户端上
+        执行命令的便捷方式。
+
+        Args:
+            alias: The alias of the Redis client to use.
+                  要使用的Redis客户端的别名。
+
+        Returns:
+            RedisExecutor: An executor for the Redis client.
+                          Redis客户端的执行器。
+        """
         return RedisExecutor(alias, self)
 
     async def close(self, alias: str):
-        """Close redis pool named `alias`"""
+        """
+        Close a specific Redis client.
+        关闭特定的Redis客户端。
+
+        This method closes the Redis client with the given alias and removes it
+        from the managed clients.
+        此方法关闭具有给定别名的Redis客户端，并将其从管理的客户端中移除。
+
+        Args:
+            alias: The alias of the Redis client to close.
+                  要关闭的Redis客户端的别名。
+
+        Returns:
+            None
+        """
+        # Remove the client from the managed clients
+        # 从管理的客户端中移除客户端
         redis = self._clients.pop(alias, None)
+
+        # Close the client if it exists
+        # 如果客户端存在，则关闭它
         if redis:
+            # Close the Redis client
+            # 关闭Redis客户端
             await redis.close()
+
+            # Disconnect the connection pool
+            # 断开连接池
             await redis.connection_pool.disconnect()
 
     async def close_all(self):
-        """Close all clients of redis"""
+        """
+        Close all Redis clients.
+        关闭所有Redis客户端。
+
+        This method closes all Redis clients managed by this manager.
+        此方法关闭此管理器管理的所有Redis客户端。
+
+        Returns:
+            None
+        """
+        # Create a copy of the keys to avoid modifying the dictionary during iteration
+        # 创建键的副本，以避免在迭代期间修改字典
         for alias in list(self._clients.keys()):
             await self.close(alias)
 
     async def from_dict(self, db_args: dict):
-        """Create redis with dict"""
+        """
+        Initialize Redis clients from a configuration dictionary.
+        从配置字典初始化Redis客户端。
+
+        This method creates Redis clients based on the configuration in db_args.
+        此方法根据db_args中的配置创建Redis客户端。
+
+        Args:
+            db_args: A dictionary mapping aliases to Redis connection parameters.
+                    将别名映射到Redis连接参数的字典。
+                    Example:
+                    {
+                        'default': {'url': 'redis://localhost:6379/0'},
+                        'cache': {'host': 'cache.example.com', 'port': 6379, 'db': 1}
+                    }
+
+        Returns:
+            None
+        """
         for alias, redis_args in db_args.items():
             await self.create(alias, redis_args)
 
     async def from_settings(self, settings: aioscrapy.Settings):
-        """Create redis with settings"""
+        """
+        Initialize Redis clients from aioscrapy settings.
+        从aioscrapy设置初始化Redis客户端。
+
+        This method creates Redis clients based on the REDIS_ARGS setting.
+        此方法根据REDIS_ARGS设置创建Redis客户端。
+
+        The REDIS_ARGS setting should be a dictionary mapping aliases to Redis
+        connection parameters, for example:
+        REDIS_ARGS设置应该是一个将别名映射到Redis连接参数的字典，例如：
+
+        ```python
+        REDIS_ARGS = {
+            'default': {'url': 'redis://localhost:6379/0'},
+            'cache': {'host': 'cache.example.com', 'port': 6379, 'db': 1}
+        }
+        ```
+
+        Args:
+            settings: The aioscrapy settings object.
+                     aioscrapy设置对象。
+
+        Returns:
+            None
+        """
         for alias, redis_args in settings.getdict('REDIS_ARGS').items():
             await self.create(alias, redis_args)
 
 
+# Singleton instance of AioRedisPoolManager
+# AioRedisPoolManager的单例实例
 redis_manager = AioRedisPoolManager()
 
+# Example usage
+# 示例用法
 if __name__ == '__main__':
     import asyncio
 
 
     async def test():
+        """
+        Test function demonstrating the usage of the Redis manager.
+        演示Redis管理器用法的测试函数。
+        """
+        # Create a Redis client with alias 'default'
+        # 创建别名为'default'的Redis客户端
         await redis_manager.create('default', {
             'url': 'redis://@192.168.234.128:6379/9',
         })
+
+        # Get a Redis executor for the 'default' client
+        # 获取'default'客户端的Redis执行器
         redis = redis_manager.executor('default')
+
+        # Add a value to a sorted set
+        # 向有序集合添加一个值
         print(await redis.zadd('key1', {'value': 2}))
 
+        # Use a pipeline to execute multiple commands atomically
+        # 使用管道原子地执行多个命令
         async with redis.pipeline(transaction=True) as pipe:
-            results, count = await (
+            # Get the first element and remove it in one transaction
+            # 在一个事务中获取第一个元素并删除它
+            results, _ = await (
                 pipe.zrange('key1', 0, 0)
                     .zremrangebyrank('key1', 0, 0)
                     .execute()
             )
 
+        # Print the results
+        # 打印结果
         print(results)
+
+        # Close all Redis clients
+        # 关闭所有Redis客户端
         await redis_manager.close_all()
 
 
-    # asyncio.run(test())
-    asyncio.get_event_loop().run_until_complete(test())
+    # Run the test function
+    # 运行测试函数
+    # asyncio.run(test())  # For Python 3.7+
+    asyncio.get_event_loop().run_until_complete(test())  # For Python 3.6

aioscrapy/dupefilters/__init__.py

@@ -1,3 +1,14 @@
+"""
+Duplicate Filter Base Module for AioScrapy
+AioScrapy的重复过滤器基础模块
+
+This module provides the abstract base class for duplicate filters in AioScrapy.
+Duplicate filters are used to avoid crawling the same URL multiple times by
+tracking request fingerprints.
+此模块提供了AioScrapy中重复过滤器的抽象基类。
+重复过滤器用于通过跟踪请求指纹来避免多次爬取相同的URL。
+"""
+
 from typing import Literal
 from abc import ABCMeta, abstractmethod
 
@@ -6,30 +17,104 @@ from aioscrapy.utils.log import logger
 
 
 class DupeFilterBase(metaclass=ABCMeta):
-    """Request Fingerprint duplicates filter"""
+    """
+    Abstract base class for request fingerprint duplicate filters.
+    请求指纹重复过滤器的抽象基类。
+
+    This class defines the interface that all duplicate filters must implement.
+    Duplicate filters are used to avoid crawling the same URL multiple times by
+    tracking request fingerprints.
+    此类定义了所有重复过滤器必须实现的接口。
+    重复过滤器用于通过跟踪请求指纹来避免多次爬取相同的URL。
+    """
 
     @classmethod
     @abstractmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
-        """ Get Instance of RFPDupeFilter from crawler """
+        """
+        Create a duplicate filter instance from a crawler.
+        从爬虫创建重复过滤器实例。
+
+        This is the factory method used by AioScrapy to create the dupefilter.
+        这是AioScrapy用于创建重复过滤器的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this dupefilter.
+                    将使用此重复过滤器的爬虫。
+
+        Returns:
+            DupeFilterBase: A new dupefilter instance.
+                           一个新的重复过滤器实例。
+        """
+        pass
 
     @abstractmethod
     async def request_seen(self, request: Request) -> bool:
-        """ Check whether fingerprint of request exists """
+        """
+        Check if a request has been seen before.
+        检查请求是否已经被看到过。
+
+        This method checks if the request's fingerprint is in the set of seen
+        fingerprints. If it is, the request is considered a duplicate.
+        此方法检查请求的指纹是否在已见过的指纹集合中。如果是，则认为请求是重复的。
+
+        Args:
+            request: The request to check.
+                    要检查的请求。
+
+        Returns:
+            bool: True if the request has been seen before, False otherwise.
+                 如果请求之前已经被看到过，则为True，否则为False。
+        """
+        pass
 
     @abstractmethod
     async def close(self, reason: str = '') -> None:
-        """ Delete data on close """
+        """
+        Close the dupefilter.
+        关闭过滤器。
+
+        This method is called when the spider is closed. It should clean up
+        any resources used by the dupefilter.
+        当爬虫关闭时调用此方法。它应该清理重复过滤器使用的任何资源。
+
+        Args:
+            reason: The reason why the spider was closed.
+                   爬虫被关闭的原因。
+        """
+        pass
 
     def log(self, request: Request, spider: Spider):
+        """
+        Log a filtered duplicate request.
+        记录被过滤的重复请求。
+
+        This method logs information about duplicate requests based on the
+        logging settings (info, debug, logdupes). It also increments the
+        dupefilter/filtered stats counter.
+        此方法根据日志设置（info、debug、logdupes）记录有关重复请求的信息。
+        它还增加dupefilter/filtered统计计数器。
+
+        Args:
+            request: The duplicate request that was filtered.
+                    被过滤的重复请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+        """
+        # Log at INFO level if info is True
+        # 如果info为True，则在INFO级别记录
         if self.info:
             logger.info("Filtered duplicate request: %(request)s" % {
                 'request': request.meta.get('dupefilter_msg') or request
             })
+        # Log at DEBUG level if debug is True
+        # 如果debug为True，则在DEBUG级别记录
         elif self.debug:
             logger.debug("Filtered duplicate request: %(request)s" % {
                 'request': request.meta.get('dupefilter_msg') or request
             })
+        # Log the first duplicate at DEBUG level and disable further logging
+        # 在DEBUG级别记录第一个重复项并禁用进一步的日志记录
         elif self.logdupes:
             msg = ("Filtered duplicate request: %(request)s"
                    " - no more duplicates will be shown"
@@ -37,6 +122,8 @@ class DupeFilterBase(metaclass=ABCMeta):
             logger.debug(msg % {'request': request.meta.get('dupefilter_msg') or request})
             self.logdupes = False
 
+        # Increment the dupefilter/filtered stats counter
+        # 增加dupefilter/filtered统计计数器
         spider.crawler.stats.inc_value('dupefilter/filtered', spider=spider)
 
     async def done(
@@ -44,4 +131,22 @@ class DupeFilterBase(metaclass=ABCMeta):
             request: Request,
             done_type: Literal["request_ok", "request_err", "parse_ok", "parse_err"]
     ) -> None:
-        """ 根据done_type的状态 控制指纹的移除 """
+        """
+        Control the removal of fingerprints based on the done_type status.
+        根据done_type的状态控制指纹的移除。
+
+        This method can be implemented by subclasses to handle the removal of
+        fingerprints from the filter based on the status of the request processing.
+        子类可以实现此方法，以根据请求处理的状态处理从过滤器中移除指纹。
+
+        Args:
+            request: The request that has been processed.
+                    已处理的请求。
+            done_type: The status of the request processing.
+                      请求处理的状态。
+                      Can be one of: "request_ok", "request_err", "parse_ok", "parse_err".
+                      可以是以下之一："request_ok"、"request_err"、"parse_ok"、"parse_err"。
+        """
+        # Default implementation does nothing
+        # 默认实现不执行任何操作
+        pass

aioscrapy/dupefilters/disk.py

@@ -1,3 +1,14 @@
+"""
+Disk-based Request Fingerprint Duplicate Filter for AioScrapy
+AioScrapy的基于磁盘的请求指纹重复过滤器
+
+This module provides a duplicate filter that stores request fingerprints on disk,
+allowing for persistence between crawler runs. It implements the DupeFilterBase
+interface and is used to avoid crawling the same URL multiple times.
+此模块提供了一个将请求指纹存储在磁盘上的重复过滤器，允许在爬虫运行之间保持持久性。
+它实现了DupeFilterBase接口，用于避免多次爬取相同的URL。
+"""
+
 import os
 from typing import Optional, Set
 
@@ -6,14 +17,51 @@ from aioscrapy.dupefilters import DupeFilterBase
 
 
 class DiskRFPDupeFilter(DupeFilterBase):
-    """Request Fingerprint duplicates filter built with Disk storage"""
+    """
+    Request Fingerprint duplicates filter built with Disk storage.
+    基于磁盘存储构建的请求指纹重复过滤器。
+
+    This filter stores request fingerprints in memory and on disk, allowing for
+    persistence between crawler runs. It implements the DupeFilterBase interface
+    and is used to avoid crawling the same URL multiple times.
+    此过滤器将请求指纹存储在内存和磁盘上，允许在爬虫运行之间保持持久性。
+    它实现了DupeFilterBase接口，用于避免多次爬取相同的URL。
+    """
 
     def __init__(self, path: Optional[str] = None, debug: bool = False, info: bool = False):
-        self.file: Optional["File object"] = None
+        """
+        Initialize the disk-based request fingerprint filter.
+        初始化基于磁盘的请求指纹过滤器。
+
+        Args:
+            path: Directory path where to store the requests.seen file.
+                 存储requests.seen文件的目录路径。
+                 If None, no persistence will be used.
+                 如果为None，则不会使用持久性存储。
+            debug: Whether to log filtered requests.
+                  是否记录被过滤的请求。
+            info: Whether to log duplicate requests.
+                 是否记录重复的请求。
+        """
+        # File handle for the requests.seen file
+        # requests.seen文件的文件句柄
+        self.file: Optional[object] = None
+
+        # Whether to log filtered requests
+        # 是否记录被过滤的请求
         self.debug = debug
+
+        # Set of request fingerprints
+        # 请求指纹的集合
         self.fingerprints: Set = set()
+
+        # Whether to log duplicate requests
+        # 是否记录重复的请求
         self.logdupes: bool = True
         self.info: bool = info
+
+        # If a path is provided, open the requests.seen file and load existing fingerprints
+        # 如果提供了路径，则打开requests.seen文件并加载现有的指纹
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
             self.file.seek(0)
@@ -21,24 +69,98 @@ class DiskRFPDupeFilter(DupeFilterBase):
 
     @classmethod
     def from_crawler(cls, crawler: "aioscrapy.crawler.Crawler"):
+        """
+        Create a DiskRFPDupeFilter instance from a crawler.
+        从爬虫创建DiskRFPDupeFilter实例。
+
+        This is the factory method used by AioScrapy to create the dupefilter.
+        这是AioScrapy用于创建重复过滤器的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this dupefilter.
+                    将使用此重复过滤器的爬虫。
+
+        Returns:
+            DiskRFPDupeFilter: A new DiskRFPDupeFilter instance.
+                              一个新的DiskRFPDupeFilter实例。
+        """
+        # Get debug setting from crawler settings
+        # 从爬虫设置获取debug设置
         debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+
+        # Get info setting from crawler settings
+        # 从爬虫设置获取info设置
         info = crawler.settings.getbool('DUPEFILTER_INFO')
+
+        # Get job directory from crawler settings, default to './job_dir'
+        # 从爬虫设置获取作业目录，默认为'./job_dir'
         path = crawler.settings.get('JOBDIR', './job_dir')
+
+        # Create the job directory if it doesn't exist
+        # 如果作业目录不存在，则创建它
         if path and not os.path.exists(path):
             os.makedirs(path)
+
+        # Create and return a new instance
+        # 创建并返回一个新实例
         return cls(path, debug, info)
 
     async def request_seen(self, request: Request) -> bool:
+        """
+        Check if a request has been seen before.
+        检查请求是否已经被看到过。
+
+        This method checks if the request's fingerprint is in the set of seen
+        fingerprints. If it is, the request is considered a duplicate. If not,
+        the fingerprint is added to the set and written to the requests.seen file.
+        此方法检查请求的指纹是否在已见过的指纹集合中。如果是，则认为请求是重复的。
+        如果不是，则将指纹添加到集合中并写入requests.seen文件。
+
+        Args:
+            request: The request to check.
+                    要检查的请求。
+
+        Returns:
+            bool: True if the request has been seen before, False otherwise.
+                 如果请求之前已经被看到过，则为True，否则为False。
+        """
+        # Check if the request's fingerprint is in the set of seen fingerprints
+        # 检查请求的指纹是否在已见过的指纹集合中
         if request.fingerprint in self.fingerprints:
             return True
+
+        # Add the fingerprint to the set
+        # 将指纹添加到集合中
         self.fingerprints.add(request.fingerprint)
+
+        # If we're using a file, write the fingerprint to it
+        # 如果我们使用文件，则将指纹写入文件
         if self.file:
             self.file.write(request.fingerprint + '\n')
+
+        # The request has not been seen before
+        # 请求之前未被看到过
         return False
 
     async def close(self, reason: str = '') -> None:
+        """
+        Close the dupefilter.
+        关闭重复过滤器。
+
+        This method is called when the spider is closed. It closes the requests.seen
+        file if it was opened.
+        当爬虫关闭时调用此方法。如果requests.seen文件已打开，则关闭它。
+
+        Args:
+            reason: The reason why the spider was closed.
+                   爬虫被关闭的原因。
+        """
+        # Close the file if it was opened
+        # 如果文件已打开，则关闭它
         if self.file:
             self.file.close()
 
 
+# Alias for backward compatibility
+# 用于向后兼容的别名
 RFPDupeFilter = DiskRFPDupeFilter