aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/process.py

@@ -1,3 +1,29 @@
+"""
+Process Management Module
+进程管理模块
+
+This module provides functions for running spiders in single or multiple processes.
+It handles the creation and management of processes, as well as the initialization
+of event loops appropriate for different platforms.
+此模块提供了在单个或多个进程中运行爬虫的函数。它处理进程的创建和管理，
+以及适合不同平台的事件循环的初始化。
+
+The main functions are:
+主要函数包括：
+
+1. single_process_run: Run multiple spiders in a single process
+                      在单个进程中运行多个爬虫
+2. multi_process_run: Run multiple spiders in separate processes
+                     在单独的进程中运行多个爬虫
+3. loop_initializer: Initialize an appropriate event loop based on the platform
+                    根据平台初始化适当的事件循环
+
+This module is particularly useful for running multiple spiders concurrently,
+either in the same process or in separate processes for better isolation and
+resource utilization.
+此模块对于并发运行多个爬虫特别有用，可以在同一进程中运行，也可以在单独的
+进程中运行，以获得更好的隔离和资源利用。
+"""
 import asyncio
 import sys
 from typing import Optional, Tuple, List, Union, Type, AnyStr
@@ -10,43 +36,217 @@ from aioscrapy.settings import Settings
 
 
 def loop_initializer():
+    """
+    Initialize and return an appropriate event loop based on the platform.
+    根据平台初始化并返回适当的事件循环。
+
+    This function selects the most efficient event loop implementation available
+    for the current platform:
+    此函数为当前平台选择最高效的事件循环实现：
+
+    - On Windows, returns a ProactorEventLoop which is optimized for Windows I/O operations.
+      在Windows上，返回ProactorEventLoop，它针对Windows I/O操作进行了优化。
+
+    - On other platforms (Linux, macOS, etc.), tries to use uvloop if available,
+      which is a fast drop-in replacement for the standard asyncio event loop.
+      在其他平台（Linux、macOS等）上，尝试使用uvloop（如果可用），
+      它是标准asyncio事件循环的快速替代品。
+
+    - If uvloop is not available, falls back to the standard asyncio event loop.
+      如果uvloop不可用，则回退到标准asyncio事件循环。
+
+    This function is used by the process management functions to ensure that
+    each process has an appropriate and efficient event loop.
+    进程管理函数使用此函数来确保每个进程都有适当且高效的事件循环。
+
+    Returns:
+        An event loop or event loop policy appropriate for the current platform.
+        适合当前平台的事件循环或事件循环策略。
+    """
+    # On Windows, use ProactorEventLoop which supports all asyncio features
+    # 在Windows上，使用支持所有asyncio功能的ProactorEventLoop
     if sys.platform.startswith('win'):
         return asyncio.windows_events.ProactorEventLoop()
+
+    # On other platforms, try to use uvloop which is much faster
+    # 在其他平台上，尝试使用更快的uvloop
     try:
         import uvloop
         return uvloop.EventLoopPolicy()
     except ImportError:
+        # If uvloop is not available, use the standard event loop
+        # 如果uvloop不可用，则使用标准事件循环
         pass
 
+    # Fall back to the standard asyncio event loop
+    # 回退到标准asyncio事件循环
     return asyncio.new_event_loop()
 
 
 def multi_process_run(*tasks: Union[Tuple[Type[Spider], Optional[AnyStr]], List]):
+    """
+    Run multiple spiders in separate processes.
+    在单独的进程中运行多个爬虫。
+
+    This function creates a new process for each task or list of tasks provided.
+    Each process runs independently with its own event loop, allowing for true
+    parallel execution across multiple CPU cores.
+    此函数为提供的每个任务或任务列表创建一个新进程。每个进程都有自己的事件循环
+    独立运行，允许在多个CPU核心上实现真正的并行执行。
+
+    Using multiple processes provides better isolation between spiders and can
+    improve performance on multi-core systems, but comes with higher memory
+    overhead compared to running all spiders in a single process.
+    使用多个进程可以提供更好的爬虫之间的隔离，并可以在多核系统上提高性能，
+    但与在单个进程中运行所有爬虫相比，会带来更高的内存开销。
+
+    Args:
+        *tasks: Each task can be either a tuple of (Spider class, settings) or a list of such tuples.
+               每个任务可以是(爬虫类, 设置)的元组，或者是这种元组的列表。
+
+               If a task is a list, all spiders in that list will run in the same process.
+               如果任务是列表，则该列表中的所有爬虫将在同一进程中运行。
+
+               The settings parameter can be a string (path to settings module) or None.
+               设置参数可以是字符串（设置模块的路径）或None。
+
+    Example:
+        ```python
+        # Run two spiders in separate processes
+        multi_process_run(
+            (MySpider1, 'myproject.settings'),
+            (MySpider2, 'myproject.settings')
+        )
+
+        # Run two spiders in one process, and a third in another process
+        multi_process_run(
+            [(MySpider1, 'myproject.settings'), (MySpider2, 'myproject.settings')],
+            (MySpider3, 'myproject.settings')
+        )
+        ```
+    """
+    # Process each task
+    # 处理每个任务
     for task in tasks:
         if isinstance(task, list):
+            # If task is a list, run all spiders in that list in the same process
+            # 如果任务是列表，则在同一进程中运行该列表中的所有爬虫
             p = Process(target=_single_process_run_async, args=(*task,), loop_initializer=loop_initializer)
         else:
+            # If task is a single spider, run it in its own process
+            # 如果任务是单个爬虫，则在其自己的进程中运行它
             p = Process(target=_single_process_run_async, args=(task,), loop_initializer=loop_initializer)
+
+        # Start the process
+        # 启动进程
         p.start()
 
 
 async def _single_process_run_async(*tasks: Tuple[Type[Spider], Optional[AnyStr]]):
+    """
+    Run multiple spiders in a single process asynchronously.
+    在单个进程中异步运行多个爬虫。
+
+    This is an internal helper function used by multi_process_run. It creates a
+    CrawlerProcess, adds all the specified spiders to it, and then runs them
+    concurrently within the same process.
+    这是一个由multi_process_run使用的内部辅助函数。它创建一个CrawlerProcess，
+    将所有指定的爬虫添加到其中，然后在同一进程中并发运行它们。
+
+    The function handles the conversion of settings from string paths to Settings
+    objects if needed.
+    如果需要，该函数会处理将设置从字符串路径转换为Settings对象。
+
+    Args:
+        *tasks: Tuples of (Spider class, settings).
+               (爬虫类, 设置)的元组。
+
+               Each tuple contains a Spider class and its settings.
+               每个元组包含一个爬虫类及其设置。
+
+               The settings parameter can be a string (path to settings module) or None.
+               If it's a string, it will be converted to a Settings object.
+               设置参数可以是字符串（设置模块的路径）或None。
+               如果是字符串，它将被转换为Settings对象。
+    """
+    # Create a crawler process to run all spiders
+    # 创建一个爬虫进程来运行所有爬虫
     cp = CrawlerProcess()
+
+    # Add each spider to the crawler process
+    # 将每个爬虫添加到爬虫进程
     for spidercls, settings in tasks:
+        # Convert string settings to Settings objects if needed
+        # 如果需要，将字符串设置转换为Settings对象
         if isinstance(settings, str):
             instance = Settings()
             instance.setmodule(settings)
             settings = instance
+
+        # Add the spider to the crawler process
+        # 将爬虫添加到爬虫进程
         cp.crawl(spidercls, settings=settings)
+
+    # Run all spiders concurrently and wait for them to finish
+    # 并发运行所有爬虫并等待它们完成
     await cp.run()
 
 
 def single_process_run(*tasks: Tuple[Type[Spider], Optional[AnyStr]]):
+    """
+    Run multiple spiders in a single process.
+    在单个进程中运行多个爬虫。
+
+    This function creates a CrawlerProcess and runs all provided spiders in it.
+    The spiders run concurrently within the same process using asyncio.
+    此函数创建一个CrawlerProcess并在其中运行所有提供的爬虫。
+    爬虫使用asyncio在同一进程中并发运行。
+
+    Running multiple spiders in a single process uses less memory than running
+    them in separate processes, but doesn't provide the same level of isolation
+    or parallel execution across CPU cores.
+    在单个进程中运行多个爬虫比在单独的进程中运行它们使用更少的内存，
+    但不提供相同级别的隔离或跨CPU核心的并行执行。
+
+    Args:
+        *tasks: Tuples of (Spider class, settings).
+               (爬虫类, 设置)的元组。
+
+               Each tuple contains a Spider class and its settings.
+               每个元组包含一个爬虫类及其设置。
+
+               The settings parameter can be a string (path to settings module) or None.
+               If it's a string, it will be converted to a Settings object.
+               设置参数可以是字符串（设置模块的路径）或None。
+               如果是字符串，它将被转换为Settings对象。
+
+    Example:
+        ```python
+        # Run two spiders in a single process
+        single_process_run(
+            (MySpider1, 'myproject.settings'),
+            (MySpider2, 'myproject.settings')
+        )
+        ```
+    """
+    # Create a crawler process to run all spiders
+    # 创建一个爬虫进程来运行所有爬虫
     cp = CrawlerProcess()
+
+    # Add each spider to the crawler process
+    # 将每个爬虫添加到爬虫进程
     for spidercls, settings in tasks:
+        # Convert string settings to Settings objects if needed
+        # 如果需要，将字符串设置转换为Settings对象
         if isinstance(settings, str):
             instance = Settings()
             instance.setmodule(settings)
             settings = instance
+
+        # Add the spider to the crawler process
+        # 将爬虫添加到爬虫进程
         cp.crawl(spidercls, settings=settings)
+
+    # Start the crawler process and block until all spiders are finished
+    # 启动爬虫进程并阻塞直到所有爬虫完成
     cp.start()

aioscrapy/proxy/__init__.py

@@ -1,3 +1,14 @@
+"""
+Proxy module for aioscrapy.
+aioscrapy的代理模块。
+
+This module provides the abstract base class for proxy handlers in aioscrapy.
+It defines the interface that all proxy handlers must implement and provides
+common functionality for proxy management.
+此模块提供了aioscrapy中代理处理程序的抽象基类。
+它定义了所有代理处理程序必须实现的接口，并提供了代理管理的通用功能。
+"""
+
 from abc import ABCMeta, abstractmethod
 
 from aioscrapy.utils.log import logger
@@ -5,7 +16,38 @@ from aioscrapy.utils.python import global_object_name
 
 
 class AbsProxy(metaclass=ABCMeta):
+    """
+    Abstract base class for proxy handlers.
+    代理处理程序的抽象基类。
+
+    This class defines the interface that all proxy handlers must implement
+    and provides common functionality for proxy management, including adding
+    proxies to requests, removing invalid proxies, and checking proxy validity.
+    此类定义了所有代理处理程序必须实现的接口，并提供了代理管理的通用功能，
+    包括向请求添加代理、移除无效代理和检查代理有效性。
+
+    Attributes:
+        use_proxy (bool): Whether to use proxies.
+                         是否使用代理。
+        max_count (int): Maximum number of proxies to maintain.
+                        要维护的最大代理数量。
+        min_count (int): Minimum number of proxies to maintain.
+                        要维护的最小代理数量。
+        allow_status_code (list): HTTP status codes that are allowed even with a proxy.
+                                 即使使用代理也允许的HTTP状态码。
+        cache (list): List of available proxies.
+                     可用代理列表。
+    """
+
     def __init__(self, settings):
+        """
+        Initialize the proxy handler.
+        初始化代理处理程序。
+
+        Args:
+            settings: The aioscrapy settings object.
+                     aioscrapy设置对象。
+        """
         self.use_proxy = settings.getbool('USE_PROXY', False)
         self.max_count = settings.getint('PROXY_MAX_COUNT', 16)
         self.min_count = settings.getint('PROXY_MIN_COUNT', 1)
@@ -13,38 +55,135 @@ class AbsProxy(metaclass=ABCMeta):
         self.cache = []
 
     async def add_proxy(self, request):
-        """add proxy for request"""
+        """
+        Add a proxy to the request if proxy usage is enabled.
+        如果启用了代理使用，则向请求添加代理。
+
+        This method checks if proxy usage is enabled both globally and for the
+        specific request. If so, it gets a proxy from the pool and adds it to
+        the request's meta. Otherwise, it removes any existing proxy from the request.
+        此方法检查代理使用是否在全局和特定请求中都启用。如果是，它从池中获取代理
+        并将其添加到请求的meta中。否则，它会从请求中移除任何现有的代理。
+
+        Args:
+            request: The request to add a proxy to.
+                    要添加代理的请求。
+
+        Returns:
+            The modified request.
+            修改后的请求。
+        """
         if self.use_proxy and request.use_proxy:
+            # Get a proxy and add it to the request's meta
+            # 获取代理并将其添加到请求的meta中
             request.meta['proxy'] = await self.get()
         else:
+            # Remove any existing proxy from the request
+            # 从请求中移除任何现有的代理
             request.meta.pop('proxy', None)
         return request
 
     def remove(self, proxy, reason=None):
+        """
+        Remove a proxy from the cache.
+        从缓存中移除代理。
+
+        This method removes a proxy from the cache when it's determined to be invalid
+        or no longer usable. It logs the removal with the provided reason.
+        当确定代理无效或不再可用时，此方法从缓存中移除代理。它记录移除的原因。
+
+        Args:
+            proxy: The proxy to remove.
+                  要移除的代理。
+            reason: The reason for removing the proxy. Can be a callable, an exception,
+                   or any other object that can be converted to a string.
+                   移除代理的原因。可以是可调用对象、异常或任何其他可以转换为字符串的对象。
+        """
+        # If reason is callable, call it to get the actual reason
+        # 如果reason是可调用的，调用它以获取实际原因
         if callable(reason):
             reason = reason()
+
+        # If reason is an exception, use its class name
+        # 如果reason是异常，使用其类名
         if isinstance(reason, Exception):
             reason = global_object_name(reason.__class__)
 
+        # Remove the proxy if it's in the cache
+        # 如果代理在缓存中，则移除它
         if proxy in self.cache:
             logger.info(f"remove proxy: {proxy}, reason: {reason}")
             self.cache.remove(proxy)
 
     def check(self, request, response=None, exception=None):
+        """
+        Check if a proxy is still valid based on response or exception.
+        根据响应或异常检查代理是否仍然有效。
+
+        This method checks if a proxy should be removed based on the response status code
+        or an exception that occurred during the request. If the response status code is
+        not in the allowed list or if an exception occurred, the proxy is removed.
+        此方法根据响应状态码或请求期间发生的异常检查是否应该移除代理。
+        如果响应状态码不在允许列表中或发生异常，则移除代理。
+
+        Args:
+            request: The request that was made.
+                    发出的请求。
+            response: The response received, if any.
+                     收到的响应（如果有）。
+            exception: The exception that occurred, if any.
+                      发生的异常（如果有）。
+        """
+        # If proxy usage is disabled, do nothing
+        # 如果禁用了代理使用，则不执行任何操作
         if not self.use_proxy:
             return
 
+        # Check if the response status code is not allowed
+        # 检查响应状态码是否不被允许
         if response and response.status >= 400 and response.status not in self.allow_status_code:
             self.remove(request.meta.get('proxy'), f"Don't allow response status code:{response.status}")
 
+        # Check if an exception occurred
+        # 检查是否发生异常
         if exception and isinstance(exception, BaseException):
             self.remove(request.meta.get('proxy'), exception)
 
     @classmethod
     @abstractmethod
     async def from_crawler(cls, crawler) -> "AbsProxy":
-        """get proxy instance from spider"""
+        """
+        Create a proxy handler instance from a crawler.
+        从爬虫创建代理处理程序实例。
+
+        This class method is used to create a proxy handler instance from a crawler.
+        It is called by the crawler when initializing the proxy handler.
+        此类方法用于从爬虫创建代理处理程序实例。
+        它在初始化代理处理程序时由爬虫调用。
+
+        Args:
+            crawler: The crawler instance.
+                    爬虫实例。
+
+        Returns:
+            AbsProxy: A proxy handler instance.
+                     代理处理程序实例。
+        """
+        pass
 
     @abstractmethod
     async def get(self) -> str:
-        """get proxy"""
+        """
+        Get a proxy from the pool.
+        从池中获取代理。
+
+        This method is called when a proxy is needed for a request.
+        It should return a proxy in the format 'scheme://host:port'.
+        当请求需要代理时调用此方法。
+        它应该以'scheme://host:port'格式返回代理。
+
+        Returns:
+            str: A proxy string in the format 'scheme://host:port'.
+                 格式为'scheme://host:port'的代理字符串。
+        """
+        pass

aioscrapy/proxy/redis.py

@@ -1,6 +1,16 @@
+"""
+Redis-based proxy implementation for aioscrapy.
+aioscrapy的基于Redis的代理实现。
+
+This module provides a Redis-based implementation of the proxy handler interface.
+It fetches proxies from a Redis sorted set and manages them for use in requests.
+此模块提供了代理处理程序接口的基于Redis的实现。
+它从Redis有序集合中获取代理，并管理它们以用于请求。
+"""
+
 import asyncio
 import time
-from typing import Optional
+from typing import Optional, Any
 
 from aioscrapy.db import db_manager
 from aioscrapy.exceptions import ProxyException
@@ -10,13 +20,47 @@ from aioscrapy.utils.tools import create_task
 
 
 class RedisProxy(AbsProxy):
+    """
+    Redis-based proxy handler implementation.
+    基于Redis的代理处理程序实现。
+
+    This class implements the AbsProxy interface using Redis as a backend.
+    It fetches proxies from a Redis sorted set and manages them for use in requests.
+    此类使用Redis作为后端实现AbsProxy接口。
+    它从Redis有序集合中获取代理，并管理它们以用于请求。
+
+    Attributes:
+        crawler: The crawler instance.
+                爬虫实例。
+        proxy_queue: The Redis client used to fetch proxies.
+                    用于获取代理的Redis客户端。
+        proxy_key: The key of the Redis sorted set containing proxies.
+                  包含代理的Redis有序集合的键。
+        lock: An asyncio lock to prevent concurrent proxy fetching.
+             防止并发代理获取的asyncio锁。
+    """
+
     def __init__(
             self,
             settings,
             crawler,
-            proxy_queue: Optional["Redis"] = None,
+            proxy_queue: Optional[Any] = None,
             proxy_key: Optional[str] = None
     ):
+        """
+        Initialize the Redis proxy handler.
+        初始化Redis代理处理程序。
+
+        Args:
+            settings: The aioscrapy settings object.
+                     aioscrapy设置对象。
+            crawler: The crawler instance.
+                    爬虫实例。
+            proxy_queue: The Redis client used to fetch proxies.
+                        用于获取代理的Redis客户端。
+            proxy_key: The key of the Redis sorted set containing proxies.
+                      包含代理的Redis有序集合的键。
+        """
         super().__init__(settings)
         self.crawler = crawler
         self.proxy_queue = proxy_queue
@@ -25,11 +69,46 @@ class RedisProxy(AbsProxy):
 
     @classmethod
     async def from_crawler(cls, crawler) -> "RedisProxy":
+        """
+        Create a RedisProxy instance from a crawler.
+        从爬虫创建RedisProxy实例。
+
+        This class method creates a RedisProxy instance from a crawler.
+        It retrieves the necessary settings and initializes the Redis client.
+        此类方法从爬虫创建RedisProxy实例。
+        它检索必要的设置并初始化Redis客户端。
+
+        Args:
+            crawler: The crawler instance.
+                    爬虫实例。
+
+        Returns:
+            RedisProxy: A RedisProxy instance.
+                       RedisProxy实例。
+
+        Raises:
+            AssertionError: If PROXY_KEY is not configured in settings.
+                           如果在设置中未配置PROXY_KEY。
+        """
+        # Get settings from crawler
+        # 从爬虫获取设置
         settings = crawler.settings
+
+        # Get proxy key from settings
+        # 从设置获取代理键
         proxy_key = settings.get('PROXY_KEY')
         assert proxy_key is not None, "Not configured：'PROXY_KEY'"
+
+        # Get Redis alias from settings, default to 'proxy'
+        # 从设置获取Redis别名，默认为'proxy'
         alias = settings.get("PROXY_QUEUE_ALIAS", 'proxy')
+
+        # Get Redis client
+        # 获取Redis客户端
         proxy_queue = db_manager.redis(alias)
+
+        # Create and return RedisProxy instance
+        # 创建并返回RedisProxy实例
         return cls(
             settings,
             crawler,
@@ -38,6 +117,23 @@ class RedisProxy(AbsProxy):
         )
 
     async def fill_proxy(self, redis_key: str, count: int) -> None:
+        """
+        Fill the proxy cache from Redis.
+        从Redis填充代理缓存。
+
+        This method fetches proxies from a Redis sorted set and adds them to the cache.
+        It uses a Lua script to randomly select proxies from the sorted set.
+        此方法从Redis有序集合中获取代理并将它们添加到缓存中。
+        它使用Lua脚本从有序集合中随机选择代理。
+
+        Args:
+            redis_key: The key of the Redis sorted set containing proxies.
+                      包含代理的Redis有序集合的键。
+            count: The number of proxies to fetch.
+                  要获取的代理数量。
+        """
+        # Lua script to randomly select proxies from a sorted set
+        # Lua脚本，用于从有序集合中随机选择代理
         script = f"""
             local redis_key = KEYS[1]
             local min_score = ARGV[1]
@@ -50,23 +146,61 @@ class RedisProxy(AbsProxy):
             end
             return redis.call('ZRANGEBYSCORE', redis_key, min_score, max_score, 'LIMIT', start, {count})
         """
+        # Register and execute the script
+        # 注册并执行脚本
         cmd_script = self.proxy_queue.register_script(script)
+
+        # Try to get proxies with score between 100 and 100 (high quality proxies)
+        # 尝试获取分数在100到100之间的代理（高质量代理）
         result = await cmd_script(keys=[redis_key], args=[100, 100])
+
+        # If no high quality proxies are available, get any proxies
+        # 如果没有高质量代理可用，获取任何代理
         if not result:
             result = await cmd_script(keys=[redis_key], args=[0, 100])
+
+        # Format proxies and add them to the cache
+        # 格式化代理并将它们添加到缓存中
         proxies = [ip.decode() if ip.decode().startswith('http') else f'http://{ip.decode()}' for ip in result]
         self.cache.extend(proxies)
         logger.info(f'Get proxy from redis: {proxies}')
 
     async def get(self) -> str:
+        """
+        Get a proxy from the cache.
+        从缓存中获取代理。
+
+        This method returns a proxy from the cache. If the cache is running low,
+        it fills the cache with more proxies from Redis. If no proxies are available,
+        it stops the crawler and raises an exception.
+        此方法从缓存中返回代理。如果缓存不足，它会从Redis中填充更多代理到缓存中。
+        如果没有可用的代理，它会停止爬虫并引发异常。
+
+        Returns:
+            str: A proxy string in the format 'scheme://host:port'.
+                 格式为'scheme://host:port'的代理字符串。
+
+        Raises:
+            ProxyException: If no proxies are available.
+                           如果没有可用的代理。
+        """
+        # If the cache is running low, fill it with more proxies
+        # 如果缓存不足，用更多代理填充它
         if len(self.cache) < self.min_count:
             async with self.lock:
+                # Check again inside the lock to avoid race conditions
+                # 在锁内再次检查以避免竞争条件
                 len(self.cache) < self.min_count and await self.fill_proxy(self.proxy_key, self.max_count - len(self.cache))
+
         try:
+            # Get a proxy from the cache and move it to the end
+            # 从缓存中获取代理并将其移到末尾
             proxy = self.cache.pop(0)
             self.cache.append(proxy)
             return proxy
         except IndexError:
+            # If no proxies are available, stop the crawler and raise an exception
+            # 如果没有可用的代理，停止爬虫并引发异常
             logger.warning("Not available proxy, Closing spider")
             create_task(self.crawler.engine.stop(reason="Not available proxy"))
             raise ProxyException("Not available proxy")