aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/core/engine.py

@@ -1,9 +1,29 @@
 # _*_ coding: utf-8 _*_
+"""
+Execution Engine Module
+执行引擎模块
+
+This module provides the core execution engine for AioScrapy, which coordinates
+the crawling process. The engine manages the scheduling of requests, downloading
+of pages, and processing of responses through the scraper.
+此模块提供了AioScrapy的核心执行引擎，它协调爬取过程。引擎管理请求的调度、
+页面的下载以及通过抓取器处理响应。
+
+The main components are:
+主要组件包括：
+
+1. ExecutionEngine: Coordinates the entire crawling process
+                   协调整个爬取过程
+2. Slot: Holds spider running state and resources
+        保存爬虫运行状态和资源
+
+The engine is the central component that connects all other parts of the crawling
+system: the scheduler, downloader, scraper, and spider.
+引擎是连接爬取系统所有其他部分的中央组件：调度器、下载器、抓取器和爬虫。
+"""
 
 import asyncio
 import time
-from asyncio import Queue
-from asyncio.queues import QueueEmpty
 from typing import Optional, AsyncGenerator, Union, Callable
 
 import aioscrapy
@@ -17,66 +37,176 @@ from aioscrapy.http import Response
 from aioscrapy.http.request import Request
 from aioscrapy.utils.log import logger
 from aioscrapy.utils.misc import load_instance
-from aioscrapy.utils.tools import call_helper, create_task
+from aioscrapy.utils.tools import call_helper
 
 
 class Slot:
+    """
+    A slot for holding spider running state and resources.
+    用于保存爬虫运行状态和资源的槽。
+
+    This class keeps track of in-progress requests and start requests
+    for a spider.
+    此类跟踪爬虫的进行中请求和起始请求。
+    """
 
     def __init__(self, start_requests: Optional[AsyncGenerator]) -> None:
-        self.inprogress: set[Request] = set()  # requests in progress
+        """
+        Initialize a new Slot.
+        初始化一个新的Slot。
+
+        Args:
+            start_requests: An async generator that yields initial requests.
+                           产生初始请求的异步生成器。
+        """
+        self.inprogress: set[Request] = set()  # requests in progress 进行中的请求
         self.start_requests = start_requests
-        self.lock: bool = False
+        self.lock: bool = False  # lock for accessing start_requests 访问start_requests的锁
 
     def add_request(self, request: Request) -> None:
+        """
+        Add a request to the set of in-progress requests.
+        将请求添加到进行中请求的集合中。
+
+        Args:
+            request: The request to add.
+                    要添加的请求。
+        """
         self.inprogress.add(request)
 
     def remove_request(self, request: Request) -> None:
+        """
+        Remove a request from the set of in-progress requests.
+        从进行中请求的集合中移除请求。
+
+        Args:
+            request: The request to remove.
+                    要移除的请求。
+        """
         self.inprogress.remove(request)
 
 
 class ExecutionEngine(object):
+    """
+    The execution engine coordinates the crawling process.
+    执行引擎协调爬取过程。
+
+    It manages the scheduling of requests, downloading of pages, and processing
+    of responses through the scraper. The engine is the central component that
+    connects all other parts of the crawling system.
+    它管理请求的调度、页面的下载以及通过抓取器处理响应。引擎是连接爬取系统
+    所有其他部分的中央组件。
+
+    The engine's main responsibilities include:
+    引擎的主要职责包括：
+
+    1. Starting and stopping the crawling process
+       启动和停止爬取过程
+    2. Scheduling requests through the scheduler
+       通过调度器调度请求
+    3. Sending requests to the downloader
+       将请求发送到下载器
+    4. Passing responses to the scraper
+       将响应传递给抓取器
+    5. Handling spider idle state
+       处理爬虫空闲状态
+
+    The engine maintains a slot for each running spider, which keeps track of
+    in-progress requests and start requests.
+    引擎为每个运行的爬虫维护一个槽，该槽跟踪进行中的请求和起始请求。
+    """
 
     def __init__(self, crawler: "aioscrapy.Crawler") -> None:
+        """
+        Initialize the execution engine.
+        初始化执行引擎。
+
+        Args:
+            crawler: The crawler instance that this engine belongs to.
+                    此引擎所属的爬虫实例。
+        """
         self.crawler = crawler
         self.settings = crawler.settings
         self.signals = crawler.signals
         self.logformatter = crawler.logformatter
 
+        # Components initialized during open()
+        # 在open()期间初始化的组件
         self.slot: Optional[Slot] = None
         self.spider: Optional[Spider] = None
         self.downloader: Optional[DownloaderTV] = None
         self.scraper: Optional[Scraper] = None
         self.scheduler: Optional[BaseScheduler] = None
 
-        self.running: bool = False
-        self.unlock: bool = True
-        self.finish: bool = False
+        # Engine state
+        # 引擎状态
+        self.running: bool = False  # True when engine is running
+        self.unlock: bool = True    # Lock for scheduler access
+        self.finish: bool = False   # True when engine is completely finished
 
     async def start(
             self,
             spider: Spider,
             start_requests: Optional[AsyncGenerator] = None
     ) -> None:
-        """Start the execution engine"""
+        """
+        Start the execution engine.
+        启动执行引擎。
+
+        This method initializes the engine components, opens the spider,
+        and starts the main crawling loop.
+        此方法初始化引擎组件，打开爬虫，并启动主爬取循环。
+
+        Args:
+            spider: The spider instance to run.
+                   要运行的爬虫实例。
+            start_requests: Optional async generator of initial requests.
+                           初始请求的可选异步生成器。
+
+        Raises:
+            RuntimeError: If the engine is already running.
+                         如果引擎已经在运行。
+        """
         if self.running:
             raise RuntimeError("Engine already running")
 
         self.running = True
         await self.signals.send_catch_log_deferred(signal=signals.engine_started)
         await self.open(spider, start_requests)
+
+        # Main crawling loop
+        # 主爬取循环
         while not self.finish:
             self.running and await self._next_request()
             await asyncio.sleep(1)
             self.running and await self._spider_idle(self.spider)
 
     async def stop(self, reason: str = 'shutdown') -> None:
-        """Stop the execution engine gracefully"""
+        """
+        Stop the execution engine gracefully.
+        优雅地停止执行引擎。
+
+        This method stops the engine, waits for all pending requests to complete,
+        closes the spider, and sends the engine_stopped signal.
+        此方法停止引擎，等待所有待处理的请求完成，关闭爬虫，并发送engine_stopped信号。
+
+        Args:
+            reason: The reason for stopping the engine.
+                   停止引擎的原因。
+
+        Raises:
+            RuntimeError: If the engine is not running.
+                         如果引擎没有运行。
+        """
         if not self.running:
             raise RuntimeError("Engine not running")
         self.running = False
 
+        # Wait for all pending requests to complete
+        # 等待所有待处理的请求完成
         while not self.is_idle():
             await asyncio.sleep(0.2)
+
         await self.close_spider(self.spider, reason=reason)
         await self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
         self.finish = True
@@ -86,39 +216,84 @@ class ExecutionEngine(object):
             spider: Spider,
             start_requests: Optional[AsyncGenerator] = None
     ) -> None:
+        """
+        Open a spider for crawling.
+        打开爬虫进行爬取。
+
+        This method initializes all the components needed for crawling:
+        scheduler, downloader, scraper, and slot. It also sends the spider_opened signal.
+        此方法初始化爬取所需的所有组件：调度器、下载器、抓取器和槽。它还发送spider_opened信号。
+
+        Args:
+            spider: The spider instance to open.
+                   要打开的爬虫实例。
+            start_requests: Optional async generator of initial requests.
+                           初始请求的可选异步生成器。
+        """
         logger.info("Spider opened")
 
         self.spider = spider
         await call_helper(self.crawler.stats.open_spider, spider)
 
+        # Initialize components
+        # 初始化组件
         self.scheduler = await load_instance(self.settings['SCHEDULER'], crawler=self.crawler)
         self.downloader = await load_instance(self.settings['DOWNLOADER'], crawler=self.crawler)
         self.scraper = await call_helper(Scraper.from_crawler, self.crawler)
 
+        # Process start requests through spider middleware
+        # 通过爬虫中间件处理起始请求
         start_requests = await call_helper(self.scraper.spidermw.process_start_requests, start_requests, spider)
         self.slot = Slot(start_requests)
 
         await self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
 
     async def close(self) -> None:
-        """Close the execution engine gracefully.
+        """
+        Close the execution engine gracefully.
+        优雅地关闭执行引擎。
 
         If it has already been started, stop it. In all cases, close all spiders
         and the downloader.
+        如果它已经启动，则停止它。在所有情况下，关闭所有爬虫和下载器。
+
+        This method is the main entry point for shutting down the engine from
+        outside the engine itself.
+        此方法是从引擎外部关闭引擎的主要入口点。
         """
         if self.running:
             # Will also close spiders and downloader
+            # 也会关闭爬虫和下载器
             await self.stop()
         elif self.spider:
             # Will also close downloader
+            # 也会关闭下载器
             await self.close_spider(self.spider, reason='shutdown')
         else:
+            # Just close the downloader if no spider is running
+            # 如果没有爬虫在运行，只关闭下载器
             self.downloader.close()
 
     async def _next_request(self) -> None:
+        """
+        Process the next request from the scheduler or start requests.
+        处理来自调度器或起始请求的下一个请求。
+
+        This method is the core of the crawling process. It handles:
+        此方法是爬取过程的核心。它处理：
+
+        1. Spider pause/resume logic
+           爬虫暂停/恢复逻辑
+        2. Getting requests from the scheduler and sending them to the downloader
+           从调度器获取请求并将其发送到下载器
+        3. Processing start requests
+           处理起始请求
+        """
         if self.slot is None or self.spider is None:
             return
 
+        # Handle spider pause/resume logic
+        # 处理爬虫暂停/恢复逻辑
         if self.spider.pause:
             now = int(time.time())
             last_log_time = getattr(self.spider, "last_log_time", None)
@@ -130,6 +305,8 @@ class ExecutionEngine(object):
                 self.spider.pause = False
             return
 
+        # Get requests from scheduler and send them to downloader
+        # 从调度器获取请求并将其发送到下载器
         while self.unlock and not self._needs_backout() and self.unlock:
             self.unlock = False
             try:
@@ -141,21 +318,54 @@ class ExecutionEngine(object):
             finally:
                 self.unlock = True
 
+        # Process start requests if available
+        # 如果可用，处理起始请求
         if self.slot.start_requests and not self._needs_backout() and not self.slot.lock:
             self.slot.lock = True
             try:
+                # Get the next request from start_requests
+                # 从start_requests获取下一个请求
                 request = await self.slot.start_requests.__anext__()
             except StopAsyncIteration:
+                # No more start requests, set to None
+                # 没有更多的起始请求，设置为None
                 self.slot.start_requests = None
-            except Exception as e:
+            except Exception as exc:
+                # Log any errors and stop processing start requests
+                # 记录任何错误并停止处理起始请求
                 self.slot.start_requests = None
-                logger.exception('Error while obtaining start requests')
+                logger.exception('Error while obtaining start requests: %s', str(exc))
             else:
+                # If we got a request, schedule it for crawling
+                # 如果我们得到了请求，安排它进行爬取
                 request and await self.crawl(request)
             finally:
+                # Always release the lock
+                # 始终释放锁
                 self.slot.lock = False
 
     def _needs_backout(self) -> bool:
+        """
+        Check if the engine should temporarily stop processing more requests.
+        检查引擎是否应该暂时停止处理更多请求。
+
+        This method determines if the request processing loop should pause by checking:
+        此方法通过检查以下条件来确定请求处理循环是否应该暂停：
+
+        1. If the engine is no longer running (self.running is False)
+           引擎是否不再运行（self.running为False）
+        2. If the downloader is at capacity or needs to pause
+           下载器是否已达到容量或需要暂停
+        3. If the scraper is at capacity or needs to pause
+           抓取器是否已达到容量或需要暂停
+
+        This is used to implement flow control in the request processing pipeline.
+        这用于在请求处理管道中实现流量控制。
+
+        Returns:
+            True if request processing should pause, False if it can continue.
+            如果请求处理应该暂停，则返回True；如果可以继续，则返回False。
+        """
         return (
                 not self.running
                 or self.downloader.needs_backout()
@@ -165,6 +375,32 @@ class ExecutionEngine(object):
     async def handle_downloader_output(
             self, result: Union[Request, Response, BaseException, None], request: Request
     ) -> None:
+        """
+        Handle the output from the downloader.
+        处理下载器的输出。
+
+        This method processes the result of a download, which can be:
+        此方法处理下载的结果，可以是：
+
+        - None: Download was cancelled or failed without an exception
+          None：下载被取消或失败，没有异常
+        - Request: A new request to crawl
+          Request：要爬取的新请求
+        - Response: A successful response
+          Response：成功的响应
+        - BaseException: An exception that occurred during download
+          BaseException：下载过程中发生的异常
+
+        Args:
+            result: The result of the download.
+                   下载的结果。
+            request: The original request that was downloaded.
+                    被下载的原始请求。
+
+        Raises:
+            TypeError: If the result is not None, Request, Response, or BaseException.
+                      如果结果不是None、Request、Response或BaseException。
+        """
         try:
             if result is None:
                 return
@@ -176,56 +412,157 @@ class ExecutionEngine(object):
                 )
 
             if isinstance(result, Request):
+                # Schedule new request
+                # 调度新请求
                 await self.crawl(result)
                 return
 
+            # Set the original request on the result
+            # 在结果上设置原始请求
             result.request = request
+
             if isinstance(result, Response):
+                # Log successful response and send signal
+                # 记录成功的响应并发送信号
                 logger.log(**self.logformatter.crawled(request, result, self.spider))
                 await self.signals.send_catch_log(signals.response_received,
                                                   response=result, request=request, spider=self.spider)
+
+            # Send result to scraper for processing
+            # 将结果发送到抓取器进行处理
             await self.scraper.enqueue_scrape(result, request)
 
         finally:
+            # Always remove the request from in-progress and process next request
+            # 始终从进行中移除请求并处理下一个请求
             self.slot.remove_request(request)
             await self._next_request()
 
     def is_idle(self) -> bool:
-
+        """
+        Check if the engine is idle.
+        检查引擎是否空闲。
+
+        The engine is considered idle when:
+        在以下情况下，引擎被认为是空闲的：
+
+        1. The downloader has no active requests
+           下载器没有活动的请求
+        2. There are no requests in progress
+           没有正在进行的请求
+        3. The scraper is idle
+           抓取器是空闲的
+
+        Returns:
+            True if the engine is idle, False otherwise.
+            如果引擎空闲，则为True，否则为False。
+        """
         if self.downloader.active:
             # downloader has pending requests
+            # 下载器有待处理的请求
             return False
 
         if self.slot.inprogress:
             # not all start requests are handled
+            # 不是所有的起始请求都已处理
             return False
 
         if not self.scraper.is_idle():
             # scraper is not idle
+            # 抓取器不是空闲的
             return False
 
         return True
 
     async def crawl(self, request: Request) -> None:
+        """
+        Schedule a request for crawling.
+        调度请求进行爬取。
+
+        This method adds the request to the scheduler's queue.
+        此方法将请求添加到调度器的队列中。
+
+        Args:
+            request: The request to schedule.
+                    要调度的请求。
+        """
         await self.scheduler.enqueue_request(request)
-        # create_task(self._next_request())
 
     async def close_spider(self, spider: Spider, reason: str = 'cancelled') -> None:
-        """Close (cancel) spider and clear all its outstanding requests"""
-
+        """
+        Close (cancel) spider and clear all its outstanding requests.
+        关闭（取消）爬虫并清除其所有未完成的请求。
+
+        This method gracefully shuts down all components related to the spider:
+        此方法优雅地关闭与爬虫相关的所有组件：
+
+        1. Downloader
+           下载器
+        2. Scraper
+           抓取器
+        3. Scheduler
+           调度器
+        4. Stats collector
+           统计收集器
+        5. Sends the spider_closed signal
+           发送spider_closed信号
+
+        Args:
+            spider: The spider to close.
+                   要关闭的爬虫。
+            reason: The reason for closing the spider.
+                   关闭爬虫的原因。
+        """
         logger.info(f"Closing spider ({reason})")
 
+        # Helper function to handle exceptions during close operations
+        # 处理关闭操作期间异常的辅助函数
         async def close_handler(
                 callback: Callable,
                 *args,
-                errmsg: str = '',
+                errmsg: str = '',  # Error message to log if an exception occurs
+                                  # 如果发生异常时记录的错误消息
                 **kwargs
         ) -> None:
+            """
+            Call a callback and log any exceptions that occur.
+            调用回调并记录发生的任何异常。
+
+            This is an internal helper function used during the spider closing process
+            to ensure that exceptions in one closing operation don't prevent other
+            closing operations from being attempted. It wraps each callback in a
+            try-except block and logs any exceptions with the provided error message.
+            这是在爬虫关闭过程中使用的内部辅助函数，用于确保一个关闭操作中的异常
+            不会阻止尝试其他关闭操作。它将每个回调包装在try-except块中，并使用
+            提供的错误消息记录任何异常。
+
+            Args:
+                callback: The callback function to call.
+                         要调用的回调函数。
+                *args: Positional arguments to pass to the callback.
+                      传递给回调的位置参数。
+                errmsg: Error message prefix to log if an exception occurs.
+                       如果发生异常时记录的错误消息前缀。
+                       This will be prepended to the exception string in the log.
+                       这将在日志中添加到异常字符串之前。
+                **kwargs: Keyword arguments to pass to the callback.
+                         传递给回调的关键字参数。
+
+            Note:
+                This function catches all exceptions (including BaseException) to ensure
+                that the closing process continues even if a critical error occurs.
+                此函数捕获所有异常（包括BaseException），以确保即使发生严重错误，
+                关闭过程也会继续。
+            """
             try:
                 await call_helper(callback, *args, **kwargs)
-            except (Exception, BaseException) as e:
-                logger.exception(errmsg)
+            except (Exception, BaseException) as exc:
+                # Log the error message along with the exception details
+                # 记录错误消息以及异常详细信息
+                logger.exception(f"{errmsg}: {str(exc)}")
 
+        # Close all components in sequence
+        # 按顺序关闭所有组件
         await close_handler(self.downloader.close, errmsg='Downloader close failure')
 
         await close_handler(self.scraper.close, errmsg='Scraper close failure')
@@ -239,17 +576,41 @@ class ExecutionEngine(object):
 
         logger.info(f"Spider closed ({reason})")
 
+        # Clean up references
+        # 清理引用
         await close_handler(setattr, self, 'slot', None, errmsg='Error while unassigning slot')
 
         await close_handler(setattr, self, 'spider', None, errmsg='Error while unassigning spider')
 
     async def _spider_idle(self, spider: Spider) -> None:
+        """
+        Handle the spider_idle signal.
+        处理spider_idle信号。
+
+        This method is called when the spider becomes idle (no more requests to process).
+        当爬虫变为空闲状态（没有更多请求要处理）时，调用此方法。
+
+        It sends the spider_idle signal, which handlers can use to add more requests.
+        它发送spider_idle信号，处理程序可以使用该信号添加更多请求。
+
+        If no handler raises DontCloseSpider and there are no pending requests,
+        the spider is stopped.
+        如果没有处理程序引发DontCloseSpider且没有待处理的请求，则停止爬虫。
+
+        Args:
+            spider: The idle spider.
+                   空闲的爬虫。
+        """
         assert self.spider is not None
+
+        # Send spider_idle signal and check if any handler wants to keep the spider open
+        # 发送spider_idle信号并检查是否有任何处理程序希望保持爬虫打开
         res = await self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider)
         if any(isinstance(x, DontCloseSpider) for _, x in res):
             return
 
         # method of 'has_pending_requests' has IO, so method of 'is_idle' execute twice
+        # 'has_pending_requests'方法有IO操作，所以'is_idle'方法执行两次
         if self.is_idle() \
                 and self.slot.start_requests is None \
                 and not await self.scheduler.has_pending_requests() \