aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/middleware/spider.py

@@ -1,7 +1,23 @@
 """
-Spider Middleware manager
+Spider Middleware Manager
+爬虫中间件管理器
+
+This module provides the SpiderMiddlewareManager class, which manages the execution
+of spider middleware components. Spider middlewares process the input and output of
+spiders, allowing modification of requests and responses, as well as handling of
+exceptions that occur during spider processing.
+此模块提供了SpiderMiddlewareManager类，用于管理爬虫中间件组件的执行。爬虫中间件
+处理爬虫的输入和输出，允许修改请求和响应，以及处理爬虫处理过程中发生的异常。
+
+Spider middlewares are loaded from the SPIDER_MIDDLEWARES setting and are executed
+in the order specified by their priority values. They can intercept and modify the
+requests generated by spiders, the responses passed to spiders, and the items and
+requests yielded by spiders.
+爬虫中间件从SPIDER_MIDDLEWARES设置加载，并按照其优先级值指定的顺序执行。它们可以
+拦截和修改由爬虫生成的请求、传递给爬虫的响应，以及由爬虫产生的项目和请求。
 
 See documentation in docs/topics/spider-middleware.rst
+参见文档 docs/topics/spider-middleware.rst
 """
 from itertools import islice
 from typing import AsyncIterable, Iterable, Callable, Union
@@ -18,115 +34,452 @@ from aioscrapy.middleware.absmanager import AbsMiddlewareManager
 
 
 def _fname(f):
+    """
+    Get the full name of a method.
+    获取方法的完整名称。
+
+    This helper function returns the full name of a method in the format
+    "ClassName.method_name". It is used for error messages to identify which
+    middleware method raised an exception or returned an invalid output.
+    此辅助函数返回格式为"ClassName.method_name"的方法的完整名称。它用于
+    错误消息，以识别哪个中间件方法引发了异常或返回了无效输出。
+
+    Args:
+        f: The method to get the name of.
+           要获取名称的方法。
+
+    Returns:
+        str: The full name of the method in the format "ClassName.method_name".
+             格式为"ClassName.method_name"的方法的完整名称。
+    """
+    # Get the class name and method name and format them
+    # 获取类名和方法名并格式化它们
     return "{}.{}".format(
-        f.__self__.__class__.__name__,
-        f.__func__.__name__
+        f.__self__.__class__.__name__,  # Class name 类名
+        f.__func__.__name__             # Method name 方法名
     )
 
 
 class SpiderMiddlewareManager(AbsMiddlewareManager):
+    """
+    Manager for spider middleware components.
+    爬虫中间件组件的管理器。
+
+    This class manages the execution of spider middleware components, which process
+    the input and output of spiders. It inherits from AbsMiddlewareManager and
+    implements the specific behavior for spider middlewares.
+    此类管理爬虫中间件组件的执行，这些组件处理爬虫的输入和输出。它继承自
+    AbsMiddlewareManager，并实现了爬虫中间件的特定行为。
+
+    Spider middlewares can intercept and modify the requests generated by spiders,
+    the responses passed to spiders, and the items and requests yielded by spiders.
+    They can also handle exceptions that occur during spider processing.
+    爬虫中间件可以拦截和修改由爬虫生成的请求、传递给爬虫的响应，以及由爬虫产生的
+    项目和请求。它们还可以处理爬虫处理过程中发生的异常。
+    """
+
+    # Name of the component
+    # 组件的名称
     component_name = 'spider middleware'
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings: Settings):
+        """
+        Get the list of spider middleware classes from settings.
+        从设置中获取爬虫中间件类列表。
+
+        This method implements the abstract method from AbsMiddlewareManager.
+        It retrieves the list of spider middleware classes from the
+        SPIDER_MIDDLEWARES setting.
+        此方法实现了AbsMiddlewareManager中的抽象方法。它从SPIDER_MIDDLEWARES
+        设置中检索爬虫中间件类列表。
+
+        Args:
+            settings: The settings object.
+                     设置对象。
+
+        Returns:
+            list: A list of spider middleware class paths.
+                 爬虫中间件类路径列表。
+        """
+        # Build component list from SPIDER_MIDDLEWARES setting
+        # 从SPIDER_MIDDLEWARES设置构建组件列表
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
+        """
+        Add a middleware instance to the manager.
+        将中间件实例添加到管理器。
+
+        This method overrides the method from AbsMiddlewareManager to register
+        the specific methods of spider middlewares: process_spider_input,
+        process_start_requests, process_spider_output, and process_spider_exception.
+        此方法覆盖了AbsMiddlewareManager中的方法，以注册爬虫中间件的特定方法：
+        process_spider_input、process_start_requests、process_spider_output和
+        process_spider_exception。
+
+        Note that process_spider_input and process_start_requests methods are called
+        in the order they are registered, while process_spider_output and
+        process_spider_exception methods are called in reverse order.
+        请注意，process_spider_input和process_start_requests方法按照它们注册的
+        顺序调用，而process_spider_output和process_spider_exception方法按照
+        相反的顺序调用。
+
+        Args:
+            mw: The middleware instance to add.
+                要添加的中间件实例。
+        """
+        # Call parent method to register open_spider and close_spider methods
+        # 调用父方法来注册open_spider和close_spider方法
         super(SpiderMiddlewareManager, self)._add_middleware(mw)
+
+        # Register process_spider_input method if it exists
+        # 如果存在，则注册process_spider_input方法
         if hasattr(mw, 'process_spider_input'):
             self.methods['process_spider_input'].append(mw.process_spider_input)
+
+        # Register process_start_requests method if it exists (added to the left for reverse order)
+        # 如果存在，则注册process_start_requests方法（添加到左侧以便逆序执行）
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].appendleft(mw.process_start_requests)
+
+        # Get process_spider_output method if it exists
+        # 如果存在，则获取process_spider_output方法
         process_spider_output = getattr(mw, 'process_spider_output', None)
+
+        # Register process_spider_output method (added to the left for reverse order)
+        # 注册process_spider_output方法（添加到左侧以便逆序执行）
         self.methods['process_spider_output'].appendleft(process_spider_output)
+
+        # Get process_spider_exception method if it exists
+        # 如果存在，则获取process_spider_exception方法
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
+
+        # Register process_spider_exception method (added to the left for reverse order)
+        # 注册process_spider_exception方法（添加到左侧以便逆序执行）
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
 
     async def scrape_response(self, scrape_func: Callable, response: Response, request: Request, spider: Spider):
+        """
+        Process a response through the spider middleware chain.
+        通过爬虫中间件链处理响应。
+
+        This method is the core of the spider middleware processing. It handles the
+        entire process of passing a response through the middleware chain, calling
+        the spider's callback function, and processing the output through the
+        middleware chain again.
+        此方法是爬虫中间件处理的核心。它处理通过中间件链传递响应、调用爬虫的回调
+        函数以及再次通过中间件链处理输出的整个过程。
+
+        Args:
+            scrape_func: The spider's callback function to process the response.
+                        处理响应的爬虫回调函数。
+            response: The response to process.
+                     要处理的响应。
+            request: The request that generated the response.
+                    生成响应的请求。
+            spider: The spider that made the request.
+                   发出请求的爬虫。
 
+        Returns:
+            The result of processing the response through the middleware chain.
+            通过中间件链处理响应的结果。
+        """
         async def process_spider_input(response_) -> Union[AsyncIterable, Iterable]:
+            """
+            Process a response through all registered process_spider_input methods.
+            通过所有已注册的process_spider_input方法处理响应。
+
+            This function calls each middleware's process_spider_input method in the
+            order they were registered. If any middleware raises an exception, it is
+            handled by the scrape_func.
+            此函数按照它们注册的顺序调用每个中间件的process_spider_input方法。
+            如果任何中间件引发异常，则由scrape_func处理。
+
+            Args:
+                response_: The response to process.
+                          要处理的响应。
+
+            Returns:
+                Union[AsyncIterable, Iterable]: The result of processing the response.
+                                              处理响应的结果。
+            """
+            # Process the response through all registered process_spider_input methods
+            # 通过所有已注册的process_spider_input方法处理响应
             for method in self.methods['process_spider_input']:
                 try:
+                    # Call the method with the response
+                    # 使用响应调用方法
                     result = await call_helper(method, response=response_, spider=spider)
+
+                    # Validate the return value
+                    # 验证返回值
                     if result is not None:
                         raise _InvalidOutput(
                             f"Middleware {_fname(method)} must return None or raise an exception, got {type(result)}"
                         )
                 except _InvalidOutput:
+                    # Re-raise _InvalidOutput exceptions
+                    # 重新引发_InvalidOutput异常
                     raise
                 except BaseException as exception:
+                    # Handle other exceptions by calling the scrape_func with the exception
+                    # 通过使用异常调用scrape_func来处理其他异常
                     iterable_or_exception = await call_helper(scrape_func, exception, request)
+
+                    # If the scrape_func returned the exception, re-raise it
+                    # 如果scrape_func返回了异常，则重新引发它
                     if iterable_or_exception is exception:
                         raise iterable_or_exception
+
+                    # Otherwise, return the result
+                    # 否则，返回结果
                     return iterable_or_exception
+
+            # If all middleware methods passed, call the scrape_func with the response
+            # 如果所有中间件方法都通过，则使用响应调用scrape_func
             return await call_helper(scrape_func, response_, request)
 
         async def _evaluate_iterable(result: Union[AsyncIterable, Iterable], exception_processor_index):
+            """
+            Evaluate an iterable and handle any exceptions that occur.
+            评估可迭代对象并处理发生的任何异常。
+
+            This function converts any iterable to an async generator and yields
+            its items. If an exception occurs during iteration, it is handled by
+            the process_spider_exception function.
+            此函数将任何可迭代对象转换为异步生成器并产生其项目。如果在迭代期间
+            发生异常，则由process_spider_exception函数处理。
+
+            Args:
+                result: The iterable to evaluate.
+                       要评估的可迭代对象。
+                exception_processor_index: The index to start processing exceptions from.
+                                          开始处理异常的索引。
+
+            Yields:
+                The items from the iterable.
+                可迭代对象中的项目。
+            """
             try:
+                # Convert all non-AsyncGeneratorType to AsyncGeneratorType objects
                 # 将所有非AsyncGeneratorType变成AsyncGeneratorType对象
                 async for r in await async_generator_wrapper(result):
                     yield r
             except BaseException as ex:
+                # Handle exceptions by calling process_spider_exception
+                # 通过调用process_spider_exception处理异常
                 exception_result = await process_spider_exception(ex, exception_processor_index)
+
+                # If the result is an exception, re-raise it
+                # 如果结果是异常，则重新引发它
                 if isinstance(exception_result, BaseException):
                     raise exception_result
+
+                # Otherwise, recursively evaluate the result
+                # 否则，递归评估结果
                 async for r in _evaluate_iterable(exception_result, exception_processor_index):
                     yield r
 
         async def process_spider_exception(exception, start_index=0):
-            # don't handle _InvalidOutput exception
+            """
+            Process an exception through all registered process_spider_exception methods.
+            通过所有已注册的process_spider_exception方法处理异常。
+
+            This function calls each middleware's process_spider_exception method in
+            reverse order from how they were registered. If any middleware returns an
+            iterable, the exception handling stops and the iterable is processed by
+            the process_spider_output function.
+            此函数按照与它们注册的相反的顺序调用每个中间件的process_spider_exception方法。
+            如果任何中间件返回可迭代对象，则异常处理停止，并且可迭代对象由
+            process_spider_output函数处理。
+
+            Args:
+                exception: The exception to process.
+                          要处理的异常。
+                start_index: The index to start processing from.
+                            开始处理的索引。
+
+            Returns:
+                The result of processing the exception, or the exception itself if
+                no middleware handled it.
+                处理异常的结果，如果没有中间件处理它，则为异常本身。
+            """
+            # Don't handle _InvalidOutput exceptions
+            # 不处理_InvalidOutput异常
             if isinstance(exception, _InvalidOutput):
                 raise exception
+
+            # Get the list of methods to call, starting from start_index
+            # 获取要调用的方法列表，从start_index开始
             method_list = islice(self.methods['process_spider_exception'], start_index, None)
+
+            # Process each method
+            # 处理每个方法
             for method_index, method in enumerate(method_list, start=start_index):
+                # Skip None methods
+                # 跳过None方法
                 if method is None:
                     continue
+
+                # Call the method with the exception
+                # 使用异常调用方法
                 result = await call_helper(method, response=response, exception=exception, spider=spider)
+
+                # If the result is an iterable, stop exception handling and process the output
+                # 如果结果是可迭代对象，则停止异常处理并处理输出
                 if isinstance(result, AsyncIterable):
-                    # stop exception handling by handing control over to the
+                    # Stop exception handling by handing control over to the
                     # process_spider_output chain if an iterable has been returned
+                    # 如果返回了可迭代对象，则通过将控制权交给process_spider_output链来停止异常处理
                     return await process_spider_output(result, method_index + 1)
                 elif result is None:
+                    # If the result is None, continue to the next method
+                    # 如果结果为None，则继续下一个方法
                     continue
                 else:
+                    # If the result is neither an iterable nor None, raise an error
+                    # 如果结果既不是可迭代对象也不是None，则引发错误
                     raise _InvalidOutput(
                             f"Middleware {_fname(method)}  must return None or an iterable, got {type(result)}"
                         )
+
+            # If no middleware handled the exception, re-raise it
+            # 如果没有中间件处理异常，则重新引发它
             raise exception
 
         async def process_spider_output(result, start_index=0):
-            # items in this iterable do not need to go through the process_spider_output
+            """
+            Process an iterable through all registered process_spider_output methods.
+            通过所有已注册的process_spider_output方法处理可迭代对象。
+
+            This function calls each middleware's process_spider_output method in
+            reverse order from how they were registered. Each middleware can modify
+            the iterable or return a new one.
+            此函数按照与它们注册的相反的顺序调用每个中间件的process_spider_output方法。
+            每个中间件可以修改可迭代对象或返回一个新的可迭代对象。
+
+            Args:
+                result: The iterable to process.
+                       要处理的可迭代对象。
+                start_index: The index to start processing from.
+                            开始处理的索引。
+
+            Returns:
+                The processed iterable.
+                处理后的可迭代对象。
+            """
+            # Items in this iterable do not need to go through the process_spider_output
             # chain, they went through it already from the process_spider_exception method
+            # 此可迭代对象中的项目不需要通过process_spider_output链，
+            # 它们已经通过process_spider_exception方法通过了它
 
+            # Get the list of methods to call, starting from start_index
+            # 获取要调用的方法列表，从start_index开始
             method_list = islice(self.methods['process_spider_output'], start_index, None)
+
+            # Process each method
+            # 处理每个方法
             for method_index, method in enumerate(method_list, start=start_index):
+                # Skip None methods
+                # 跳过None方法
                 if method is None:
                     continue
+
                 try:
-                    # might fail directly if the output value is not a generator
+                    # Call the method with the result
+                    # 使用结果调用方法
+                    # Might fail directly if the output value is not a generator
+                    # 如果输出值不是生成器，可能会直接失败
                     result = await call_helper(method, response=response, result=result, spider=spider)
                 except BaseException as ex:
+                    # Handle exceptions by calling process_spider_exception
+                    # 通过调用process_spider_exception处理异常
                     exception_result = await process_spider_exception(ex, method_index + 1)
+
+                    # If the result is an exception, re-raise it
+                    # 如果结果是异常，则重新引发它
                     if isinstance(exception_result, BaseException):
                         raise
+
+                    # Otherwise, return the result
+                    # 否则，返回结果
                     return exception_result
+
+                # Validate the return value
+                # 验证返回值
                 if isinstance(result, AsyncIterable):
+                    # If the result is an iterable, evaluate it
+                    # 如果结果是可迭代对象，则评估它
                     result = _evaluate_iterable(result, method_index + 1)
                 else:
+                    # If the result is not an iterable, raise an error
+                    # 如果结果不是可迭代对象，则引发错误
                     raise _InvalidOutput(f"Middleware {_fname(method)} must return an iterable, got {type(result)}")
 
+            # Return the final result
+            # 返回最终结果
             return result
 
         async def process_callback_output(result: Union[AsyncIterable, Iterable]):
+            """
+            Process the output of the spider's callback function.
+            处理爬虫回调函数的输出。
+
+            This function evaluates the iterable returned by the spider's callback
+            function and processes it through the process_spider_output chain.
+            此函数评估爬虫回调函数返回的可迭代对象，并通过process_spider_output链处理它。
+
+            Args:
+                result: The iterable returned by the spider's callback function.
+                       爬虫回调函数返回的可迭代对象。
+
+            Returns:
+                The processed iterable.
+                处理后的可迭代对象。
+            """
+            # Evaluate the iterable
+            # 评估可迭代对象
             result: AsyncIterable = _evaluate_iterable(result, 0)
+
+            # Process the result through the process_spider_output chain
+            # 通过process_spider_output链处理结果
             return await process_spider_output(result)
 
         try:
+            # Process the response through the process_spider_input chain
+            # 通过process_spider_input链处理响应
             _iterable = await process_spider_input(response)
         except BaseException as exc:
+            # If an exception occurs, process it through the process_spider_exception chain
+            # 如果发生异常，则通过process_spider_exception链处理它
             return await process_spider_exception(exc)
         else:
+            # If no exception occurs, process the output through the process_callback_output chain
+            # 如果没有发生异常，则通过process_callback_output链处理输出
             return await process_callback_output(_iterable)
 
     async def process_start_requests(self, start_requests, spider):
+        """
+        Process start requests through all registered process_start_requests methods.
+        通过所有已注册的process_start_requests方法处理起始请求。
+
+        This method calls each middleware's process_start_requests method in the
+        order they were registered. Each middleware can modify the start requests
+        or return a new iterable of requests.
+        此方法按照它们注册的顺序调用每个中间件的process_start_requests方法。
+        每个中间件可以修改起始请求或返回一个新的请求可迭代对象。
+
+        Args:
+            start_requests: The iterable of start requests from the spider.
+                           来自爬虫的起始请求可迭代对象。
+            spider: The spider that generated the start requests.
+                   生成起始请求的爬虫。
+
+        Returns:
+            The processed iterable of start requests.
+            处理后的起始请求可迭代对象。
+        """
+        # Process the start requests through the chain of process_start_requests methods
+        # 通过process_start_requests方法链处理起始请求
         return await self._process_chain('process_start_requests', start_requests, spider)