aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/middleware/downloader.py

@@ -1,3 +1,17 @@
+"""
+Downloader Middleware Manager Module
+下载器中间件管理器模块
+
+This module provides the DownloaderMiddlewareManager class, which manages the execution
+of downloader middleware components. Downloader middlewares process requests before they
+are sent to the downloader and responses before they are sent to the spider.
+此模块提供了DownloaderMiddlewareManager类，用于管理下载器中间件组件的执行。
+下载器中间件在请求发送到下载器之前处理请求，在响应发送到爬虫之前处理响应。
+
+Downloader middlewares can modify requests and responses, return alternative responses,
+or handle exceptions that occur during the download process.
+下载器中间件可以修改请求和响应，返回替代响应，或处理下载过程中发生的异常。
+"""
 from aioscrapy.exceptions import _InvalidOutput
 from aioscrapy.http import Request, Response
 from aioscrapy.middleware.absmanager import AbsMiddlewareManager
@@ -6,64 +20,268 @@ from aioscrapy.utils.tools import call_helper
 
 
 class DownloaderMiddlewareManager(AbsMiddlewareManager):
+    """
+    Manager for downloader middleware components.
+    下载器中间件组件的管理器。
+
+    This class manages the execution of downloader middleware components, which process
+    requests before they are sent to the downloader and responses before they are sent
+    to the spider. It inherits from AbsMiddlewareManager and implements the specific
+    behavior for downloader middlewares.
+    此类管理下载器中间件组件的执行，这些组件在请求发送到下载器之前处理请求，
+    在响应发送到爬虫之前处理响应。它继承自AbsMiddlewareManager，并实现了
+    下载器中间件的特定行为。
+    """
+
+    # Name of the middleware component
+    # 中间件组件的名称
     component_name = 'downloader middleware'
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
+        """
+        Get the list of middleware classes from settings.
+        从设置中获取中间件类列表。
+
+        This method implements the abstract method from AbsMiddlewareManager.
+        It retrieves the list of downloader middleware classes from the
+        DOWNLOADER_MIDDLEWARES setting.
+        此方法实现了AbsMiddlewareManager中的抽象方法。它从DOWNLOADER_MIDDLEWARES
+        设置中检索下载器中间件类列表。
+
+        Args:
+            settings: The settings object.
+                     设置对象。
+
+        Returns:
+            list: A list of middleware class paths.
+                 中间件类路径列表。
+        """
+        # Build component list from DOWNLOADER_MIDDLEWARES setting
+        # 从DOWNLOADER_MIDDLEWARES设置构建组件列表
         return build_component_list(
             settings.getwithbase('DOWNLOADER_MIDDLEWARES'))
 
     def _add_middleware(self, mw):
+        """
+        Add a middleware instance to the manager.
+        将中间件实例添加到管理器。
+
+        This method overrides the method from AbsMiddlewareManager to register
+        the specific methods of downloader middlewares: process_request,
+        process_response, and process_exception.
+        此方法覆盖了AbsMiddlewareManager中的方法，以注册下载器中间件的特定方法：
+        process_request、process_response和process_exception。
+
+        Note that process_request methods are called in the order they are registered,
+        while process_response and process_exception methods are called in reverse order.
+        请注意，process_request方法按照它们注册的顺序调用，而process_response
+        和process_exception方法按照相反的顺序调用。
+
+        Args:
+            mw: The middleware instance to add.
+                要添加的中间件实例。
+        """
+        # Register process_request method if it exists
+        # 如果存在，则注册process_request方法
         if hasattr(mw, 'process_request'):
             self.methods['process_request'].append(mw.process_request)
+
+        # Register process_response method if it exists (added to the left for reverse order)
+        # 如果存在，则注册process_response方法（添加到左侧以便逆序执行）
         if hasattr(mw, 'process_response'):
             self.methods['process_response'].appendleft(mw.process_response)
+
+        # Register process_exception method if it exists (added to the left for reverse order)
+        # 如果存在，则注册process_exception方法（添加到左侧以便逆序执行）
         if hasattr(mw, 'process_exception'):
             self.methods['process_exception'].appendleft(mw.process_exception)
 
     def iter_mw_method(self, spider, process_type: str):
+        """
+        Iterate over middleware methods of a specific type.
+        迭代特定类型的中间件方法。
+
+        This method yields all middleware methods of the specified type, followed
+        by the spider's method of the same type if it exists.
+        此方法产生指定类型的所有中间件方法，如果存在，则后跟爬虫的同类型方法。
+
+        Args:
+            spider: The spider instance.
+                   爬虫实例。
+            process_type: The type of middleware method to iterate over.
+                         要迭代的中间件方法类型。
+
+        Yields:
+            callable: Middleware methods of the specified type.
+                     指定类型的中间件方法。
+        """
+        # Get the spider's method of the specified type if it exists
+        # 如果存在，则获取爬虫的指定类型方法
         spider_method = getattr(spider, process_type, None)
+
+        # Yield all middleware methods of the specified type
+        # 产生指定类型的所有中间件方法
         for method in self.methods[process_type]:
             yield method
+
+        # Yield the spider's method if it exists
+        # 如果存在，则产生爬虫的方法
         if spider_method:
             yield spider_method
 
     async def process_request(self, spider, request):
+        """
+        Process a request through all registered process_request methods.
+        通过所有已注册的process_request方法处理请求。
+
+        This method calls each middleware's process_request method in the order
+        they were registered. If any middleware returns a Response or Request,
+        the process stops and that object is returned.
+        此方法按照它们注册的顺序调用每个中间件的process_request方法。如果任何
+        中间件返回Response或Request，则过程停止并返回该对象。
+
+        Args:
+            spider: The spider instance.
+                   爬虫实例。
+            request: The request to process.
+                    要处理的请求。
+
+        Returns:
+            None, Response, or Request: The result of processing the request.
+                                       处理请求的结果。
+
+        Raises:
+            _InvalidOutput: If a middleware returns a value that is not None, Response, or Request.
+                           如果中间件返回的值不是None、Response或Request。
+        """
+        # Iterate over all process_request methods
+        # 迭代所有process_request方法
         for method in self.iter_mw_method(spider, 'process_request'):
+            # Call the method with the request
+            # 使用请求调用方法
             response = await call_helper(method, request=request, spider=spider)
+
+            # Validate the return value
+            # 验证返回值
             if response is not None and not isinstance(response, (Response, Request)):
                 raise _InvalidOutput(
                     "Middleware %s.process_request must return None, Response or Request, got %s"
                     % (method.__self__.__class__.__name__, response.__class__.__name__)
                 )
+
+            # If a non-None value was returned, return it and stop processing
+            # 如果返回了非None值，则返回它并停止处理
             if response:
                 return response
 
     async def process_response(self, spider, request, response):
+        """
+        Process a response through all registered process_response methods.
+        通过所有已注册的process_response方法处理响应。
+
+        This method calls each middleware's process_response method in reverse
+        order from how they were registered. If any middleware returns a Request,
+        the process stops and that Request is returned.
+        此方法按照与它们注册的相反的顺序调用每个中间件的process_response方法。
+        如果任何中间件返回Request，则过程停止并返回该Request。
+
+        Args:
+            spider: The spider instance.
+                   爬虫实例。
+            request: The request that generated the response.
+                    生成响应的请求。
+            response: The response to process.
+                     要处理的响应。
+
+        Returns:
+            Response or Request: The result of processing the response.
+                                处理响应的结果。
+
+        Raises:
+            TypeError: If response is None.
+                      如果响应为None。
+            _InvalidOutput: If a middleware returns a value that is not Response or Request.
+                           如果中间件返回的值不是Response或Request。
+        """
+        # Validate the response
+        # 验证响应
         if response is None:
             raise TypeError("Received None in process_response")
         elif isinstance(response, Request):
             return response
 
+        # Iterate over all process_response methods
+        # 迭代所有process_response方法
         for method in self.iter_mw_method(spider, 'process_response'):
+            # Call the method with the request and response
+            # 使用请求和响应调用方法
             response = await call_helper(method, request=request, response=response, spider=spider)
+
+            # Validate the return value
+            # 验证返回值
             if not isinstance(response, (Response, Request)):
                 raise _InvalidOutput(
                     "Middleware %s.process_response must return Response or Request, got %s"
                     % (method.__self__.__class__.__name__, type(response))
                 )
+
+            # If a Request was returned, return it and stop processing
+            # 如果返回了Request，则返回它并停止处理
             if isinstance(response, Request):
                 return response
+
+        # Return the final response
+        # 返回最终响应
         return response
 
     async def process_exception(self, spider, request, exception):
+        """
+        Process an exception through all registered process_exception methods.
+        通过所有已注册的process_exception方法处理异常。
+
+        This method calls each middleware's process_exception method in reverse
+        order from how they were registered. If any middleware returns a Response
+        or Request, the process stops and that object is returned.
+        此方法按照与它们注册的相反的顺序调用每个中间件的process_exception方法。
+        如果任何中间件返回Response或Request，则过程停止并返回该对象。
+
+        Args:
+            spider: The spider instance.
+                   爬虫实例。
+            request: The request that caused the exception.
+                    导致异常的请求。
+            exception: The exception that occurred.
+                      发生的异常。
+
+        Returns:
+            Exception, Response, or Request: The result of processing the exception.
+                                           处理异常的结果。
+
+        Raises:
+            _InvalidOutput: If a middleware returns a value that is not None, Response, or Request.
+                           如果中间件返回的值不是None、Response或Request。
+        """
+        # Iterate over all process_exception methods
+        # 迭代所有process_exception方法
         for method in self.iter_mw_method(spider, 'process_exception'):
+            # Call the method with the request and exception
+            # 使用请求和异常调用方法
             response = await call_helper(method, request=request, exception=exception, spider=spider)
+
+            # Validate the return value
+            # 验证返回值
             if response is not None and not isinstance(response, (Response, Request)):
                 raise _InvalidOutput(
                     "Middleware %s.process_exception must return None, Response or Request, got %s"
                     % (method.__self__.__class__.__name__, type(response))
                 )
+
+            # If a non-None value was returned, return it and stop processing
+            # 如果返回了非None值，则返回它并停止处理
             if response:
                 return response
+
+        # If no middleware handled the exception, return it
+        # 如果没有中间件处理异常，则返回它
         return exception

aioscrapy/middleware/extension.py

@@ -1,16 +1,65 @@
 """
 The Extension Manager
+扩展管理器
 
-See documentation in docs/topics/extensions.rst
+This module provides the ExtensionManager class, which manages the loading and
+execution of extensions. Extensions are components that can hook into various
+parts of the Scrapy process to add functionality or modify behavior.
+此模块提供了ExtensionManager类，用于管理扩展的加载和执行。扩展是可以挂钩到
+Scrapy流程的各个部分以添加功能或修改行为的组件。
+
+Extensions are loaded from the EXTENSIONS setting and can be enabled or disabled
+through this setting. They can connect to signals to execute code at specific
+points in the crawling process.
+扩展从EXTENSIONS设置加载，可以通过此设置启用或禁用。它们可以连接到信号，
+以在爬取过程的特定点执行代码。
 """
 from aioscrapy.middleware.absmanager import AbsMiddlewareManager
 from aioscrapy.utils.conf import build_component_list
 
 
 class ExtensionManager(AbsMiddlewareManager):
+    """
+    Manager for extension components.
+    扩展组件的管理器。
+
+    This class manages the loading and execution of extensions. It inherits from
+    AbsMiddlewareManager and implements the specific behavior for extensions.
+    Extensions are components that can hook into various parts of the Scrapy
+    process to add functionality or modify behavior.
+    此类管理扩展的加载和执行。它继承自AbsMiddlewareManager，并实现了扩展的特定行为。
+    扩展是可以挂钩到Scrapy流程的各个部分以添加功能或修改行为的组件。
+
+    Extensions typically connect to signals to execute code at specific points in
+    the crawling process. They can be enabled or disabled through the EXTENSIONS
+    setting.
+    扩展通常连接到信号，以在爬取过程的特定点执行代码。它们可以通过EXTENSIONS设置
+    启用或禁用。
+    """
 
+    # Name of the component
+    # 组件的名称
     component_name = 'extension'
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
+        """
+        Get the list of extension classes from settings.
+        从设置中获取扩展类列表。
+
+        This method implements the abstract method from AbsMiddlewareManager.
+        It retrieves the list of extension classes from the EXTENSIONS setting.
+        此方法实现了AbsMiddlewareManager中的抽象方法。它从EXTENSIONS设置中检索
+        扩展类列表。
+
+        Args:
+            settings: The settings object.
+                     设置对象。
+
+        Returns:
+            list: A list of extension class paths.
+                 扩展类路径列表。
+        """
+        # Build component list from EXTENSIONS setting
+        # 从EXTENSIONS设置构建组件列表
         return build_component_list(settings.getwithbase('EXTENSIONS'))

aioscrapy/middleware/itempipeline.py

@@ -1,18 +1,114 @@
+"""
+Item Pipeline Manager Module
+项目管道管理器模块
+
+This module provides the ItemPipelineManager class, which manages the execution
+of item pipeline components. Item pipelines are components that process items
+after they have been extracted by spiders, typically for cleaning, validation,
+persistence, or other post-processing tasks.
+此模块提供了ItemPipelineManager类，用于管理项目管道组件的执行。项目管道是
+在项目被爬虫提取后处理项目的组件，通常用于清洗、验证、持久化或其他后处理任务。
+
+Item pipelines are loaded from the ITEM_PIPELINES setting and are executed in
+the order specified by their priority values. Each pipeline component can process
+an item and either return it for further processing, drop it, or raise an exception.
+项目管道从ITEM_PIPELINES设置加载，并按照其优先级值指定的顺序执行。每个管道组件
+可以处理一个项目，并返回它以供进一步处理、丢弃它或引发异常。
+"""
 from aioscrapy.middleware.absmanager import AbsMiddlewareManager
 from aioscrapy.utils.conf import build_component_list
 
 
 class ItemPipelineManager(AbsMiddlewareManager):
+    """
+    Manager for item pipeline components.
+    项目管道组件的管理器。
+
+    This class manages the execution of item pipeline components, which process items
+    after they have been extracted by spiders. It inherits from AbsMiddlewareManager
+    and implements the specific behavior for item pipelines.
+    此类管理项目管道组件的执行，这些组件在项目被爬虫提取后进行处理。它继承自
+    AbsMiddlewareManager，并实现了项目管道的特定行为。
+
+    Item pipelines are executed in the order specified by their priority values in
+    the ITEM_PIPELINES setting. Each pipeline can process an item and either return
+    it for further processing, drop it, or raise an exception.
+    项目管道按照ITEM_PIPELINES设置中指定的优先级值顺序执行。每个管道可以处理一个
+    项目，并返回它以供进一步处理、丢弃它或引发异常。
+    """
+
+    # Name of the component
+    # 组件的名称
     component_name = 'item pipeline'
 
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
+        """
+        Get the list of item pipeline classes from settings.
+        从设置中获取项目管道类列表。
+
+        This method implements the abstract method from AbsMiddlewareManager.
+        It retrieves the list of item pipeline classes from the ITEM_PIPELINES setting.
+        此方法实现了AbsMiddlewareManager中的抽象方法。它从ITEM_PIPELINES设置中
+        检索项目管道类列表。
+
+        Args:
+            settings: The settings object.
+                     设置对象。
+
+        Returns:
+            list: A list of item pipeline class paths.
+                 项目管道类路径列表。
+        """
+        # Build component list from ITEM_PIPELINES setting
+        # 从ITEM_PIPELINES设置构建组件列表
         return build_component_list(settings.getwithbase('ITEM_PIPELINES'))
 
     def _add_middleware(self, pipe):
+        """
+        Add a pipeline instance to the manager.
+        将管道实例添加到管理器。
+
+        This method overrides the method from AbsMiddlewareManager to register
+        the process_item method of item pipelines. It first calls the parent method
+        to register open_spider and close_spider methods if they exist.
+        此方法覆盖了AbsMiddlewareManager中的方法，以注册项目管道的process_item方法。
+        它首先调用父方法来注册open_spider和close_spider方法（如果存在）。
+
+        Args:
+            pipe: The pipeline instance to add.
+                 要添加的管道实例。
+        """
+        # Call parent method to register open_spider and close_spider methods
+        # 调用父方法来注册open_spider和close_spider方法
         super()._add_middleware(pipe)
+
+        # Register process_item method if it exists
+        # 如果存在，则注册process_item方法
         if hasattr(pipe, 'process_item'):
             self.methods['process_item'].append(pipe.process_item)
 
     async def process_item(self, item, spider):
+        """
+        Process an item through all registered process_item methods.
+        通过所有已注册的process_item方法处理项目。
+
+        This method calls each pipeline's process_item method in the order they
+        were registered. The result of each pipeline is passed to the next one
+        in a chain, allowing pipelines to modify the item or drop it by returning None.
+        此方法按照它们注册的顺序调用每个管道的process_item方法。每个管道的结果
+        以链式方式传递给下一个管道，允许管道修改项目或通过返回None来丢弃它。
+
+        Args:
+            item: The item to process.
+                 要处理的项目。
+            spider: The spider that generated the item.
+                   生成项目的爬虫。
+
+        Returns:
+            The processed item, or None if it was dropped by a pipeline.
+            处理后的项目，如果被管道丢弃则为None。
+        """
+        # Process the item through the chain of process_item methods
+        # 通过process_item方法链处理项目
         return await self._process_chain('process_item', item, spider)