aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/signalmanager.py

@@ -1,68 +1,199 @@
+"""
+Signal Manager for AioScrapy
+AioScrapy的信号管理器
+
+This module provides the SignalManager class which is responsible for coordinating
+signals and receivers in the AioScrapy framework. It wraps PyDispatcher to provide
+a more convenient API for connecting, disconnecting, and sending signals.
+此模块提供了SignalManager类，负责协调AioScrapy框架中的信号和接收器。
+它封装了PyDispatcher，提供了更方便的API用于连接、断开和发送信号。
+"""
+
 from pydispatch import dispatcher
 from aioscrapy.utils import signal as _signal
 
 
 class SignalManager:
+    """
+    Class for managing signals in AioScrapy.
+    用于管理AioScrapy中信号的类。
+
+    This class provides methods to connect and disconnect receivers to signals,
+    as well as to send signals with proper exception handling. It serves as a
+    wrapper around PyDispatcher, providing a more convenient API for AioScrapy.
+    此类提供了将接收器连接到信号和断开连接的方法，以及发送带有适当异常处理的信号的方法。
+    它作为PyDispatcher的包装器，为AioScrapy提供了更方便的API。
+    """
 
     def __init__(self, sender=dispatcher.Anonymous):
+        """
+        Initialize a SignalManager.
+        初始化一个SignalManager。
+
+        Args:
+            sender: The default sender to use when connecting or sending signals.
+                   连接或发送信号时使用的默认发送者。
+                   Defaults to dispatcher.Anonymous, which is a special object
+                   used to identify anonymous senders.
+                   默认为dispatcher.Anonymous，这是一个用于标识匿名发送者的特殊对象。
+        """
         self.sender = sender
 
     def connect(self, receiver, signal, **kwargs):
         """
         Connect a receiver function to a signal.
+        将接收器函数连接到信号。
 
-        The signal can be any object, although Scrapy comes with some
-        predefined signals that are documented in the :ref:`topics-signals`
-        section.
+        This method connects a receiver function to a signal so that the function
+        is called when the signal is sent. The receiver function will receive the
+        signal object and any additional keyword arguments passed when the signal
+        is sent.
+        此方法将接收器函数连接到信号，以便在发送信号时调用该函数。
+        接收器函数将接收信号对象和发送信号时传递的任何其他关键字参数。
 
-        :param receiver: the function to be connected
-        :type receiver: callable
+        Args:
+            receiver: The function to be connected to the signal.
+                     要连接到信号的函数。
+                     This function will be called when the signal is sent.
+                     当信号发送时，将调用此函数。
+            signal: The signal to connect to.
+                   要连接的信号。
+                   This can be any object, although AioScrapy comes with predefined
+                   signals in the aioscrapy.signals module.
+                   这可以是任何对象，尽管AioScrapy在aioscrapy.signals模块中
+                   提供了预定义的信号。
+            **kwargs: Additional keyword arguments to pass to PyDispatcher's connect.
+                     传递给PyDispatcher的connect的其他关键字参数。
 
-        :param signal: the signal to connect to
-        :type signal: object
+        Returns:
+            bool: True if the receiver was successfully connected, False otherwise.
+                 如果接收器成功连接，则为True，否则为False。
         """
+        # Set the default sender if not provided
+        # 如果未提供，则设置默认发送者
         kwargs.setdefault('sender', self.sender)
+        # Connect the receiver to the signal using PyDispatcher
+        # 使用PyDispatcher将接收器连接到信号
         return dispatcher.connect(receiver, signal, **kwargs)
 
     def disconnect(self, receiver, signal, **kwargs):
         """
-        Disconnect a receiver function from a signal. This has the
-        opposite effect of the :meth:`connect` method, and the arguments
-        are the same.
+        Disconnect a receiver function from a signal.
+        断开接收器函数与信号的连接。
+
+        This method disconnects a previously connected receiver function from a signal.
+        It has the opposite effect of the connect method, and the arguments are the same.
+        此方法断开先前连接到信号的接收器函数。
+        它具有与connect方法相反的效果，参数相同。
+
+        Args:
+            receiver: The function to be disconnected from the signal.
+                     要从信号断开连接的函数。
+            signal: The signal to disconnect from.
+                   要断开连接的信号。
+            **kwargs: Additional keyword arguments to pass to PyDispatcher's disconnect.
+                     传递给PyDispatcher的disconnect的其他关键字参数。
+
+        Returns:
+            bool: True if the receiver was successfully disconnected, False otherwise.
+                 如果接收器成功断开连接，则为True，否则为False。
+                 False might indicate that the receiver was not connected to the signal.
+                 False可能表示接收器未连接到信号。
         """
+        # Set the default sender if not provided
+        # 如果未提供，则设置默认发送者
         kwargs.setdefault('sender', self.sender)
+        # Disconnect the receiver from the signal using PyDispatcher
+        # 使用PyDispatcher断开接收器与信号的连接
         return dispatcher.disconnect(receiver, signal, **kwargs)
 
     async def send_catch_log(self, signal, **kwargs):
         """
         Send a signal, catch exceptions and log them.
+        发送信号，捕获异常并记录它们。
+
+        This method sends a signal to all connected receivers. If a receiver raises
+        an exception, it is caught and logged, but the signal continues to be sent
+        to other receivers. This ensures that one failing receiver doesn't prevent
+        other receivers from receiving the signal.
+        此方法向所有连接的接收器发送信号。如果接收器引发异常，
+        则会捕获并记录该异常，但信号继续发送给其他接收器。
+        这确保一个失败的接收器不会阻止其他接收器接收信号。
+
+        Args:
+            signal: The signal to send.
+                   要发送的信号。
+            **kwargs: Keyword arguments to pass to the signal handlers.
+                     传递给信号处理程序的关键字参数。
 
-        The keyword arguments are passed to the signal handlers (connected
-        through the :meth:`connect` method).
+        Returns:
+            list: A list of (receiver, response) tuples, where response is either
+                 the return value of the handler or the exception that was caught.
+                 (接收器, 响应)元组的列表，其中响应是处理程序的返回值或捕获的异常。
         """
+        # Set the default sender if not provided
+        # 如果未提供，则设置默认发送者
         kwargs.setdefault('sender', self.sender)
+        # Send the signal using the utility function from aioscrapy.utils.signal
+        # 使用aioscrapy.utils.signal中的实用函数发送信号
         return await _signal.send_catch_log(signal, **kwargs)
 
     async def send_catch_log_deferred(self, signal, **kwargs):
         """
-        Like :meth:`send_catch_log` but supports returning
-        :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
+        Send a signal and gather results from all handlers concurrently.
+        发送信号并同时收集所有处理程序的结果。
 
-        Returns a Deferred that gets fired once all signal handlers
-        deferreds were fired. Send a signal, catch exceptions and log them.
+        This method is similar to send_catch_log but runs all signal handlers
+        concurrently using asyncio tasks. It waits for all handlers to complete
+        before returning the results. This is useful when signal handlers are
+        independent of each other and can run in parallel.
+        此方法类似于send_catch_log，但使用asyncio任务同时运行所有信号处理程序。
+        它在返回结果之前等待所有处理程序完成。当信号处理程序彼此独立并且
+        可以并行运行时，这很有用。
 
-        The keyword arguments are passed to the signal handlers (connected
-        through the :meth:`connect` method).
+        Args:
+            signal: The signal to send.
+                   要发送的信号。
+            **kwargs: Keyword arguments to pass to the signal handlers.
+                     传递给信号处理程序的关键字参数。
+
+        Returns:
+            list: A list of results from all signal handlers, in the order they were
+                 registered. Each result is either the return value of the handler
+                 or the exception that was caught.
+                 所有信号处理程序的结果列表，按它们注册的顺序排列。
+                 每个结果要么是处理程序的返回值，要么是捕获的异常。
         """
+        # Set the default sender if not provided
+        # 如果未提供，则设置默认发送者
         kwargs.setdefault('sender', self.sender)
+        # Send the signal using the utility function from aioscrapy.utils.signal
+        # 使用aioscrapy.utils.signal中的实用函数发送信号
         return await _signal.send_catch_log_deferred(signal, **kwargs)
 
     def disconnect_all(self, signal, **kwargs):
         """
         Disconnect all receivers from the given signal.
+        断开给定信号的所有接收器。
+
+        This method disconnects all receivers that are connected to the specified
+        signal. It's useful for cleaning up signal connections, especially during
+        testing or when shutting down a component.
+        此方法断开连接到指定信号的所有接收器。
+        它对于清理信号连接很有用，特别是在测试期间或关闭组件时。
+
+        Args:
+            signal: The signal to disconnect all receivers from.
+                   要断开所有接收器的信号。
+            **kwargs: Additional keyword arguments to pass to the disconnect_all function.
+                     传递给disconnect_all函数的其他关键字参数。
 
-        :param signal: the signal to disconnect from
-        :type signal: object
+        Returns:
+            None
         """
+        # Set the default sender if not provided
+        # 如果未提供，则设置默认发送者
         kwargs.setdefault('sender', self.sender)
+        # Disconnect all receivers using the utility function from aioscrapy.utils.signal
+        # 使用aioscrapy.utils.signal中的实用函数断开所有接收器
         return _signal.disconnect_all(signal, **kwargs)

aioscrapy/signals.py

@@ -1,24 +1,206 @@
 """
-Scrapy signals
+AioScrapy Signals
+AioScrapy信号
+
+This module defines all signals that the AioScrapy framework emits during the
+execution of a crawl. These signals allow developers to hook into various points
+of the crawling process to add custom functionality.
+此模块定义了AioScrapy框架在爬取执行过程中发出的所有信号。
+这些信号允许开发人员挂钩到爬取过程的各个点，以添加自定义功能。
+
+Signals are implemented using the PyDispatcher library and are represented as
+unique objects. To connect to a signal, use the crawler.signals.connect method.
+信号使用PyDispatcher库实现，并表示为唯一对象。
+要连接到信号，请使用crawler.signals.connect方法。
+
+Example:
+    def handle_spider_opened(spider):
+        print(f"Spider {spider.name} opened")
+
+    crawler.signals.connect(handle_spider_opened, signal=signals.spider_opened)
 
 These signals are documented in docs/topics/signals.rst. Please don't add new
 signals here without documenting them there.
+这些信号在docs/topics/signals.rst中有文档说明。
+请不要在此处添加新信号，除非在那里记录它们。
 """
 
+# Engine signals
+# 引擎信号
+
+#: Signal sent when the aioscrapy engine has started.
+#: 当aioscrapy引擎启动时发送的信号。
+#: Args: None
 engine_started = object()
+
+#: Signal sent when the aioscrapy engine has stopped.
+#: 当aioscrapy引擎停止时发送的信号。
+#: Args: None
 engine_stopped = object()
+
+
+# Spider signals
+# 爬虫信号
+
+#: Signal sent when a spider has been opened for crawling.
+#: 当爬虫被打开进行爬取时发送的信号。
+#: Args:
+#:     spider (Spider): The spider that has been opened.
+#:                      已被打开的爬虫。
 spider_opened = object()
+
+#: Signal sent when a spider has no more requests to process.
+#: 当爬虫没有更多请求要处理时发送的信号。
+#: Args:
+#:     spider (Spider): The spider that has become idle.
+#:                      变为空闲的爬虫。
 spider_idle = object()
+
+#: Signal sent when a spider has been closed.
+#: 当爬虫被关闭时发送的信号。
+#: Args:
+#:     spider (Spider): The spider that has been closed.
+#:                      已被关闭的爬虫。
+#:     reason (str): A string describing the reason why the spider was closed.
+#:                   描述爬虫被关闭原因的字符串。
 spider_closed = object()
+
+#: Signal sent when a spider callback generates an error.
+#: 当爬虫回调生成错误时发送的信号。
+#: Args:
+#:     failure (Failure): The exception information.
+#:                        异常信息。
+#:     response (Response): The response that caused the error.
+#:                          导致错误的响应。
+#:     spider (Spider): The spider that raised the exception.
+#:                      引发异常的爬虫。
 spider_error = object()
+
+
+# Request signals
+# 请求信号
+
+#: Signal sent when a new Request is scheduled to be downloaded.
+#: 当新的Request被安排下载时发送的信号。
+#: Args:
+#:     request (Request): The request that reached the scheduler.
+#:                        到达调度器的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 request_scheduled = object()
+
+#: Signal sent when a Request is dropped by the scheduler.
+#: 当请求被调度器丢弃时发送的信号。
+#: Args:
+#:     request (Request): The request that was dropped.
+#:                        被丢弃的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 request_dropped = object()
+
+#: Signal sent when a Request reaches the downloader.
+#: 当请求到达下载器时发送的信号。
+#: Args:
+#:     request (Request): The request that reached the downloader.
+#:                        到达下载器的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 request_reached_downloader = object()
+
+#: Signal sent when a Request leaves the downloader.
+#: 当请求离开下载器时发送的信号。
+#: Args:
+#:     request (Request): The request that left the downloader.
+#:                        离开下载器的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 request_left_downloader = object()
+
+
+# Response signals
+# 响应信号
+
+#: Signal sent when the downloader receives a response from the web server.
+#: 当下载器从Web服务器接收到响应时发送的信号。
+#: Args:
+#:     response (Response): The response received.
+#:                          接收到的响应。
+#:     request (Request): The request that generated the response.
+#:                        生成响应的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 response_received = object()
+
+#: Signal sent when a Response has been downloaded.
+#: 当响应已被下载时发送的信号。
+#: Args:
+#:     response (Response): The response downloaded.
+#:                          下载的响应。
+#:     request (Request): The request that generated the response.
+#:                        生成响应的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 response_downloaded = object()
+
+#: Signal sent when the HTTP headers are received for a request.
+#: 当接收到请求的HTTP头时发送的信号。
+#: Args:
+#:     headers (dict): The HTTP headers received.
+#:                     接收到的HTTP头。
+#:     body_length (int): Expected size of the response body.
+#:                        预期的响应正文大小。
+#:     request (Request): The request that generated the response.
+#:                        生成响应的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 headers_received = object()
+
+#: Signal sent when a chunk of response data is received.
+#: 当接收到响应数据块时发送的信号。
+#: Args:
+#:     data (bytes): The chunk of data received.
+#:                   接收到的数据块。
+#:     request (Request): The request that generated the response.
+#:                        生成响应的请求。
+#:     spider (Spider): The spider that generated the request.
+#:                      生成请求的爬虫。
 bytes_received = object()
+
+
+# Item signals
+# 项目信号
+
+#: Signal sent when an item has been scraped by a spider.
+#: 当项目被爬虫抓取时发送的信号。
+#: Args:
+#:     item (Item or dict): The item scraped.
+#:                          抓取的项目。
+#:     response (Response): The response from which the item was scraped.
+#:                          项目被抓取的响应。
+#:     spider (Spider): The spider which scraped the item.
+#:                      抓取项目的爬虫。
 item_scraped = object()
+
+#: Signal sent when an item is dropped by an item pipeline.
+#: 当项目被项目管道丢弃时发送的信号。
+#: Args:
+#:     item (Item or dict): The item dropped from the pipeline.
+#:                          从管道丢弃的项目。
+#:     exception (Exception): The exception that caused the item to be dropped.
+#:                            导致项目被丢弃的异常。
+#:     spider (Spider): The spider which scraped the item.
+#:                      抓取项目的爬虫。
 item_dropped = object()
+
+#: Signal sent when an item causes an error in an item pipeline.
+#: 当项目在项目管道中导致错误时发送的信号。
+#: Args:
+#:     item (Item or dict): The item that caused the error.
+#:                          导致错误的项目。
+#:     exception (Exception): The exception raised.
+#:                            引发的异常。
+#:     spider (Spider): The spider which scraped the item.
+#:                      抓取项目的爬虫。
+#:     response (Response): The response from which the item was scraped.
+#:                          项目被抓取的响应。
 item_error = object()

aioscrapy/spiderloader.py

@@ -9,37 +9,113 @@ from aioscrapy.utils.spider import iter_spider_classes
 
 
 class ISpiderLoader(Interface):
+    """
+    Interface for spider loader implementations.
+    爬虫加载器实现的接口。
+
+    This interface defines the methods that spider loader implementations
+    must provide.
+    此接口定义了爬虫加载器实现必须提供的方法。
+    """
 
     def from_settings(settings):
-        """Return an instance of the class for the given settings"""
+        """
+        Return an instance of the class for the given settings.
+        返回给定设置的类实例。
+
+        Args:
+            settings: The settings to use for the spider loader.
+                     用于爬虫加载器的设置。
+
+        Returns:
+            An instance of the spider loader.
+            爬虫加载器的实例。
+        """
 
     def load(spider_name):
-        """Return the Spider class for the given spider name. If the spider
-        name is not found, it must raise a KeyError."""
+        """
+        Return the Spider class for the given spider name.
+        返回给定爬虫名称的Spider类。
+
+        Args:
+            spider_name: The name of the spider to load.
+                        要加载的爬虫的名称。
+
+        Returns:
+            The Spider class for the given spider name.
+            给定爬虫名称的Spider类。
+
+        Raises:
+            KeyError: If the spider name is not found.
+                     如果找不到爬虫名称。
+        """
 
     def list():
-        """Return a list with the names of all spiders available in the
-        project"""
+        """
+        Return a list with the names of all spiders available in the project.
+        返回项目中所有可用爬虫的名称列表。
+
+        Returns:
+            A list of spider names.
+            爬虫名称列表。
+        """
 
     def find_by_request(request):
-        """Return the list of spiders names that can handle the given request"""
+        """
+        Return the list of spider names that can handle the given request.
+        返回可以处理给定请求的爬虫名称列表。
+
+        Args:
+            request: The request to check.
+                    要检查的请求。
+
+        Returns:
+            A list of spider names that can handle the request.
+            可以处理请求的爬虫名称列表。
+        """
 
 
 @implementer(ISpiderLoader)
 class SpiderLoader:
     """
-    SpiderLoader is a class which locates and loads spiders
-    in a aioscrapy project.
+    SpiderLoader is a class which locates and loads spiders in a aioscrapy project.
+    SpiderLoader是一个定位和加载aioscrapy项目中爬虫的类。
+
+    This class implements the ISpiderLoader interface and provides methods to
+    find, load, and list spiders in a project.
+    此类实现了ISpiderLoader接口，并提供了在项目中查找、加载和列出爬虫的方法。
     """
 
     def __init__(self, settings):
+        """
+        Initialize the SpiderLoader.
+        初始化SpiderLoader。
+
+        This method initializes the SpiderLoader with the given settings and
+        loads all spiders from the specified modules.
+        此方法使用给定的设置初始化SpiderLoader，并从指定的模块加载所有爬虫。
+
+        Args:
+            settings: The settings object containing spider loader configuration.
+                     包含爬虫加载器配置的设置对象。
+        """
         self.spider_modules = settings.getlist('SPIDER_MODULES')
         self.warn_only = settings.getbool('SPIDER_LOADER_WARN_ONLY')
-        self._spiders = {}
-        self._found = defaultdict(list)
+        self._spiders = {}  # Dict of spider name -> spider class
+                           # 爬虫名称 -> 爬虫类的字典
+        self._found = defaultdict(list)  # Dict of spider name -> list of (module, class) locations
+                                        # 爬虫名称 -> (模块, 类)位置列表的字典
         self._load_all_spiders()
 
     def _check_name_duplicates(self):
+        """
+        Check for duplicate spider names and issue warnings if found.
+        检查重复的爬虫名称，如果发现则发出警告。
+
+        This method checks if there are multiple spider classes with the same name
+        and issues a warning if duplicates are found.
+        此方法检查是否有多个具有相同名称的爬虫类，如果发现重复则发出警告。
+        """
         dupes = []
         for name, locations in self._found.items():
             dupes.extend([
@@ -57,11 +133,35 @@ class SpiderLoader:
             )
 
     def _load_spiders(self, module):
+        """
+        Load spiders from a given module.
+        从给定模块加载爬虫。
+
+        This method finds all spider classes in the given module and adds them
+        to the internal dictionaries.
+        此方法查找给定模块中的所有爬虫类，并将它们添加到内部字典中。
+
+        Args:
+            module: The module to load spiders from.
+                   要从中加载爬虫的模块。
+        """
         for spcls in iter_spider_classes(module):
             self._found[spcls.name].append((module.__name__, spcls.__name__))
             self._spiders[spcls.name] = spcls
 
     def _load_all_spiders(self):
+        """
+        Load all spiders from all modules specified in SPIDER_MODULES setting.
+        从SPIDER_MODULES设置中指定的所有模块加载所有爬虫。
+
+        This method walks through all the modules specified in the SPIDER_MODULES
+        setting, loads all spiders from them, and checks for duplicate names.
+        此方法遍历SPIDER_MODULES设置中指定的所有模块，从中加载所有爬虫，并检查重复的名称。
+
+        If an import error occurs and SPIDER_LOADER_WARN_ONLY is True, a warning
+        is issued instead of raising the exception.
+        如果发生导入错误且SPIDER_LOADER_WARN_ONLY为True，则发出警告而不是引发异常。
+        """
         for name in self.spider_modules:
             try:
                 for module in walk_modules(name):
@@ -80,12 +180,43 @@ class SpiderLoader:
 
     @classmethod
     def from_settings(cls, settings):
+        """
+        Create a SpiderLoader instance from settings.
+        从设置创建SpiderLoader实例。
+
+        This is a factory method that creates a new SpiderLoader instance
+        with the given settings.
+        这是一个工厂方法，使用给定的设置创建一个新的SpiderLoader实例。
+
+        Args:
+            settings: The settings to use for the spider loader.
+                     用于爬虫加载器的设置。
+
+        Returns:
+            A new SpiderLoader instance.
+            一个新的SpiderLoader实例。
+        """
         return cls(settings)
 
     def load(self, spider_name):
         """
-        Return the Spider class for the given spider name. If the spider
-        name is not found, raise a KeyError.
+        Return the Spider class for the given spider name.
+        返回给定爬虫名称的Spider类。
+
+        This method looks up the spider class by name in the internal dictionary.
+        此方法在内部字典中按名称查找爬虫类。
+
+        Args:
+            spider_name: The name of the spider to load.
+                        要加载的爬虫的名称。
+
+        Returns:
+            The Spider class for the given spider name.
+            给定爬虫名称的Spider类。
+
+        Raises:
+            KeyError: If the spider name is not found.
+                     如果找不到爬虫名称。
         """
         try:
             return self._spiders[spider_name]
@@ -95,6 +226,19 @@ class SpiderLoader:
     def find_by_request(self, request):
         """
         Return the list of spider names that can handle the given request.
+        返回可以处理给定请求的爬虫名称列表。
+
+        This method checks each spider's handles_request method to determine
+        if it can handle the given request.
+        此方法检查每个爬虫的handles_request方法，以确定它是否可以处理给定的请求。
+
+        Args:
+            request: The request to check.
+                    要检查的请求。
+
+        Returns:
+            A list of spider names that can handle the request.
+            可以处理请求的爬虫名称列表。
         """
         return [
             name for name, cls in self._spiders.items()
@@ -104,5 +248,14 @@ class SpiderLoader:
     def list(self):
         """
         Return a list with the names of all spiders available in the project.
+        返回项目中所有可用爬虫的名称列表。
+
+        This method returns a list of all spider names that have been loaded
+        by the spider loader.
+        此方法返回已由爬虫加载器加载的所有爬虫名称的列表。
+
+        Returns:
+            A list of spider names.
+            爬虫名称列表。
         """
         return list(self._spiders.keys())