aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/libs/downloader/retry.py

@@ -1,13 +1,30 @@
 """
+Retry Middleware for AioScrapy
+AioScrapy的重试中间件
+
 An extension to retry failed requests that are potentially caused by temporary
 problems such as a connection timeout or HTTP 500 error.
+一个扩展，用于重试可能由临时问题（如连接超时或HTTP 500错误）导致的失败请求。
+
+You can change the behavior of this middleware by modifying the scraping settings:
+您可以通过修改抓取设置来更改此中间件的行为：
+
+RETRY_ENABLED - whether to enable the retry middleware (default: True)
+RETRY_ENABLED - 是否启用重试中间件（默认：True）
+
+RETRY_TIMES - how many times to retry a failed page (default: 2)
+RETRY_TIMES - 重试失败页面的次数（默认：2）
 
-You can change the behaviour of this middleware by modifing the scraping settings:
-RETRY_TIMES - how many times to retry a failed page
-RETRY_HTTP_CODES - which HTTP response codes to retry
+RETRY_HTTP_CODES - which HTTP response codes to retry (default: [500, 502, 503, 504, 522, 524, 408, 429])
+RETRY_HTTP_CODES - 要重试的HTTP响应代码（默认：[500, 502, 503, 504, 522, 524, 408, 429]）
 
-Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non failed) pages.
+RETRY_PRIORITY_ADJUST - adjust retry request priority (default: -1)
+RETRY_PRIORITY_ADJUST - 调整重试请求优先级（默认：-1）
+
+Failed pages are collected during the scraping process and rescheduled,
+allowing the spider to continue crawling other pages while retrying failed ones.
+失败的页面在抓取过程中被收集并重新安排，
+允许爬虫在重试失败页面的同时继续抓取其他页面。
 """
 from typing import Optional, Union
 
@@ -24,6 +41,8 @@ from aioscrapy.spiders import Spider
 from aioscrapy.utils.log import logger as retry_logger
 from aioscrapy.utils.python import global_object_name
 
+# Tuple of exception types that should trigger a retry
+# 应触发重试的异常类型元组
 NEED_RETRY_ERROR = (TimeoutError, ConnectionRefusedError, IOError, ProxyException, DownloadError, EndOfStream)
 
 
@@ -38,7 +57,50 @@ def get_retry_request(
         stats_base_key: str = 'retry',
 ):
     """
-    使用了scrapy的retry，将日志等级改为info
+    Create a new request object to retry the specified failed request.
+    创建一个新的请求对象来重试指定的失败请求。
+
+    This function is based on Scrapy's retry functionality but uses INFO level
+    logging instead of DEBUG level for retry attempts. It creates a copy of the
+    original request with updated retry count and adjusted priority.
+    此函数基于Scrapy的重试功能，但对重试尝试使用INFO级别的日志记录而不是DEBUG级别。
+    它创建原始请求的副本，更新重试计数并调整优先级。
+
+    Args:
+        request: The original Request object that failed.
+                失败的原始Request对象。
+        spider: The spider instance that generated the request.
+               生成请求的爬虫实例。
+        reason: The reason for the retry, either a string or an exception.
+               重试的原因，可以是字符串或异常。
+               Defaults to 'unspecified'.
+               默认为'unspecified'。
+        max_retry_times: Maximum number of times to retry this request.
+                        重试此请求的最大次数。
+                        If None, uses the value from request.meta['max_retry_times']
+                        or the RETRY_TIMES setting.
+                        如果为None，则使用request.meta['max_retry_times']
+                        或RETRY_TIMES设置中的值。
+        priority_adjust: Amount to adjust the request priority.
+                        调整请求优先级的数量。
+                        If None, uses the value from request.meta['priority_adjust']
+                        or the RETRY_PRIORITY_ADJUST setting.
+                        如果为None，则使用request.meta['priority_adjust']
+                        或RETRY_PRIORITY_ADJUST设置中的值。
+        logger: The logger to use for logging retry attempts.
+               用于记录重试尝试的日志记录器。
+               Defaults to the retry_logger.
+               默认为retry_logger。
+        stats_base_key: The base key to use for recording retry statistics.
+                       用于记录重试统计信息的基本键。
+                       Defaults to 'retry'.
+                       默认为'retry'。
+
+    Returns:
+        Request: A new Request object with updated retry count and priority,
+                or None if max_retry_times has been reached.
+                具有更新的重试计数和优先级的新Request对象，
+                如果已达到max_retry_times，则为None。
     """
     settings = spider.crawler.settings
     stats = spider.crawler.stats
@@ -78,9 +140,35 @@ def get_retry_request(
 
 
 class RetryMiddleware:
+    """
+    Middleware to retry failed requests.
+    重试失败请求的中间件。
+
+    This middleware retries requests that have failed due to temporary issues
+    such as connection problems or certain HTTP error codes. It works by
+    intercepting responses with error status codes and exceptions, then
+    creating new retry requests with updated retry counts.
+    此中间件重试由于临时问题（如连接问题或某些HTTP错误代码）而失败的请求。
+    它通过拦截具有错误状态代码和异常的响应，然后创建具有更新的重试计数的新重试请求来工作。
+    """
+
+    # List of exceptions that should trigger a retry
+    # 应触发重试的异常列表
     EXCEPTIONS_TO_RETRY = NEED_RETRY_ERROR
 
     def __init__(self, settings):
+        """
+        Initialize the RetryMiddleware.
+        初始化RetryMiddleware。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+
+        Raises:
+            NotConfigured: If RETRY_ENABLED is False.
+                          如果RETRY_ENABLED为False。
+        """
         if not settings.getbool('RETRY_ENABLED'):
             raise NotConfigured
         self.max_retry_times = settings.getint('RETRY_TIMES')
@@ -89,26 +177,124 @@ class RetryMiddleware:
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a RetryMiddleware instance from a crawler.
+        从爬虫创建RetryMiddleware实例。
+
+        This is the factory method used by AioScrapy to create middleware instances.
+        这是AioScrapy用于创建中间件实例的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            RetryMiddleware: A new RetryMiddleware instance.
+                            一个新的RetryMiddleware实例。
+        """
         return cls(crawler.settings)
 
     def process_response(self, request, response, spider):
+        """
+        Process a response to check if it needs to be retried.
+        处理响应以检查是否需要重试。
+
+        This method checks if the response status code is in the list of
+        status codes that should be retried. If so, it creates a retry request.
+        此方法检查响应状态代码是否在应重试的状态代码列表中。
+        如果是，则创建重试请求。
+
+        Args:
+            request: The original request that generated the response.
+                    生成响应的原始请求。
+            response: The response to process.
+                     要处理的响应。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            Response or Request: The original response or a new retry request.
+                                原始响应或新的重试请求。
+        """
+        # Don't retry if the request has dont_retry set to True
+        # 如果请求的dont_retry设置为True，则不重试
         if request.meta.get('dont_retry', False):
             return response
+
+        # Retry if the status code is in the list of codes to retry
+        # 如果状态代码在要重试的代码列表中，则重试
         if response.status in self.retry_http_codes:
             reason = f"Retry response status code： {response.status}"
             return self._retry(request, reason, spider) or response
+
+        # Otherwise, return the response as is
+        # 否则，按原样返回响应
         return response
 
     def process_exception(self, request, exception, spider):
+        """
+        Process an exception to check if the request should be retried.
+        处理异常以检查是否应重试请求。
+
+        This method checks if the exception is in the list of exceptions
+        that should trigger a retry. If so, it creates a retry request.
+        此方法检查异常是否在应触发重试的异常列表中。
+        如果是，则创建重试请求。
+
+        Args:
+            request: The request that caused the exception.
+                    导致异常的请求。
+            exception: The exception that was raised.
+                      引发的异常。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            Request or None: A new retry request or None if the request should not be retried.
+                            新的重试请求，如果不应重试请求，则为None。
+        """
+        # Retry if the exception is in the list of exceptions to retry
+        # and the request doesn't have dont_retry set to True
+        # 如果异常在要重试的异常列表中，并且请求的dont_retry未设置为True，则重试
         if (
                 isinstance(exception, self.EXCEPTIONS_TO_RETRY)
                 and not request.meta.get('dont_retry', False)
         ):
             return self._retry(request, exception, spider)
 
+        # Otherwise, return None to let the exception be processed by other middleware
+        # 否则，返回None以让异常被其他中间件处理
+        return None
+
     def _retry(self, request, reason, spider):
+        """
+        Create a retry request for the given request.
+        为给定请求创建重试请求。
+
+        This internal method gets the retry parameters from the request metadata
+        or middleware settings, then calls get_retry_request to create a new request.
+        此内部方法从请求元数据或中间件设置获取重试参数，
+        然后调用get_retry_request创建新请求。
+
+        Args:
+            request: The original request to retry.
+                    要重试的原始请求。
+            reason: The reason for the retry (string or exception).
+                   重试的原因（字符串或异常）。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            Request or None: A new retry request or None if max retries has been reached.
+                            新的重试请求，如果已达到最大重试次数，则为None。
+        """
+        # Get retry parameters from request metadata or middleware settings
+        # 从请求元数据或中间件设置获取重试参数
         max_retry_times = request.meta.get('max_retry_times', self.max_retry_times)
         priority_adjust = request.meta.get('priority_adjust', self.priority_adjust)
+
+        # Create and return a retry request
+        # 创建并返回重试请求
         return get_retry_request(
             request,
             reason=reason,

aioscrapy/libs/downloader/stats.py

@@ -1,31 +1,173 @@
+"""
+Downloader Statistics Middleware for AioScrapy
+AioScrapy的下载器统计中间件
+
+This module provides a middleware that collects statistics about the downloader
+component, including request and response counts, bytes transferred, HTTP methods,
+response status codes, and exceptions.
+此模块提供了一个中间件，用于收集有关下载器组件的统计信息，
+包括请求和响应计数、传输的字节数、HTTP方法、响应状态码和异常。
+"""
+
 from aioscrapy.exceptions import NotConfigured
 from aioscrapy.utils.request import request_httprepr
 from aioscrapy.utils.python import global_object_name
 
 
 class DownloaderStats:
+    """
+    Middleware to collect statistics about the downloader component.
+    用于收集下载器组件统计信息的中间件。
+
+    This middleware collects various statistics about the downloader component,
+    such as the number of requests and responses, bytes transferred, HTTP methods used,
+    response status codes, and exceptions encountered. These statistics are stored
+    in the crawler's stats collector and can be used for monitoring and debugging.
+    此中间件收集有关下载器组件的各种统计信息，例如请求和响应的数量、
+    传输的字节数、使用的HTTP方法、响应状态码和遇到的异常。
+    这些统计信息存储在爬虫的统计收集器中，可用于监控和调试。
+    """
 
     def __init__(self, stats):
+        """
+        Initialize the DownloaderStats middleware.
+        初始化DownloaderStats中间件。
+
+        Args:
+            stats: The stats collector instance to use for storing statistics.
+                  用于存储统计信息的统计收集器实例。
+        """
         self.stats = stats
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a DownloaderStats instance from a crawler.
+        从爬虫创建DownloaderStats实例。
+
+        This is the factory method used by AioScrapy to create middleware instances.
+        It checks if the DOWNLOADER_STATS setting is enabled before creating the middleware.
+        这是AioScrapy用于创建中间件实例的工厂方法。
+        它在创建中间件之前检查DOWNLOADER_STATS设置是否启用。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            DownloaderStats: A new DownloaderStats instance.
+                            一个新的DownloaderStats实例。
+
+        Raises:
+            NotConfigured: If DOWNLOADER_STATS setting is disabled.
+                          如果DOWNLOADER_STATS设置被禁用。
+        """
         if not crawler.settings.getbool('DOWNLOADER_STATS'):
             raise NotConfigured
         return cls(crawler.stats)
 
     def process_request(self, request, spider):
+        """
+        Process a request to collect request statistics.
+        处理请求以收集请求统计信息。
+
+        This method is called for every request that passes through the middleware.
+        It increments counters for the total number of requests, requests by HTTP method,
+        and the total number of bytes in requests.
+        此方法在每个通过中间件的请求上调用。
+        它增加总请求数、按HTTP方法的请求数和请求中的总字节数的计数器。
+
+        Args:
+            request: The request being processed.
+                    正在处理的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method returns None to continue processing the request.
+                 此方法返回None以继续处理请求。
+        """
+        # Increment the total request count
+        # 增加总请求计数
         self.stats.inc_value('downloader/request_count', spider=spider)
+
+        # Increment the count for this specific HTTP method
+        # 增加此特定HTTP方法的计数
         self.stats.inc_value(f'downloader/request_method_count/{request.method}', spider=spider)
+
+        # Add the request size to the total bytes counter
+        # 将请求大小添加到总字节计数器
         self.stats.inc_value('downloader/request_bytes', len(request_httprepr(request)), spider=spider)
 
     def process_response(self, request, response, spider):
+        """
+        Process a response to collect response statistics.
+        处理响应以收集响应统计信息。
+
+        This method is called for every response that passes through the middleware.
+        It increments counters for the total number of responses, responses by status code,
+        and the total number of bytes in responses.
+        此方法在每个通过中间件的响应上调用。
+        它增加总响应数、按状态码的响应数和响应中的总字节数的计数器。
+
+        Args:
+            request: The request that generated this response.
+                    生成此响应的请求。
+            response: The response being processed.
+                     正在处理的响应。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            Response: The response object, unchanged.
+                     响应对象，未更改。
+        """
+        # Increment the total response count
+        # 增加总响应计数
         self.stats.inc_value('downloader/response_count', spider=spider)
+
+        # Increment the count for this specific status code
+        # 增加此特定状态码的计数
         self.stats.inc_value(f'downloader/response_status_count/{response.status}', spider=spider)
+
+        # Add the response size to the total bytes counter
+        # 将响应大小添加到总字节计数器
         self.stats.inc_value('downloader/response_bytes', len(response.body), spider=spider)
+
+        # Return the response unchanged
+        # 返回未更改的响应
         return response
 
     def process_exception(self, request, exception, spider):
+        """
+        Process an exception to collect exception statistics.
+        处理异常以收集异常统计信息。
+
+        This method is called when an exception occurs during request processing.
+        It increments counters for the total number of exceptions and exceptions by type.
+        当请求处理期间发生异常时调用此方法。
+        它增加总异常数和按类型的异常数的计数器。
+
+        Args:
+            request: The request that caused the exception.
+                    导致异常的请求。
+            exception: The exception that was raised.
+                      引发的异常。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method returns None to continue processing the exception.
+                 此方法返回None以继续处理异常。
+        """
+        # Get the full class name of the exception
+        # 获取异常的完整类名
         ex_class = global_object_name(exception.__class__)
+
+        # Increment the total exception count
+        # 增加总异常计数
         self.stats.inc_value('downloader/exception_count', spider=spider)
+
+        # Increment the count for this specific exception type
+        # 增加此特定异常类型的计数
         self.stats.inc_value(f'downloader/exception_type_count/{ex_class}', spider=spider)

aioscrapy/libs/downloader/useragent.py

@@ -1,23 +1,114 @@
-"""Set User-Agent header per spider or use a default value from settings"""
+"""
+User-Agent Middleware
+用户代理中间件
+
+This middleware sets the User-Agent header for all requests, using either a
+spider-specific user_agent attribute, or a default value from the USER_AGENT setting.
+此中间件为所有请求设置User-Agent头，使用爬虫特定的user_agent属性，
+或来自USER_AGENT设置的默认值。
+
+The User-Agent header is important for identifying your crawler to websites and
+can affect how websites respond to your requests.
+User-Agent头对于向网站标识您的爬虫很重要，可能会影响网站对您的请求的响应方式。
+"""
 
 from aioscrapy import signals
 
 
 class UserAgentMiddleware:
-    """This middleware allows spiders to override the user_agent"""
+    """
+    Middleware for setting the User-Agent header on requests.
+    用于在请求上设置User-Agent头的中间件。
+
+    This middleware allows spiders to override the default User-Agent by specifying
+    a user_agent attribute. If no spider-specific User-Agent is defined, it uses
+    the default value from the USER_AGENT setting.
+    此中间件允许爬虫通过指定user_agent属性来覆盖默认的User-Agent。
+    如果未定义爬虫特定的User-Agent，则使用USER_AGENT设置中的默认值。
+    """
 
     def __init__(self, user_agent='Scrapy'):
+        """
+        Initialize the UserAgentMiddleware.
+        初始化UserAgentMiddleware。
+
+        Args:
+            user_agent: The default User-Agent string to use.
+                       要使用的默认User-Agent字符串。
+                       Defaults to 'Scrapy'.
+                       默认为'Scrapy'。
+        """
+        # Store the default user agent
+        # 存储默认用户代理
         self.user_agent = user_agent
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a UserAgentMiddleware instance from a crawler.
+        从爬虫创建UserAgentMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            UserAgentMiddleware: A new UserAgentMiddleware instance.
+                               一个新的UserAgentMiddleware实例。
+        """
+        # Create a new instance with the user agent from settings
+        # 使用来自设置的用户代理创建一个新实例
         o = cls(crawler.settings['USER_AGENT'])
+
+        # Connect to the spider_opened signal
+        # 连接到spider_opened信号
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+
+        # Return the new instance
+        # 返回新实例
         return o
 
     def spider_opened(self, spider):
+        """
+        Handle the spider_opened signal.
+        处理spider_opened信号。
+
+        This method is called when a spider is opened. It updates the user agent
+        with the spider's user_agent attribute if it exists.
+        当爬虫打开时调用此方法。如果存在，它会使用爬虫的user_agent属性更新用户代理。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
+        # Update the user agent with the spider's user_agent attribute if it exists
+        # 如果存在，则使用爬虫的user_agent属性更新用户代理
         self.user_agent = getattr(spider, 'user_agent', self.user_agent)
 
     def process_request(self, request, spider):
+        """
+        Process a request before it is sent to the downloader.
+        在请求发送到下载器之前处理它。
+
+        This method sets the User-Agent header in the request if it's not already set
+        and if a user agent is configured.
+        如果尚未设置User-Agent头且已配置用户代理，此方法会在请求中设置它。
+
+        Args:
+            request: The request being processed.
+                    正在处理的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            None: This method does not return a response or a deferred.
+                 此方法不返回响应或延迟对象。
+        """
+        # Set the User-Agent header in the request if it's not already set
+        # and if a user agent is configured
+        # 如果尚未设置User-Agent头且已配置用户代理，则在请求中设置它
         if self.user_agent:
             request.headers.setdefault('User-Agent', self.user_agent)