aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/libs/spider/httperror.py

@@ -1,54 +1,194 @@
 """
 HttpError Spider Middleware
+HTTP错误爬虫中间件
 
-See documentation in docs/topics/spider-middleware.rst
+This middleware filters out responses with non-200 status codes and generates
+appropriate exceptions. It allows you to specify which status codes should be
+allowed through to the spider via settings or meta attributes.
+此中间件过滤掉具有非200状态码的响应并生成适当的异常。它允许您通过设置或
+元属性指定哪些状态码应该被允许传递给爬虫。
 """
 
 from aioscrapy.exceptions import IgnoreRequest
-
 from aioscrapy.utils.log import logger
 
 
 class HttpError(IgnoreRequest):
-    """A non-200 response was filtered"""
+    """
+    Exception raised when a non-200 response is filtered.
+    当过滤非200响应时引发的异常。
+
+    This exception is raised by the HttpErrorMiddleware when it encounters a
+    response with a status code that is not in the allowed list. It is a subclass
+    of IgnoreRequest, which means the response will be ignored by the spider.
+    当HttpErrorMiddleware遇到状态码不在允许列表中的响应时，会引发此异常。
+    它是IgnoreRequest的子类，这意味着该响应将被爬虫忽略。
+    """
 
     def __init__(self, response, *args, **kwargs):
+        """
+        Initialize the HttpError exception.
+        初始化HttpError异常。
+
+        Args:
+            response: The response that triggered the exception.
+                     触发异常的响应。
+            *args: Variable length argument list passed to the parent class.
+                  传递给父类的可变长度参数列表。
+            **kwargs: Arbitrary keyword arguments passed to the parent class.
+                     传递给父类的任意关键字参数。
+        """
+        # Store the response that triggered the exception
+        # 存储触发异常的响应
         self.response = response
+
+        # Initialize the parent IgnoreRequest class
+        # 初始化父类IgnoreRequest
         super().__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware:
+    """
+    Spider middleware to filter out responses with non-200 status codes.
+    用于过滤掉具有非200状态码的响应的爬虫中间件。
+
+    This middleware checks the status code of each response and raises an HttpError
+    exception for responses with status codes that are not in the allowed list.
+    The allowed list can be specified via settings, spider attributes, or response
+    meta attributes.
+    此中间件检查每个响应的状态码，并为状态码不在允许列表中的响应引发HttpError异常。
+    允许列表可以通过设置、爬虫属性或响应元属性指定。
+    """
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a HttpErrorMiddleware instance from a crawler.
+        从爬虫创建HttpErrorMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            HttpErrorMiddleware: A new HttpErrorMiddleware instance.
+                                一个新的HttpErrorMiddleware实例。
+        """
+        # Create and return a new instance with the crawler's settings
+        # 使用爬虫的设置创建并返回一个新实例
         return cls(crawler.settings)
 
     def __init__(self, settings):
+        """
+        Initialize the HttpErrorMiddleware.
+        初始化HttpErrorMiddleware。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+        """
+        # Whether to allow all HTTP status codes
+        # 是否允许所有HTTP状态码
         self.handle_httpstatus_all = settings.getbool('HTTPERROR_ALLOW_ALL')
+
+        # List of allowed HTTP status codes
+        # 允许的HTTP状态码列表
         self.handle_httpstatus_list = settings.getlist('HTTPERROR_ALLOWED_CODES')
 
     def process_spider_input(self, response, spider):
-        if 200 <= response.status < 300:  # common case
+        """
+        Process a response before it is sent to the spider.
+        在响应发送到爬虫之前处理它。
+
+        This method checks if the response's status code is allowed. If not, it
+        raises an HttpError exception, which will be caught by process_spider_exception.
+        此方法检查响应的状态码是否被允许。如果不允许，它会引发HttpError异常，
+        该异常将被process_spider_exception捕获。
+
+        Args:
+            response: The response being processed.
+                     正在处理的响应。
+            spider: The spider that will receive the response.
+                   将接收响应的爬虫。
+
+        Raises:
+            HttpError: If the response's status code is not allowed.
+                      如果响应的状态码不被允许。
+        """
+        # Allow responses with status codes in the 200-299 range (common case)
+        # 允许状态码在200-299范围内的响应（常见情况）
+        if 200 <= response.status < 300:
             return
+
+        # Allow all status codes if specified in the response meta
+        # 如果在响应元数据中指定，则允许所有状态码
         if response.meta.get('handle_httpstatus_all', False):
             return
+
+        # Get the list of allowed status codes
+        # 获取允许的状态码列表
         if 'handle_httpstatus_list' in response.meta:
+            # Use the list from response meta if available
+            # 如果可用，使用来自响应元数据的列表
             allowed_statuses = response.meta['handle_httpstatus_list']
         elif self.handle_httpstatus_all:
+            # Allow all status codes if specified in settings
+            # 如果在设置中指定，则允许所有状态码
             return
         else:
+            # Use the list from spider attribute or middleware settings
+            # 使用来自爬虫属性或中间件设置的列表
             allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list)
+
+        # Allow the response if its status code is in the allowed list
+        # 如果响应的状态码在允许列表中，则允许该响应
         if response.status in allowed_statuses:
             return
+
+        # Raise an HttpError for responses with disallowed status codes
+        # 为具有不允许状态码的响应引发HttpError
         raise HttpError(response, 'Ignoring non-200 response')
 
     async def process_spider_exception(self, response, exception, spider):
+        """
+        Handle exceptions raised during spider processing.
+        处理爬虫处理期间引发的异常。
+
+        This method catches HttpError exceptions, logs them, updates statistics,
+        and returns an empty result list to suppress the exception.
+        此方法捕获HttpError异常，记录它们，更新统计信息，并返回一个空结果列表以抑制异常。
+
+        Args:
+            response: The response being processed when the exception was raised.
+                     引发异常时正在处理的响应。
+            exception: The exception raised.
+                      引发的异常。
+            spider: The spider that was processing the response.
+                   正在处理响应的爬虫。
+
+        Returns:
+            list: An empty list if the exception is an HttpError, None otherwise.
+                 如果异常是HttpError，则返回空列表；否则返回None。
+        """
+        # Only handle HttpError exceptions
+        # 只处理HttpError异常
         if isinstance(exception, HttpError):
+            # Update statistics
+            # 更新统计信息
             spider.crawler.stats.inc_value('httperror/response_ignored_count')
             spider.crawler.stats.inc_value(
                 f'httperror/response_ignored_status_count/{response.status}'
             )
+
+            # Log the ignored response
+            # 记录被忽略的响应
             logger.info("Ignoring response %(response)r: HTTP status code is not handled or not allowed" % {
                 'response': response
             })
+
+            # Return an empty list to suppress the exception
+            # 返回空列表以抑制异常
             return []

aioscrapy/libs/spider/offsite.py

@@ -1,7 +1,13 @@
 """
 Offsite Spider Middleware
+站外爬虫中间件
 
-See documentation in docs/topics/spider-middleware.rst
+This middleware filters out requests to URLs not belonging to the domains specified
+in the spider's allowed_domains attribute. It helps prevent the crawler from
+following links to external sites, which is useful for keeping crawls focused on
+specific domains.
+此中间件过滤掉对不属于爬虫的allowed_domains属性中指定的域的URL的请求。
+它有助于防止爬虫跟随指向外部站点的链接，这对于使爬取集中在特定域上很有用。
 """
 import re
 import warnings
@@ -13,71 +19,265 @@ from aioscrapy.utils.log import logger
 
 
 class OffsiteMiddleware:
+    """
+    Spider middleware to filter out requests to offsite domains.
+    用于过滤掉对站外域的请求的爬虫中间件。
+
+    This middleware filters out requests to URLs not belonging to the domains specified
+    in the spider's allowed_domains attribute. It helps prevent the crawler from
+    following links to external sites, which is useful for keeping crawls focused on
+    specific domains.
+    此中间件过滤掉对不属于爬虫的allowed_domains属性中指定的域的URL的请求。
+    它有助于防止爬虫跟随指向外部站点的链接，这对于使爬取集中在特定域上很有用。
+    """
 
     def __init__(self, stats):
+        """
+        Initialize the offsite middleware.
+        初始化站外中间件。
+
+        Args:
+            stats: Stats collector instance.
+                  统计收集器实例。
+        """
+        # Stats collector instance
+        # 统计收集器实例
         self.stats = stats
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create an OffsiteMiddleware instance from a crawler.
+        从爬虫创建OffsiteMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            OffsiteMiddleware: A new OffsiteMiddleware instance.
+                              一个新的OffsiteMiddleware实例。
+        """
+        # Create a new instance with the crawler's stats collector
+        # 使用爬虫的统计收集器创建一个新实例
         o = cls(crawler.stats)
+
+        # Connect the spider_opened method to the spider_opened signal
+        # 将spider_opened方法连接到spider_opened信号
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+
+        # Return the new instance
+        # 返回新实例
         return o
 
     async def process_spider_output(self, response, result, spider):
+        """
+        Process the spider output to filter out offsite requests.
+        处理爬虫输出以过滤掉站外请求。
+
+        This method processes each request yielded by the spider and filters out
+        requests to URLs not belonging to the domains specified in the spider's
+        allowed_domains attribute.
+        此方法处理爬虫产生的每个请求，并过滤掉对不属于爬虫的allowed_domains属性中
+        指定的域的URL的请求。
+
+        Args:
+            response: The response being processed.
+                     正在处理的响应。
+            result: The result returned by the spider.
+                   爬虫返回的结果。
+            spider: The spider that generated the result.
+                   生成结果的爬虫。
+
+        Returns:
+            An async generator yielding filtered requests and other items.
+            一个产生过滤后的请求和其他项目的异步生成器。
+        """
+        # Process each item in the result
+        # 处理结果中的每个项目
         async for x in result:
+            # If the item is a Request, check if it should be followed
+            # 如果项目是一个Request，检查是否应该跟随它
             if isinstance(x, Request):
+                # If the request has dont_filter set or it's for an allowed domain, yield it
+                # 如果请求设置了dont_filter或它是针对允许的域，则产生它
                 if x.dont_filter or self.should_follow(x, spider):
                     yield x
                 else:
+                    # Get the domain of the request
+                    # 获取请求的域
                     domain = urlparse_cached(x).hostname
+
+                    # If this is a new domain, log it and update stats
+                    # 如果这是一个新域，记录它并更新统计信息
                     if domain and domain not in self.domains_seen:
                         self.domains_seen.add(domain)
                         logger.debug(
                             "Filtered offsite request to %(domain)r: %(request)s" % {'domain': domain, 'request': x}
                         )
                         self.stats.inc_value('offsite/domains', spider=spider)
+
+                    # Update filtered requests stats
+                    # 更新过滤的请求统计信息
                     self.stats.inc_value('offsite/filtered', spider=spider)
             else:
+                # If the item is not a Request, yield it unchanged
+                # 如果项目不是一个Request，则不变地产生它
                 yield x
 
     def should_follow(self, request, spider):
+        """
+        Check if a request should be followed.
+        检查是否应该跟随请求。
+
+        This method checks if the hostname of the request URL matches the allowed
+        domains pattern.
+        此方法检查请求URL的主机名是否匹配允许的域模式。
+
+        Args:
+            request: The request to check.
+                    要检查的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+
+        Returns:
+            bool: True if the request should be followed, False otherwise.
+                 如果应该跟随请求，则为True；否则为False。
+        """
+        # Get the compiled regex pattern for allowed domains
+        # 获取允许的域的编译正则表达式模式
         regex = self.host_regex
+
+        # Get the hostname from the request URL
+        # 从请求URL获取主机名
         # hostname can be None for wrong urls (like javascript links)
+        # 对于错误的URL（如javascript链接），主机名可能为None
         host = urlparse_cached(request).hostname or ''
+
+        # Check if the hostname matches the allowed domains pattern
+        # 检查主机名是否匹配允许的域模式
         return bool(regex.search(host))
 
     def get_host_regex(self, spider):
-        """Override this method to implement a different offsite policy"""
+        """
+        Get a regex pattern for the allowed domains.
+        获取允许的域的正则表达式模式。
+
+        This method creates a regex pattern that matches hostnames belonging to
+        the domains specified in the spider's allowed_domains attribute.
+        此方法创建一个正则表达式模式，匹配属于爬虫的allowed_domains属性中指定的域的主机名。
+
+        Args:
+            spider: The spider whose allowed_domains attribute to use.
+                   使用其allowed_domains属性的爬虫。
+
+        Returns:
+            re.Pattern: A compiled regex pattern for the allowed domains.
+                       允许的域的编译正则表达式模式。
+
+        Note:
+            Override this method to implement a different offsite policy.
+            覆盖此方法以实现不同的站外策略。
+        """
+        # Get the allowed_domains attribute from the spider
+        # 从爬虫获取allowed_domains属性
         allowed_domains = getattr(spider, 'allowed_domains', None)
+
+        # If no allowed_domains are specified, allow all domains
+        # 如果未指定allowed_domains，则允许所有域
         if not allowed_domains:
             return re.compile('')  # allow all by default
+
+        # Compile patterns for validating domains
+        # 编译用于验证域的模式
         url_pattern = re.compile(r"^https?://.*$")
         port_pattern = re.compile(r":\d+$")
+
+        # Process each domain in allowed_domains
+        # 处理allowed_domains中的每个域
         domains = []
         for domain in allowed_domains:
+            # Skip None values
+            # 跳过None值
             if domain is None:
                 continue
+            # Warn about URL entries
+            # 警告URL条目
             elif url_pattern.match(domain):
                 message = ("allowed_domains accepts only domains, not URLs. "
                            f"Ignoring URL entry {domain} in allowed_domains.")
                 warnings.warn(message, URLWarning)
+            # Warn about domains with ports
+            # 警告带有端口的域
             elif port_pattern.search(domain):
                 message = ("allowed_domains accepts only domains without ports. "
                            f"Ignoring entry {domain} in allowed_domains.")
                 warnings.warn(message, PortWarning)
             else:
+                # Add valid domains to the list, escaping special regex characters
+                # 将有效域添加到列表中，转义特殊的正则表达式字符
                 domains.append(re.escape(domain))
+
+        # Create a regex pattern that matches the allowed domains and their subdomains
+        # 创建一个正则表达式模式，匹配允许的域及其子域
         regex = fr'^(.*\.)?({"|".join(domains)})$'
+
+        # Compile and return the regex pattern
+        # 编译并返回正则表达式模式
         return re.compile(regex)
 
     def spider_opened(self, spider):
+        """
+        Initialize middleware state when a spider is opened.
+        当爬虫打开时初始化中间件状态。
+
+        This method is called when a spider is opened. It initializes the regex
+        pattern for allowed domains and the set of seen domains.
+        当爬虫打开时调用此方法。它初始化允许的域的正则表达式模式和已见过的域集合。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
+        # Initialize the regex pattern for allowed domains
+        # 初始化允许的域的正则表达式模式
         self.host_regex = self.get_host_regex(spider)
+
+        # Initialize the set of seen domains
+        # 初始化已见过的域集合
         self.domains_seen = set()
 
 
 class URLWarning(Warning):
+    """
+    Warning raised when a URL is provided in allowed_domains.
+    当在allowed_domains中提供URL时引发的警告。
+
+    This warning is raised by the OffsiteMiddleware when it encounters a URL
+    (e.g., 'http://example.com') in the spider's allowed_domains attribute.
+    The allowed_domains attribute should only contain domain names without
+    the protocol (e.g., 'example.com').
+    当OffsiteMiddleware在爬虫的allowed_domains属性中遇到URL
+    （例如，'http://example.com'）时，会引发此警告。
+    allowed_domains属性应该只包含没有协议的域名（例如，'example.com'）。
+    """
     pass
 
 
 class PortWarning(Warning):
+    """
+    Warning raised when a domain with port is provided in allowed_domains.
+    当在allowed_domains中提供带有端口的域时引发的警告。
+
+    This warning is raised by the OffsiteMiddleware when it encounters a domain
+    with a port (e.g., 'example.com:8080') in the spider's allowed_domains attribute.
+    The allowed_domains attribute should only contain domain names without
+    ports (e.g., 'example.com').
+    当OffsiteMiddleware在爬虫的allowed_domains属性中遇到带有端口的域
+    （例如，'example.com:8080'）时，会引发此警告。
+    allowed_domains属性应该只包含没有端口的域名（例如，'example.com'）。
+    """
     pass