aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/libs/spider/urllength.py

@@ -1,7 +1,12 @@
 """
 Url Length Spider Middleware
+URL长度爬虫中间件
 
-See documentation in docs/topics/spider-middleware.rst
+This middleware filters out requests with URLs that exceed a configurable maximum
+length. It helps prevent issues with excessively long URLs that might cause problems
+with servers, proxies, or browsers.
+此中间件过滤掉URL超过可配置最大长度的请求。它有助于防止过长的URL可能导致
+服务器、代理或浏览器出现问题。
 """
 
 from aioscrapy.exceptions import NotConfigured
@@ -10,28 +15,119 @@ from aioscrapy.utils.log import logger
 
 
 class UrlLengthMiddleware:
+    """
+    Spider middleware to filter out requests with excessively long URLs.
+    用于过滤掉URL过长的请求的爬虫中间件。
+
+    This middleware checks the length of URLs in requests and filters out those
+    that exceed a configurable maximum length. This helps prevent issues with
+    servers, proxies, or browsers that might have trouble handling very long URLs.
+    此中间件检查请求中URL的长度，并过滤掉超过可配置最大长度的URL。
+    这有助于防止服务器、代理或浏览器在处理非常长的URL时可能遇到的问题。
+    """
 
     def __init__(self, maxlength):
+        """
+        Initialize the URL length middleware.
+        初始化URL长度中间件。
+
+        Args:
+            maxlength: The maximum allowed URL length in characters.
+                      允许的URL最大长度（以字符为单位）。
+        """
+        # Maximum allowed URL length
+        # 允许的URL最大长度
         self.maxlength = maxlength
 
     @classmethod
     def from_settings(cls, settings):
+        """
+        Create a UrlLengthMiddleware instance from settings.
+        从设置创建UrlLengthMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+
+        Returns:
+            UrlLengthMiddleware: A new UrlLengthMiddleware instance.
+                                一个新的UrlLengthMiddleware实例。
+
+        Raises:
+            NotConfigured: If URLLENGTH_LIMIT is not set or is zero in the settings.
+                          如果在设置中未设置URLLENGTH_LIMIT或其值为零。
+        """
+        # Get the maximum URL length from settings
+        # 从设置获取最大URL长度
         maxlength = settings.getint('URLLENGTH_LIMIT')
+
+        # If no maximum length is configured, disable the middleware
+        # 如果未配置最大长度，则禁用中间件
         if not maxlength:
             raise NotConfigured
+
+        # Create and return a new instance
+        # 创建并返回一个新实例
         return cls(maxlength)
 
     async def process_spider_output(self, response, result, spider):
+        """
+        Process the spider output to filter out requests with long URLs.
+        处理爬虫输出以过滤掉具有长URL的请求。
+
+        This method processes each request yielded by the spider and filters out
+        those with URLs that exceed the configured maximum length.
+        此方法处理爬虫产生的每个请求，并过滤掉URL超过配置的最大长度的请求。
+
+        Args:
+            response: The response being processed.
+                     正在处理的响应。
+            result: The result returned by the spider.
+                   爬虫返回的结果。
+            spider: The spider that generated the result.
+                   生成结果的爬虫。
+
+        Returns:
+            An async generator yielding filtered requests and other items.
+            一个产生过滤后的请求和其他项目的异步生成器。
+        """
         def _filter(request):
+            """
+            Filter function to check if a request's URL is too long.
+            检查请求的URL是否过长的过滤函数。
+
+            Args:
+                request: The request to check.
+                         要检查的请求。
+
+            Returns:
+                bool: True if the request should be kept, False if it should be filtered out.
+                     如果应保留请求，则为True；如果应过滤掉请求，则为False。
+            """
+            # Check if the item is a Request and if its URL exceeds the maximum length
+            # 检查项目是否为Request，以及其URL是否超过最大长度
             if isinstance(request, Request) and len(request.url) > self.maxlength:
+                # Log the ignored request
+                # 记录被忽略的请求
                 logger.info(
                     "Ignoring link (url length > %(maxlength)d): %(url)s " % {
                         'maxlength': self.maxlength, 'url': request.url
                     }
                 )
+                # Update statistics
+                # 更新统计信息
                 spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
+                # Filter out this request
+                # 过滤掉此请求
                 return False
             else:
+                # Keep all other items
+                # 保留所有其他项目
                 return True
 
+        # Filter the results using the _filter function
+        # 使用_filter函数过滤结果
         return (r async for r in result or () if _filter(r))

aioscrapy/link.py

@@ -1,41 +1,122 @@
 """
-This module defines the Link object used in Link extractors.
+Link Module
+链接模块
+
+This module defines the Link object used in Link extractors. The Link class
+represents an extracted link from a web page, containing information such as
+the URL, anchor text, URL fragment, and nofollow status.
+此模块定义了链接提取器中使用的Link对象。Link类表示从网页中提取的链接，
+包含URL、锚文本、URL片段和nofollow状态等信息。
 
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
+有关实际链接提取器的实现，请参见scrapy.linkextractors，
+或其文档：docs/topics/link-extractors.rst
 """
 
 
 class Link:
-    """Link objects represent an extracted link by the LinkExtractor.
+    """
+    Represents an extracted link from a web page.
+    表示从网页中提取的链接。
+
+    Link objects are created by LinkExtractors to represent links extracted from web pages.
+    Each Link object contains information about the URL, anchor text, URL fragment, and
+    nofollow status of the extracted link.
+    Link对象由LinkExtractor创建，用于表示从网页中提取的链接。每个Link对象包含有关
+    提取链接的URL、锚文本、URL片段和nofollow状态的信息。
 
     Using the anchor tag sample below to illustrate the parameters::
+    使用下面的锚标签示例来说明参数::
 
             <a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
 
-    :param url: the absolute url being linked to in the anchor tag.
-                From the sample, this is ``https://example.com/nofollow.html``.
+    Args:
+        url: The absolute URL being linked to in the anchor tag.
+             锚标签中链接到的绝对URL。
+             From the sample, this is ``https://example.com/nofollow.html``.
+             从示例中，这是``https://example.com/nofollow.html``。
 
-    :param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
+        text: The text in the anchor tag.
+              锚标签中的文本。
+              From the sample, this is ``Dont follow this one``.
+              从示例中，这是``Dont follow this one``。
+              Defaults to an empty string.
+              默认为空字符串。
 
-    :param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
+        fragment: The part of the URL after the hash symbol.
+                 URL中哈希符号后的部分。
+                 From the sample, this is ``foo``.
+                 从示例中，这是``foo``。
+                 Defaults to an empty string.
+                 默认为空字符串。
 
-    :param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
-                    of the anchor tag.
+        nofollow: An indication of the presence or absence of a nofollow value
+                 in the ``rel`` attribute of the anchor tag.
+                 表示锚标签的``rel``属性中是否存在nofollow值。
+                 Defaults to False.
+                 默认为False。
     """
 
+    # Define __slots__ to save memory when creating many Link objects
+    # 定义__slots__以在创建多个Link对象时节省内存
     __slots__ = ['url', 'text', 'fragment', 'nofollow']
 
     def __init__(self, url, text='', fragment='', nofollow=False):
+        """
+        Initialize a Link object.
+        初始化Link对象。
+
+        Args:
+            url: The absolute URL being linked to.
+                 被链接到的绝对URL。
+            text: The anchor text of the link.
+                  链接的锚文本。
+                  Defaults to an empty string.
+                  默认为空字符串。
+            fragment: The URL fragment (part after the # symbol).
+                     URL片段（#符号后的部分）。
+                     Defaults to an empty string.
+                     默认为空字符串。
+            nofollow: Whether the link has a nofollow attribute.
+                     链接是否具有nofollow属性。
+                     Defaults to False.
+                     默认为False。
+
+        Raises:
+            TypeError: If the URL is not a string.
+                      如果URL不是字符串。
+        """
+        # Ensure the URL is a string
+        # 确保URL是字符串
         if not isinstance(url, str):
             got = url.__class__.__name__
             raise TypeError(f"Link urls must be str objects, got {got}")
+
+        # Store the link attributes
+        # 存储链接属性
         self.url = url
         self.text = text
         self.fragment = fragment
         self.nofollow = nofollow
 
     def __eq__(self, other):
+        """
+        Compare two Link objects for equality.
+        比较两个Link对象是否相等。
+
+        Two Link objects are considered equal if they have the same URL, text,
+        fragment, and nofollow status.
+        如果两个Link对象具有相同的URL、文本、片段和nofollow状态，则它们被认为是相等的。
+
+        Args:
+            other: The other Link object to compare with.
+                  要比较的其他Link对象。
+
+        Returns:
+            bool: True if the Link objects are equal, False otherwise.
+                 如果Link对象相等，则为True，否则为False。
+        """
         return (
             self.url == other.url
             and self.text == other.text
@@ -44,9 +125,35 @@ class Link:
         )
 
     def __hash__(self):
+        """
+        Calculate a hash value for the Link object.
+        计算Link对象的哈希值。
+
+        This method is implemented to allow Link objects to be used as dictionary
+        keys or in sets. The hash value is based on the URL, text, fragment, and
+        nofollow status.
+        实现此方法是为了允许将Link对象用作字典键或集合中的元素。哈希值基于URL、
+        文本、片段和nofollow状态。
+
+        Returns:
+            int: A hash value for the Link object.
+                Link对象的哈希值。
+        """
         return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
 
     def __repr__(self):
+        """
+        Return a string representation of the Link object.
+        返回Link对象的字符串表示。
+
+        This method returns a string that, when passed to eval(), would create a
+        new Link object with the same attributes.
+        此方法返回一个字符串，当传递给eval()时，将创建一个具有相同属性的新Link对象。
+
+        Returns:
+            str: A string representation of the Link object.
+                Link对象的字符串表示。
+        """
         return (
             f'Link(url={self.url!r}, text={self.text!r}, '
             f'fragment={self.fragment!r}, nofollow={self.nofollow!r})'

aioscrapy/logformatter.py

@@ -1,7 +1,23 @@
+"""
+Log Formatter Module
+日志格式化模块
+
+This module provides the LogFormatter class, which formats log messages for various
+events that occur during the crawling process, such as crawling a page, scraping an item,
+dropping an item, or encountering errors.
+此模块提供LogFormatter类，用于格式化爬取过程中发生的各种事件的日志消息，
+例如爬取页面、抓取项目、丢弃项目或遇到错误。
+
+The LogFormatter can be customized by subclassing and overriding its methods to
+change the format of log messages.
+可以通过子类化并覆盖其方法来自定义LogFormatter，以更改日志消息的格式。
+"""
 import os
 
 from aioscrapy.utils.request import referer_str
 
+# Standard log message templates
+# 标准日志消息模板
 SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
 DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
 CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
@@ -12,11 +28,48 @@ DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
 
 
 class LogFormatter:
+    """
+    Formats log messages for various events during the crawling process.
+    格式化爬取过程中各种事件的日志消息。
+
+    This class provides methods to format log messages for events such as crawling a page,
+    scraping an item, dropping an item, or encountering errors. It can be customized by
+    subclassing and overriding its methods to change the format of log messages.
+    此类提供方法来格式化爬取页面、抓取项目、丢弃项目或遇到错误等事件的日志消息。
+    可以通过子类化并覆盖其方法来自定义它，以更改日志消息的格式。
+    """
+
     @staticmethod
     def crawled(request, response, spider):
-        """Logs a message when the crawler finds a webpage."""
+        """
+        Format a log message for a crawled page.
+        格式化已爬取页面的日志消息。
+
+        This method is called when the crawler successfully downloads a webpage.
+        It formats a log message that includes the response status, request URL,
+        request flags, referer, and response flags.
+        当爬虫成功下载网页时调用此方法。它格式化一条包含响应状态、请求URL、
+        请求标志、引用者和响应标志的日志消息。
+
+        Args:
+            request: The request that was made.
+                    发出的请求。
+            response: The response that was received.
+                     接收到的响应。
+            spider: The spider that made the request.
+                   发出请求的爬虫。
+
+        Returns:
+            dict: A dictionary with log level and message.
+                 包含日志级别和消息的字典。
+        """
+        # Format request and response flags if they exist
+        # 如果存在，则格式化请求和响应标志
         request_flags = f' {str(request.flags)}' if request.flags else ''
         response_flags = f' {str(response.flags)}' if response.flags else ''
+
+        # Return a dictionary with log level and formatted message
+        # 返回包含日志级别和格式化消息的字典
         return {
             '_Logger__level': "DEBUG",
             '_Logger__message': CRAWLEDMSG % {
@@ -31,8 +84,32 @@ class LogFormatter:
 
     @staticmethod
     def scraped(item, response, spider):
-        """Logs a message when an item is scraped by a spider."""
+        """
+        Format a log message for a scraped item.
+        格式化已抓取项目的日志消息。
+
+        This method is called when a spider successfully scrapes an item from a response.
+        It formats a log message that includes the response source and the item details.
+        当爬虫成功从响应中抓取项目时调用此方法。它格式化一条包含响应源和项目详细信息的日志消息。
+
+        Args:
+            item: The item that was scraped.
+                 被抓取的项目。
+            response: The response from which the item was scraped.
+                     项目被抓取的响应。
+            spider: The spider that scraped the item.
+                   抓取项目的爬虫。
+
+        Returns:
+            dict: A dictionary with log level and message.
+                 包含日志级别和消息的字典。
+        """
+        # Use the response as the source
+        # 使用响应作为源
         src = response
+
+        # Return a dictionary with log level and formatted message
+        # 返回包含日志级别和格式化消息的字典
         return {
             '_Logger__level': "DEBUG",
             '_Logger__message': SCRAPEDMSG % {
@@ -43,7 +120,32 @@ class LogFormatter:
 
     @staticmethod
     def dropped(item, exception, response, spider):
-        """Logs a message when an item is dropped while it is passing through the item pipeline."""
+        """
+        Format a log message for a dropped item.
+        格式化已丢弃项目的日志消息。
+
+        This method is called when an item is dropped while passing through the item pipeline.
+        It formats a log message that includes the exception that caused the item to be dropped
+        and the item details.
+        当项目在通过项目管道时被丢弃时调用此方法。它格式化一条包含导致项目被丢弃的异常
+        和项目详细信息的日志消息。
+
+        Args:
+            item: The item that was dropped.
+                 被丢弃的项目。
+            exception: The exception that caused the item to be dropped.
+                      导致项目被丢弃的异常。
+            response: The response from which the item was scraped.
+                     项目被抓取的响应。
+            spider: The spider that scraped the item.
+                   抓取项目的爬虫。
+
+        Returns:
+            dict: A dictionary with log level and message.
+                 包含日志级别和消息的字典。
+        """
+        # Return a dictionary with log level and formatted message
+        # 返回包含日志级别和格式化消息的字典
         return {
             '_Logger__level': "WARNING",
             '_Logger__message': DROPPEDMSG % {
@@ -54,21 +156,64 @@ class LogFormatter:
 
     @staticmethod
     def item_error(item, exception, response, spider):
-        """Logs a message when an item causes an error while it is passing
-        through the item pipeline.
+        """
+        Format a log message for an item processing error.
+        格式化项目处理错误的日志消息。
+
+        This method is called when an item causes an error while passing through the item pipeline.
+        It formats a log message that includes the item details.
+        当项目在通过项目管道时导致错误时调用此方法。它格式化一条包含项目详细信息的日志消息。
+
+        Args:
+            item: The item that caused the error.
+                 导致错误的项目。
+            exception: The exception that was raised.
+                      引发的异常。
+            response: The response from which the item was scraped.
+                     项目被抓取的响应。
+            spider: The spider that scraped the item.
+                   抓取项目的爬虫。
+
+        Returns:
+            str: A formatted log message.
+                格式化的日志消息。
 
         .. versionadded:: 2.0
         """
+        # Return a formatted message with the item details
+        # 返回包含项目详细信息的格式化消息
         return ITEMERRORMSG % {
             'item': item,
         }
 
     @staticmethod
     def spider_error(failure, request, response, spider):
-        """Logs an error message from a spider.
+        """
+        Format a log message for a spider error.
+        格式化爬虫错误的日志消息。
+
+        This method is called when a spider raises an exception while processing a response.
+        It formats a log message that includes the request URL and referer.
+        当爬虫在处理响应时引发异常时调用此方法。它格式化一条包含请求URL和引用者的日志消息。
+
+        Args:
+            failure: The failure that occurred.
+                    发生的失败。
+            request: The request that was being processed.
+                    正在处理的请求。
+            response: The response that was being processed.
+                     正在处理的响应。
+            spider: The spider that raised the exception.
+                   引发异常的爬虫。
+
+        Returns:
+            str: A formatted log message.
+                格式化的日志消息。
 
         .. versionadded:: 2.0
         """
+        # Return a formatted message with the request and referer
+        # 返回包含请求和引用者的格式化消息
         return SPIDERERRORMSG % {
             'request': request,
             'referer': referer_str(request),
@@ -76,19 +221,65 @@ class LogFormatter:
 
     @staticmethod
     def download_error(failure, request, spider, errmsg=None):
-        """Logs a download error message from a spider (typically coming from
-        the engine).
+        """
+        Format a log message for a download error.
+        格式化下载错误的日志消息。
+
+        This method is called when there is an error downloading a request.
+        It formats a log message that includes the request URL and optionally the error message.
+        当下载请求时出错时调用此方法。它格式化一条包含请求URL和可选的错误消息的日志消息。
+
+        Args:
+            failure: The failure that occurred.
+                    发生的失败。
+            request: The request that failed to download.
+                    下载失败的请求。
+            spider: The spider that made the request.
+                   发出请求的爬虫。
+            errmsg: An optional error message.
+                   可选的错误消息。
+                   Defaults to None.
+                   默认为None。
+
+        Returns:
+            str: A formatted log message.
+                格式化的日志消息。
 
         .. versionadded:: 2.0
         """
+        # Prepare arguments for the message
+        # 准备消息的参数
         args = {'request': request}
+
+        # Choose the appropriate message template based on whether an error message is provided
+        # 根据是否提供错误消息选择适当的消息模板
         if errmsg:
             msg = DOWNLOADERRORMSG_LONG
             args['errmsg'] = errmsg
         else:
             msg = DOWNLOADERRORMSG_SHORT
+
+        # Return the formatted message
+        # 返回格式化的消息
         return msg % args
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a LogFormatter instance from a crawler.
+        从爬虫创建LogFormatter实例。
+
+        This is the factory method used by AioScrapy to create the log formatter.
+        这是AioScrapy用于创建日志格式化器的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this log formatter.
+                    将使用此日志格式化器的爬虫。
+
+        Returns:
+            LogFormatter: A new LogFormatter instance.
+                         一个新的LogFormatter实例。
+        """
+        # Create and return a new instance
+        # 创建并返回一个新实例
         return cls()