aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/libs/extensions/closespider.py

@@ -1,7 +1,27 @@
-"""CloseSpider is an extension that forces spiders to be closed after certain
-conditions are met.
-
-See documentation in docs/topics/extensions.rst
+"""
+CloseSpider Extension for AioScrapy
+AioScrapy的CloseSpider扩展
+
+CloseSpider is an extension that forces spiders to be closed after certain
+conditions are met, such as a timeout, a maximum number of items scraped,
+pages downloaded, or errors encountered.
+CloseSpider是一个扩展，在满足特定条件后强制关闭爬虫，
+例如超时、抓取的最大项目数、下载的页面数或遇到的错误数。
+
+This extension can be configured using the following settings:
+此扩展可以使用以下设置进行配置：
+
+* CLOSESPIDER_TIMEOUT: Number of seconds after which the spider will be closed
+                      爬虫将被关闭的秒数
+* CLOSESPIDER_ITEMCOUNT: Maximum number of items to scrape before closing
+                        关闭前要抓取的最大项目数
+* CLOSESPIDER_PAGECOUNT: Maximum number of responses to download before closing
+                        关闭前要下载的最大响应数
+* CLOSESPIDER_ERRORCOUNT: Maximum number of errors to allow before closing
+                         关闭前允许的最大错误数
+
+See documentation in docs/topics/extensions.rst for more details.
+有关更多详细信息，请参阅docs/topics/extensions.rst中的文档。
 """
 import asyncio
 from typing import Optional
@@ -13,10 +33,34 @@ from aioscrapy.utils.tools import create_task
 
 
 class CloseSpider:
+    """
+    Extension to close spiders when certain conditions are met.
+    当满足特定条件时关闭爬虫的扩展。
+
+    This extension monitors the spider's activity and closes it when one of the
+    configured conditions is met: timeout, maximum number of items scraped,
+    maximum number of pages downloaded, or maximum number of errors encountered.
+    此扩展监控爬虫的活动，并在满足配置的条件之一时关闭它：
+    超时、抓取的最大项目数、下载的最大页面数或遇到的最大错误数。
+    """
 
     def __init__(self, crawler):
+        """
+        Initialize the CloseSpider extension.
+        初始化CloseSpider扩展。
+
+        Args:
+            crawler: The crawler instance that will use this extension.
+                    将使用此扩展的爬虫实例。
+
+        Raises:
+            NotConfigured: If none of the CLOSESPIDER_* settings are set.
+                          如果未设置任何CLOSESPIDER_*设置。
+        """
         self.crawler = crawler
 
+        # Dictionary of closing conditions from settings
+        # 来自设置的关闭条件字典
         self.close_on = {
             'timeout': crawler.settings.getfloat('CLOSESPIDER_TIMEOUT'),
             'itemcount': crawler.settings.getint('CLOSESPIDER_ITEMCOUNT'),
@@ -24,12 +68,21 @@ class CloseSpider:
             'errorcount': crawler.settings.getint('CLOSESPIDER_ERRORCOUNT'),
         }
 
+        # If no closing conditions are configured, don't enable the extension
+        # 如果未配置关闭条件，则不启用扩展
         if not any(self.close_on.values()):
             raise NotConfigured
 
+        # Counter for each condition
+        # 每个条件的计数器
         self.counter = defaultdict(int)
+
+        # Task for timeout handling
+        # 用于超时处理的任务
         self.task: Optional[asyncio.tasks.Task] = None
 
+        # Connect to signals based on configured conditions
+        # 根据配置的条件连接到信号
         if self.close_on.get('errorcount'):
             crawler.signals.connect(self.error_count, signal=signals.spider_error)
         if self.close_on.get('pagecount'):
@@ -38,34 +91,143 @@ class CloseSpider:
             crawler.signals.connect(self.timeout_close, signal=signals.spider_opened)
         if self.close_on.get('itemcount'):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+
+        # Always connect to spider_closed to clean up
+        # 始终连接到spider_closed以进行清理
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a CloseSpider instance from a crawler.
+        从爬虫创建CloseSpider实例。
+
+        This is the factory method used by AioScrapy to create extension instances.
+        这是AioScrapy用于创建扩展实例的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this extension.
+                    将使用此扩展的爬虫。
+
+        Returns:
+            CloseSpider: A new CloseSpider instance.
+                        一个新的CloseSpider实例。
+        """
         return cls(crawler)
 
     async def error_count(self, failure, response, spider):
+        """
+        Signal handler for the spider_error signal.
+        spider_error信号的处理程序。
+
+        Increments the error counter and closes the spider if the maximum
+        number of errors has been reached.
+        增加错误计数器，如果达到最大错误数，则关闭爬虫。
+
+        Args:
+            failure: The exception that was raised.
+                    引发的异常。
+            response: The response that caused the error.
+                     导致错误的响应。
+            spider: The spider that raised the exception.
+                   引发异常的爬虫。
+        """
+        # Increment the error counter
+        # 增加错误计数器
         self.counter['errorcount'] += 1
+
+        # Check if we've reached the maximum number of errors
+        # 检查是否达到最大错误数
         if self.counter['errorcount'] == self.close_on['errorcount']:
             create_task(self.crawler.engine.stop(reason='closespider_errorcount'))
 
     async def page_count(self, response, request, spider):
+        """
+        Signal handler for the response_received signal.
+        response_received信号的处理程序。
+
+        Increments the page counter and closes the spider if the maximum
+        number of pages has been downloaded.
+        增加页面计数器，如果下载的页面达到最大数量，则关闭爬虫。
+
+        Args:
+            response: The response that was received.
+                     接收到的响应。
+            request: The request that generated the response.
+                    生成响应的请求。
+            spider: The spider that generated the request.
+                   生成请求的爬虫。
+        """
+        # Increment the page counter
+        # 增加页面计数器
         self.counter['pagecount'] += 1
+
+        # Check if we've reached the maximum number of pages
+        # 检查是否达到最大页面数
         if self.counter['pagecount'] == self.close_on['pagecount']:
             create_task(self.crawler.engine.stop(reason='closespider_pagecount'))
 
     async def timeout_close(self, spider):
+        """
+        Signal handler for the spider_opened signal.
+        spider_opened信号的处理程序。
+
+        Starts a task that will close the spider after the configured timeout.
+        启动一个任务，该任务将在配置的超时后关闭爬虫。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
         async def close():
+            """
+            Inner function that waits for the timeout and then stops the engine.
+            等待超时然后停止引擎的内部函数。
+            """
             await asyncio.sleep(self.close_on['timeout'])
             create_task(self.crawler.engine.stop(reason='closespider_timeout'))
 
+        # Start the timeout task
+        # 启动超时任务
         self.task = create_task(close())
 
     async def item_scraped(self, item, spider):
+        """
+        Signal handler for the item_scraped signal.
+        item_scraped信号的处理程序。
+
+        Increments the item counter and closes the spider if the maximum
+        number of items has been scraped.
+        增加项目计数器，如果抓取的项目达到最大数量，则关闭爬虫。
+
+        Args:
+            item: The item that was scraped.
+                 抓取的项目。
+            spider: The spider that scraped the item.
+                   抓取项目的爬虫。
+        """
+        # Increment the item counter
+        # 增加项目计数器
         self.counter['itemcount'] += 1
+
+        # Check if we've reached the maximum number of items
+        # 检查是否达到最大项目数
         if self.counter['itemcount'] == self.close_on['itemcount']:
             create_task(self.crawler.engine.stop(reason='closespider_itemcount'))
 
     def spider_closed(self, spider):
+        """
+        Signal handler for the spider_closed signal.
+        spider_closed信号的处理程序。
+
+        Cancels the timeout task if it's still running when the spider is closed.
+        如果爬虫关闭时超时任务仍在运行，则取消该任务。
+
+        Args:
+            spider: The spider that was closed.
+                   被关闭的爬虫。
+        """
+        # Cancel the timeout task if it exists and is not done
+        # 如果超时任务存在且未完成，则取消它
         if self.task and not self.task.done():
             self.task.cancel()

aioscrapy/libs/extensions/corestats.py

@@ -1,5 +1,16 @@
 """
-Extension for collecting core stats like items scraped and start/finish times
+Core Stats Extension
+核心统计扩展
+
+This extension collects and records essential statistics about the crawling process,
+including start and finish times, elapsed time, number of items scraped, number of
+responses received, and information about dropped items.
+此扩展收集并记录有关爬取过程的基本统计信息，包括开始和结束时间、经过的时间、
+已抓取的项目数量、已接收的响应数量以及有关丢弃项目的信息。
+
+These statistics are useful for monitoring the performance and behavior of spiders,
+and can be accessed through the Scrapy stats collector.
+这些统计信息对于监控爬虫的性能和行为很有用，可以通过Scrapy统计收集器访问。
 """
 from datetime import datetime
 
@@ -7,40 +18,179 @@ from aioscrapy import signals
 
 
 class CoreStats:
+    """
+    Extension for collecting core statistics about the crawling process.
+    用于收集有关爬取过程的核心统计信息的扩展。
+
+    This extension hooks into various Scrapy signals to collect statistics about
+    the crawling process, such as start and finish times, number of items scraped,
+    number of responses received, and information about dropped items.
+    此扩展挂钩到各种Scrapy信号，以收集有关爬取过程的统计信息，例如开始和结束时间、
+    已抓取的项目数量、已接收的响应数量以及有关丢弃项目的信息。
+    """
 
     def __init__(self, stats):
+        """
+        Initialize the CoreStats extension.
+        初始化CoreStats扩展。
+
+        Args:
+            stats: The Scrapy stats collector.
+                  Scrapy统计收集器。
+        """
+        # Stats collector
+        # 统计收集器
         self.stats = stats
+
+        # Spider start time (will be set when spider opens)
+        # 爬虫开始时间（将在爬虫打开时设置）
         self.start_time = None
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a CoreStats instance from a crawler.
+        从爬虫创建CoreStats实例。
+
+        This is the factory method used by Scrapy to create the extension.
+        这是Scrapy用于创建扩展的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this extension.
+                    将使用此扩展的爬虫。
+
+        Returns:
+            CoreStats: A new CoreStats instance.
+                      一个新的CoreStats实例。
+        """
+        # Create a new instance with the crawler's stats collector
+        # 使用爬虫的统计收集器创建一个新实例
         o = cls(crawler.stats)
+
+        # Connect to signals
+        # 连接到信号
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         crawler.signals.connect(o.item_scraped, signal=signals.item_scraped)
         crawler.signals.connect(o.item_dropped, signal=signals.item_dropped)
         crawler.signals.connect(o.response_received, signal=signals.response_received)
+
+        # Return the new instance
+        # 返回新实例
         return o
 
     def spider_opened(self, spider):
+        """
+        Handle the spider_opened signal.
+        处理spider_opened信号。
+
+        This method is called when a spider is opened. It records the start time
+        of the spider.
+        当爬虫打开时调用此方法。它记录爬虫的开始时间。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
+        # Record the start time
+        # 记录开始时间
         self.start_time = datetime.now()
+
+        # Store the start time in the stats
+        # 将开始时间存储在统计信息中
         self.stats.set_value('start_time', str(self.start_time), spider=spider)
 
     def spider_closed(self, spider, reason):
+        """
+        Handle the spider_closed signal.
+        处理spider_closed信号。
+
+        This method is called when a spider is closed. It calculates and records
+        the finish time, elapsed time, and finish reason.
+        当爬虫关闭时调用此方法。它计算并记录结束时间、经过的时间和结束原因。
+
+        Args:
+            spider: The spider that was closed.
+                   被关闭的爬虫。
+            reason: The reason why the spider was closed.
+                   爬虫被关闭的原因。
+        """
+        # Record the finish time
+        # 记录结束时间
         finish_time = datetime.now()
+
+        # Calculate elapsed time
+        # 计算经过的时间
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
+
+        # Store finish statistics in the stats
+        # 将结束统计信息存储在统计信息中
         self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
         self.stats.set_value('finish_time', str(finish_time), spider=spider)
         self.stats.set_value('finish_reason', reason, spider=spider)
 
     def item_scraped(self, item, spider):
+        """
+        Handle the item_scraped signal.
+        处理item_scraped信号。
+
+        This method is called when an item is scraped by a spider. It increments
+        the item_scraped_count statistic.
+        当爬虫抓取项目时调用此方法。它增加item_scraped_count统计信息。
+
+        Args:
+            item: The item that was scraped.
+                 被抓取的项目。
+            spider: The spider that scraped the item.
+                   抓取项目的爬虫。
+        """
+        # Increment the item scraped count
+        # 增加已抓取项目计数
         self.stats.inc_value('item_scraped_count', spider=spider)
 
     def response_received(self, spider):
+        """
+        Handle the response_received signal.
+        处理response_received信号。
+
+        This method is called when a response is received by a spider. It increments
+        the response_received_count statistic.
+        当爬虫接收到响应时调用此方法。它增加response_received_count统计信息。
+
+        Args:
+            spider: The spider that received the response.
+                   接收响应的爬虫。
+        """
+        # Increment the response received count
+        # 增加已接收响应计数
         self.stats.inc_value('response_received_count', spider=spider)
 
     def item_dropped(self, item, spider, exception):
+        """
+        Handle the item_dropped signal.
+        处理item_dropped信号。
+
+        This method is called when an item is dropped by a spider. It increments
+        the item_dropped_count statistic and records the reason why the item was dropped.
+        当爬虫丢弃项目时调用此方法。它增加item_dropped_count统计信息，并记录项目被丢弃的原因。
+
+        Args:
+            item: The item that was dropped.
+                 被丢弃的项目。
+            spider: The spider that dropped the item.
+                   丢弃项目的爬虫。
+            exception: The exception that caused the item to be dropped.
+                      导致项目被丢弃的异常。
+        """
+        # Get the reason from the exception class name
+        # 从异常类名获取原因
         reason = exception.__class__.__name__
+
+        # Increment the item dropped count
+        # 增加已丢弃项目计数
         self.stats.inc_value('item_dropped_count', spider=spider)
+
+        # Increment the count for this specific reason
+        # 增加此特定原因的计数
         self.stats.inc_value(f'item_dropped_reasons_count/{reason}', spider=spider)

aioscrapy/libs/extensions/logstats.py

@@ -1,3 +1,14 @@
+"""
+Log Stats Extension
+日志统计扩展
+
+This extension logs basic crawling statistics periodically during the spider run.
+It provides information about the number of pages crawled, items scraped, and their
+respective rates per minute, which is useful for monitoring the progress and
+performance of spiders in real-time.
+此扩展在爬虫运行期间定期记录基本的爬取统计信息。它提供有关已爬取的页面数量、
+已抓取的项目数量及其各自的每分钟速率的信息，这对于实时监控爬虫的进度和性能很有用。
+"""
 import asyncio
 
 from aioscrapy import signals
@@ -7,44 +18,177 @@ from aioscrapy.utils.tools import create_task
 
 
 class LogStats:
-    """Log basic scraping stats periodically"""
+    """
+    Extension for logging basic crawling statistics periodically.
+    用于定期记录基本爬取统计信息的扩展。
+
+    This extension logs information about the number of pages crawled and items
+    scraped, along with their respective rates per minute. The statistics are
+    logged at regular intervals during the spider run, providing real-time
+    feedback on the spider's performance.
+    此扩展记录有关已爬取的页面数量和已抓取的项目数量的信息，以及它们各自的
+    每分钟速率。统计信息在爬虫运行期间以固定的时间间隔记录，提供有关爬虫性能
+    的实时反馈。
+    """
 
     def __init__(self, stats, interval=60.0):
+        """
+        Initialize the LogStats extension.
+        初始化LogStats扩展。
+
+        Args:
+            stats: The Scrapy stats collector.
+                  Scrapy统计收集器。
+            interval: The time interval (in seconds) between log messages.
+                     日志消息之间的时间间隔（以秒为单位）。
+                     Defaults to 60.0 seconds.
+                     默认为60.0秒。
+        """
+        # Stats collector
+        # 统计收集器
         self.stats = stats
+
+        # Interval between log messages (in seconds)
+        # 日志消息之间的间隔（以秒为单位）
         self.interval = interval
+
+        # Multiplier to convert stats to per-minute rates
+        # 将统计数据转换为每分钟速率的乘数
         self.multiplier = 60.0 / self.interval
+
+        # Async task for periodic logging
+        # 用于定期记录的异步任务
         self.task = None
+
+        # Previous values for calculating rates
+        # 用于计算速率的先前值
         self.pagesprev = 0
         self.itemsprev = 0
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a LogStats instance from a crawler.
+        从爬虫创建LogStats实例。
+
+        This is the factory method used by Scrapy to create the extension.
+        这是Scrapy用于创建扩展的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this extension.
+                    将使用此扩展的爬虫。
+
+        Returns:
+            LogStats: A new LogStats instance.
+                     一个新的LogStats实例。
+
+        Raises:
+            NotConfigured: If LOGSTATS_INTERVAL is not set or is zero in the settings.
+                          如果在设置中未设置LOGSTATS_INTERVAL或其值为零。
+        """
+        # Get the log interval from settings
+        # 从设置获取日志间隔
         interval = crawler.settings.getfloat('LOGSTATS_INTERVAL')
+
+        # If no interval is configured, disable the extension
+        # 如果未配置间隔，则禁用扩展
         if not interval:
             raise NotConfigured
+
+        # Create a new instance with the crawler's stats collector and the configured interval
+        # 使用爬虫的统计收集器和配置的间隔创建一个新实例
         o = cls(crawler.stats, interval)
+
+        # Connect to signals
+        # 连接到信号
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
+
+        # Return the new instance
+        # 返回新实例
         return o
 
     def spider_opened(self, spider):
+        """
+        Handle the spider_opened signal.
+        处理spider_opened信号。
+
+        This method is called when a spider is opened. It starts the periodic
+        logging task.
+        当爬虫打开时调用此方法。它启动定期记录任务。
+
+        Args:
+            spider: The spider that was opened.
+                   被打开的爬虫。
+        """
+        # Start the periodic logging task
+        # 启动定期记录任务
         self.task = create_task(self.log(spider))
 
     async def log(self, spider):
+        """
+        Log the current crawling statistics and schedule the next log.
+        记录当前爬取统计信息并安排下一次记录。
+
+        This method retrieves the current statistics, calculates the rates,
+        logs the information, and then schedules itself to run again after
+        the configured interval.
+        此方法检索当前统计信息，计算速率，记录信息，然后安排自己在配置的
+        间隔后再次运行。
+
+        Args:
+            spider: The spider whose statistics to log.
+                   要记录其统计信息的爬虫。
+        """
+        # Wait for the configured interval
+        # 等待配置的间隔
         await asyncio.sleep(self.interval)
+
+        # Get current statistics
+        # 获取当前统计信息
         items = self.stats.get_value('item_scraped_count', 0)
         pages = self.stats.get_value('response_received_count', 0)
+
+        # Calculate rates (per minute)
+        # 计算速率（每分钟）
         irate = (items - self.itemsprev) * self.multiplier
         prate = (pages - self.pagesprev) * self.multiplier
+
+        # Update previous values for next calculation
+        # 更新先前值以供下次计算
         self.pagesprev, self.itemsprev = pages, items
 
+        # Prepare log message
+        # 准备日志消息
         msg = ("<%(spider_name)s> Crawled %(pages)d pages (at %(pagerate)d pages/min), "
                "scraped %(items)d items (at %(itemrate)d items/min)")
         log_args = {'pages': pages, 'pagerate': prate, 'spider_name': spider.name,
                     'items': items, 'itemrate': irate}
+
+        # Log the statistics
+        # 记录统计信息
         logger.info(msg % log_args)
+
+        # Schedule the next log
+        # 安排下一次记录
         self.task = create_task(self.log(spider))
 
     def spider_closed(self, spider, reason):
+        """
+        Handle the spider_closed signal.
+        处理spider_closed信号。
+
+        This method is called when a spider is closed. It cancels the periodic
+        logging task if it's still running.
+        当爬虫关闭时调用此方法。如果定期记录任务仍在运行，它会取消该任务。
+
+        Args:
+            spider: The spider that was closed.
+                   被关闭的爬虫。
+            reason: The reason why the spider was closed.
+                   爬虫被关闭的原因。
+        """
+        # Cancel the logging task if it's still running
+        # 如果记录任务仍在运行，则取消它
         if self.task and not self.task.done():
             self.task.cancel()