aio-scrapy 2.1.4__py3-none-any.whl → 2.1.7__py3-none-any.whl

aioscrapy/libs/pipelines/mongo.py

@@ -1,3 +1,14 @@
+"""
+MongoDB Pipeline for AioScrapy
+AioScrapy的MongoDB管道
+
+This module provides a pipeline for storing scraped items in a MongoDB database.
+It extends the base database pipeline to implement MongoDB-specific functionality
+for batch inserting items.
+此模块提供了一个用于将抓取的项目存储在MongoDB数据库中的管道。
+它扩展了基本数据库管道，以实现MongoDB特定的批量插入项目功能。
+"""
+
 from aioscrapy.db import db_manager
 from aioscrapy.libs.pipelines import DBPipelineBase
 
@@ -5,28 +16,114 @@ from aioscrapy.utils.log import logger
 
 
 class MongoPipeline(DBPipelineBase):
+    """
+    Pipeline for storing scraped items in a MongoDB database.
+    用于将抓取的项目存储在MongoDB数据库中的管道。
+
+    This pipeline extends the base database pipeline to implement MongoDB-specific
+    functionality for batch inserting items. It supports multiple database connections,
+    custom database names, and ordered/unordered inserts.
+    此管道扩展了基本数据库管道，以实现MongoDB特定的批量插入项目功能。
+    它支持多个数据库连接、自定义数据库名称和有序/无序插入。
+    """
 
     def __init__(self, settings, db_type: str):
+        """
+        Initialize the MongoDB pipeline.
+        初始化MongoDB管道。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+            db_type: The database type, should be 'mongo'.
+                    数据库类型，应为'mongo'。
+        """
         super().__init__(settings, db_type)
+
+        # Dictionary to cache database names by cache key
+        # 按缓存键缓存数据库名称的字典
         self.db_cache = {}
+
+        # Dictionary to cache ordered insert flags by cache key
+        # 按缓存键缓存有序插入标志的字典
         self.ordered_cache = {}
+
+        # Number of times to retry MongoDB operations on timeout
+        # MongoDB操作超时时重试的次数
         self.retry_times = settings.getint("MONGO_TIMEOUT_RETRY_TIMES", 3)
 
     @classmethod
     def from_settings(cls, settings):
+        """
+        Create a MongoPipeline instance from settings.
+        从设置创建MongoPipeline实例。
+
+        This is the factory method used by AioScrapy to create pipeline instances.
+        It initializes the pipeline with the appropriate database type ('mongo').
+        这是AioScrapy用于创建管道实例的工厂方法。
+        它使用适当的数据库类型（'mongo'）初始化管道。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+
+        Returns:
+            MongoPipeline: A new MongoPipeline instance.
+                          一个新的MongoPipeline实例。
+        """
         return cls(settings, 'mongo')
 
     def parse_item_to_cache(self, item: dict, save_info: dict):
+        """
+        Parse an item and add it to the cache.
+        解析项目并将其添加到缓存中。
+
+        This method overrides the base class method to handle MongoDB-specific
+        caching requirements, such as database names and ordered insert flags.
+        此方法覆盖基类方法，以处理MongoDB特定的缓存需求，如数据库名称和有序插入标志。
+
+        Args:
+            item: The item to cache.
+                 要缓存的项目。
+            save_info: Dictionary containing information about how to save the item.
+                      包含有关如何保存项目的信息的字典。
+                      Must contain 'table_name' and may contain 'db_name',
+                      'ordered', and 'db_alias'.
+                      必须包含'table_name'，可能包含'db_name'、'ordered'和'db_alias'。
+
+        Returns:
+            tuple: A tuple containing the cache key and the number of items in the cache.
+                  包含缓存键和缓存中项目数量的元组。
+
+        Raises:
+            AssertionError: If table_name is not provided in save_info.
+                           如果在save_info中未提供table_name。
+        """
+        # Extract information from save_info
+        # 从save_info中提取信息
         db_name = save_info.get('db_name')
         table_name = save_info.get('table_name')
         ordered = save_info.get('ordered', False)
+
+        # Ensure table_name is provided
+        # 确保提供了table_name
         assert table_name is not None, 'please set table_name'
+
+        # Get database aliases, defaulting to ['default']
+        # 获取数据库别名，默认为['default']
         db_alias = save_info.get('db_alias', ['default'])
+
+        # Convert string db_alias to list
+        # 将字符串db_alias转换为列表
         if isinstance(db_alias, str):
             db_alias = [db_alias]
 
+        # Generate a unique cache key based on the save_info
+        # 根据save_info生成唯一的缓存键
         cache_key = ''.join(db_alias) + (db_name or '') + table_name + str(ordered)
 
+        # If this is a new cache key, initialize the caches
+        # 如果这是一个新的缓存键，初始化缓存
         if self.table_cache.get(cache_key) is None:
             self.db_alias_cache[cache_key] = db_alias
             self.table_cache[cache_key] = table_name
@@ -34,23 +131,58 @@ class MongoPipeline(DBPipelineBase):
             self.ordered_cache[cache_key] = ordered
             self.item_cache[cache_key] = []
 
+        # Add the item to the cache
+        # 将项目添加到缓存
         self.item_cache[cache_key].append(item)
+
+        # Return the cache key and the number of items in the cache
+        # 返回缓存键和缓存中的项目数量
         return cache_key, len(self.item_cache[cache_key])
 
     async def _save(self, cache_key):
+        """
+        Save cached items with the given cache key to the MongoDB database.
+        将具有给定缓存键的缓存项目保存到MongoDB数据库。
+
+        This method implements the abstract _save method from the base class.
+        It retrieves the cached items for the given cache key, then executes
+        a batch insert operation on each configured database connection.
+        此方法实现了基类中的抽象_save方法。
+        它检索给定缓存键的缓存项目，然后在每个配置的数据库连接上执行批量插入操作。
+
+        Args:
+            cache_key: The cache key used to retrieve the cached items and metadata.
+                      用于检索缓存项目和元数据的缓存键。
+        """
+        # Get the table name from the cache
+        # 从缓存获取表名
         table_name = self.table_cache[cache_key]
         try:
+            # Process each database alias (connection) configured for this cache key
+            # 处理为此缓存键配置的每个数据库别名（连接）
             for alias in self.db_alias_cache[cache_key]:
                 try:
+                    # Get a MongoDB executor for this alias
+                    # 获取此别名的MongoDB执行器
                     executor = db_manager.mongo.executor(alias)
+
+                    # Execute the batch insert operation
+                    # 执行批量插入操作
                     result = await executor.insert(
                         table_name, self.item_cache[cache_key], db_name=self.db_cache[cache_key],
                         ordered=self.ordered_cache[cache_key], retry_times=self.retry_times
                     )
+
+                    # Log the result of the operation
+                    # 记录操作结果
                     logger.info(
                         f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{len(result.inserted_ids)}'
                     )
                 except Exception as e:
+                    # Log any errors that occur during the operation
+                    # 记录操作期间发生的任何错误
                     logger.exception(f'save data error, table:{alias}->{table_name}, err_msg:{e}')
         finally:
+            # Clear the cache after processing, regardless of success or failure
+            # 处理后清除缓存，无论成功或失败
             self.item_cache[cache_key] = []

aioscrapy/libs/pipelines/mysql.py

@@ -1,3 +1,14 @@
+"""
+MySQL Pipeline for AioScrapy
+AioScrapy的MySQL管道
+
+This module provides a pipeline for storing scraped items in a MySQL database.
+It extends the base database pipeline to implement MySQL-specific functionality
+for batch inserting items.
+此模块提供了一个用于将抓取的项目存储在MySQL数据库中的管道。
+它扩展了基本数据库管道，以实现MySQL特定的批量插入项目功能。
+"""
+
 from aioscrapy.db import db_manager
 from aioscrapy.libs.pipelines import DBPipelineBase
 
@@ -5,22 +16,78 @@ from aioscrapy.utils.log import logger
 
 
 class MysqlPipeline(DBPipelineBase):
+    """
+    Pipeline for storing scraped items in a MySQL database.
+    用于将抓取的项目存储在MySQL数据库中的管道。
+
+    This pipeline extends the base database pipeline to implement MySQL-specific
+    functionality for batch inserting items. It uses the database manager to
+    handle connections and transactions.
+    此管道扩展了基本数据库管道，以实现MySQL特定的批量插入项目功能。
+    它使用数据库管理器来处理连接和事务。
+    """
 
     @classmethod
     def from_settings(cls, settings):
+        """
+        Create a MysqlPipeline instance from settings.
+        从设置创建MysqlPipeline实例。
+
+        This is the factory method used by AioScrapy to create pipeline instances.
+        It initializes the pipeline with the appropriate database type ('mysql').
+        这是AioScrapy用于创建管道实例的工厂方法。
+        它使用适当的数据库类型（'mysql'）初始化管道。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+
+        Returns:
+            MysqlPipeline: A new MysqlPipeline instance.
+                          一个新的MysqlPipeline实例。
+        """
         return cls(settings, 'mysql')
 
     async def _save(self, cache_key):
+        """
+        Save cached items with the given cache key to the MySQL database.
+        将具有给定缓存键的缓存项目保存到MySQL数据库。
+
+        This method implements the abstract _save method from the base class.
+        It retrieves the cached items and SQL statement for the given cache key,
+        then executes a batch insert operation on each configured database connection.
+        此方法实现了基类中的抽象_save方法。
+        它检索给定缓存键的缓存项目和SQL语句，然后在每个配置的数据库连接上执行批量插入操作。
+
+        Args:
+            cache_key: The cache key used to retrieve the cached items, SQL statement,
+                      and other metadata needed for the database operation.
+                      用于检索缓存项目、SQL语句和数据库操作所需的其他元数据的缓存键。
+        """
+        # Get the table name from the cache
+        # 从缓存获取表名
         table_name = self.table_cache[cache_key]
         try:
+            # Process each database alias (connection) configured for this cache key
+            # 处理为此缓存键配置的每个数据库别名（连接）
             for alias in self.db_alias_cache[cache_key]:
+                # Get a database connection and cursor with ping to ensure the connection is alive
+                # 获取数据库连接和游标，并使用ping确保连接处于活动状态
                 async with db_manager.mysql.get(alias, ping=True) as (conn, cursor):
                     try:
+                        # Execute the batch insert operation
+                        # 执行批量插入操作
                         num = await cursor.executemany(
                             self.insert_sql_cache[cache_key], self.item_cache[cache_key]
                         )
+                        # Log the result of the operation
+                        # 记录操作结果
                         logger.info(f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{num}')
                     except Exception as e:
+                        # Log any errors that occur during the operation
+                        # 记录操作期间发生的任何错误
                         logger.exception(f'save data error, table:{alias}->{table_name}, err_msg:{e}')
         finally:
+            # Clear the cache after processing, regardless of success or failure
+            # 处理后清除缓存，无论成功或失败
             self.item_cache[cache_key] = []

aioscrapy/libs/pipelines/pg.py

@@ -1,3 +1,14 @@
+"""
+PostgreSQL Pipeline for AioScrapy
+AioScrapy的PostgreSQL管道
+
+This module provides a pipeline for storing scraped items in a PostgreSQL database.
+It extends the base database pipeline to implement PostgreSQL-specific functionality
+for batch inserting items.
+此模块提供了一个用于将抓取的项目存储在PostgreSQL数据库中的管道。
+它扩展了基本数据库管道，以实现PostgreSQL特定的批量插入项目功能。
+"""
+
 from aioscrapy.db import db_manager
 from aioscrapy.libs.pipelines import DBPipelineBase
 
@@ -5,22 +16,78 @@ from aioscrapy.utils.log import logger
 
 
 class PGPipeline(DBPipelineBase):
+    """
+    Pipeline for storing scraped items in a PostgreSQL database.
+    用于将抓取的项目存储在PostgreSQL数据库中的管道。
+
+    This pipeline extends the base database pipeline to implement PostgreSQL-specific
+    functionality for batch inserting items. It uses the database manager to handle
+    connections and transactions.
+    此管道扩展了基本数据库管道，以实现PostgreSQL特定的批量插入项目功能。
+    它使用数据库管理器来处理连接和事务。
+    """
 
     @classmethod
     def from_settings(cls, settings):
+        """
+        Create a PGPipeline instance from settings.
+        从设置创建PGPipeline实例。
+
+        This is the factory method used by AioScrapy to create pipeline instances.
+        It initializes the pipeline with the appropriate database type ('pg').
+        这是AioScrapy用于创建管道实例的工厂方法。
+        它使用适当的数据库类型（'pg'）初始化管道。
+
+        Args:
+            settings: The AioScrapy settings object.
+                     AioScrapy设置对象。
+
+        Returns:
+            PGPipeline: A new PGPipeline instance.
+                       一个新的PGPipeline实例。
+        """
         return cls(settings, 'pg')
 
     async def _save(self, cache_key):
+        """
+        Save cached items with the given cache key to the PostgreSQL database.
+        将具有给定缓存键的缓存项目保存到PostgreSQL数据库。
+
+        This method implements the abstract _save method from the base class.
+        It retrieves the cached items and SQL statement for the given cache key,
+        then executes a batch insert operation on each configured database connection.
+        此方法实现了基类中的抽象_save方法。
+        它检索给定缓存键的缓存项目和SQL语句，然后在每个配置的数据库连接上执行批量插入操作。
+
+        Args:
+            cache_key: The cache key used to retrieve the cached items, SQL statement,
+                      and other metadata needed for the database operation.
+                      用于检索缓存项目、SQL语句和数据库操作所需的其他元数据的缓存键。
+        """
+        # Get the table name from the cache
+        # 从缓存获取表名
         table_name = self.table_cache[cache_key]
         try:
+            # Process each database alias (connection) configured for this cache key
+            # 处理为此缓存键配置的每个数据库别名（连接）
             for alias in self.db_alias_cache[cache_key]:
+                # Get a database connection with context manager to ensure proper cleanup
+                # 使用上下文管理器获取数据库连接，以确保正确清理
                 async with db_manager.pg.get(alias) as conn:
                     try:
+                        # Execute the batch insert operation
+                        # 执行批量插入操作
                         num = await conn.executemany(
                             self.insert_sql_cache[cache_key], self.item_cache[cache_key]
                         )
+                        # Log the result of the operation
+                        # 记录操作结果
                         logger.info(f'table:{alias}->{table_name} sum:{len(self.item_cache[cache_key])} ok:{num}')
                     except Exception as e:
+                        # Log any errors that occur during the operation
+                        # 记录操作期间发生的任何错误
                         logger.exception(f'save data error, table:{alias}->{table_name}, err_msg:{e}')
         finally:
+            # Clear the cache after processing, regardless of success or failure
+            # 处理后清除缓存，无论成功或失败
             self.item_cache[cache_key] = []

aioscrapy/libs/spider/depth.py

@@ -1,54 +1,192 @@
 """
 Depth Spider Middleware
+深度爬虫中间件
 
-See documentation in docs/topics/spider-middleware.rst
+This middleware tracks the depth of requests and can be used to limit the maximum
+depth of crawls. It also adjusts request priorities based on depth and collects
+depth statistics.
+此中间件跟踪请求的深度，可用于限制爬取的最大深度。它还根据深度调整请求优先级
+并收集深度统计信息。
 """
 
 from aioscrapy.http import Request
-
 from aioscrapy.utils.log import logger
 
 
 class DepthMiddleware:
+    """
+    Spider middleware to track the depth of requests.
+    用于跟踪请求深度的爬虫中间件。
+
+    This middleware tracks how many nested links the crawler has followed from the
+    initial request (depth). It can be used to limit the maximum depth of crawls,
+    adjust request priorities based on depth, and collect depth statistics.
+    此中间件跟踪爬虫从初始请求开始已经跟随了多少层嵌套链接（深度）。它可用于限制
+    爬取的最大深度，根据深度调整请求优先级，并收集深度统计信息。
+    """
 
     def __init__(self, maxdepth, stats, verbose_stats=False, prio=1):
+        """
+        Initialize the depth middleware.
+        初始化深度中间件。
+
+        Args:
+            maxdepth: Maximum allowed depth. If None or 0, no limit is imposed.
+                     允许的最大深度。如果为None或0，则不施加限制。
+            stats: Stats collector instance.
+                  统计收集器实例。
+            verbose_stats: Whether to collect detailed stats for each depth level.
+                          是否收集每个深度级别的详细统计信息。
+                          Defaults to False.
+                          默认为False。
+            prio: Priority adjustment per depth level.
+                 每个深度级别的优先级调整。
+                 Defaults to 1.
+                 默认为1。
+        """
+        # Maximum allowed depth
+        # 允许的最大深度
         self.maxdepth = maxdepth
+
+        # Stats collector instance
+        # 统计收集器实例
         self.stats = stats
+
+        # Whether to collect detailed stats for each depth level
+        # 是否收集每个深度级别的详细统计信息
         self.verbose_stats = verbose_stats
+
+        # Priority adjustment per depth level
+        # 每个深度级别的优先级调整
         self.prio = prio
 
     @classmethod
     def from_crawler(cls, crawler):
+        """
+        Create a DepthMiddleware instance from a crawler.
+        从爬虫创建DepthMiddleware实例。
+
+        This is the factory method used by AioScrapy to create the middleware.
+        这是AioScrapy用于创建中间件的工厂方法。
+
+        Args:
+            crawler: The crawler that will use this middleware.
+                    将使用此中间件的爬虫。
+
+        Returns:
+            DepthMiddleware: A new DepthMiddleware instance.
+                            一个新的DepthMiddleware实例。
+        """
+        # Get settings from crawler
+        # 从爬虫获取设置
         settings = crawler.settings
+
+        # Get maximum depth from settings
+        # 从设置获取最大深度
         maxdepth = settings.getint('DEPTH_LIMIT')
+
+        # Get verbose stats setting
+        # 获取详细统计设置
         verbose = settings.getbool('DEPTH_STATS_VERBOSE')
+
+        # Get priority adjustment setting
+        # 获取优先级调整设置
         prio = settings.getint('DEPTH_PRIORITY')
+
+        # Create and return a new instance
+        # 创建并返回一个新实例
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     async def process_spider_output(self, response, result, spider):
+        """
+        Process the spider output to track request depth.
+        处理爬虫输出以跟踪请求深度。
+
+        This method processes each request yielded by the spider, tracks its depth,
+        adjusts its priority, and filters out requests that exceed the maximum depth.
+        此方法处理爬虫产生的每个请求，跟踪其深度，调整其优先级，并过滤掉超过最大深度的请求。
+
+        Args:
+            response: The response being processed.
+                     正在处理的响应。
+            result: The result returned by the spider.
+                   爬虫返回的结果。
+            spider: The spider that generated the result.
+                   生成结果的爬虫。
+
+        Returns:
+            An async generator yielding filtered requests.
+            一个产生过滤后请求的异步生成器。
+        """
         def _filter(request):
+            """
+            Filter function to process and possibly filter out requests based on depth.
+            基于深度处理并可能过滤掉请求的过滤函数。
+
+            Args:
+                request: The request to process.
+                         要处理的请求。
+
+            Returns:
+                bool: True if the request should be kept, False if it should be filtered out.
+                     如果应保留请求，则为True；如果应过滤掉请求，则为False。
+            """
+            # Only process Request objects
+            # 只处理Request对象
             if isinstance(request, Request):
+                # Calculate depth of this request (parent depth + 1)
+                # 计算此请求的深度（父深度 + 1）
                 depth = response.meta['depth'] + 1
+
+                # Store depth in request metadata
+                # 将深度存储在请求元数据中
                 request.meta['depth'] = depth
+
+                # Adjust priority based on depth if enabled
+                # 如果启用，则根据深度调整优先级
                 if self.prio:
                     request.priority -= depth * self.prio
+
+                # Check if request exceeds maximum depth
+                # 检查请求是否超过最大深度
                 if self.maxdepth and depth > self.maxdepth:
+                    # Log ignored request
+                    # 记录被忽略的请求
                     logger.debug("Ignoring link (depth > %(maxdepth)d): %(requrl)s " % {
                         'maxdepth': self.maxdepth, 'requrl': request.url
                     })
+                    # Filter out this request
+                    # 过滤掉此请求
                     return False
                 else:
+                    # Update depth statistics
+                    # 更新深度统计信息
                     if self.verbose_stats:
+                        # Increment count for this depth level
+                        # 增加此深度级别的计数
                         self.stats.inc_value(f'request_depth_count/{depth}',
                                              spider=spider)
+
+                    # Update maximum depth reached
+                    # 更新达到的最大深度
                     self.stats.max_value('request_depth_max', depth,
                                          spider=spider)
+            # Keep all non-Request objects and requests that didn't exceed max depth
+            # 保留所有非Request对象和未超过最大深度的请求
             return True
 
-        # base case (depth=0)
+        # Handle the base case (initial response with no depth)
+        # 处理基本情况（没有深度的初始响应）
         if 'depth' not in response.meta:
+            # Set depth to 0 for the initial response
+            # 为初始响应设置深度为0
             response.meta['depth'] = 0
+
+            # Update depth statistics for depth 0
+            # 更新深度0的深度统计信息
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
+        # Filter the results using the _filter function
+        # 使用_filter函数过滤结果
         return (r async for r in result or () if _filter(r))