crawlo 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

crawlo/pipelines/mysql_pipeline.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 import asyncio
 import aiomysql
-from typing import Optional
+from typing import Optional, List, Dict
 from asyncmy import create_pool
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import ItemDiscard
-from crawlo.utils.db_helper import make_insert_sql, logger
+from crawlo.utils.db_helper import make_insert_sql, make_batch_sql, logger
 
 
 class AsyncmyMySQLPipeline:
@@ -24,6 +24,11 @@ class AsyncmyMySQLPipeline:
                 f"{crawler.spider.name}_items"
         )
 
+        # 批量插入配置
+        self.batch_size = self.settings.get_int('MYSQL_BATCH_SIZE', 100)
+        self.use_batch = self.settings.get_bool('MYSQL_USE_BATCH', False)
+        self.batch_buffer: List[Dict] = []  # 批量缓冲区
+
         # 注册关闭事件
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
@@ -59,30 +64,45 @@ class AsyncmyMySQLPipeline:
         """处理item的核心方法"""
         kwargs = kwargs or {}
         spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
-        try:
-            await self._ensure_pool()
-            item_dict = dict(item)
-            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
-
-            rowcount = await self._execute_sql(sql=sql)
-            if rowcount > 1:
-                self.logger.info(
-                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
-                )
-            elif rowcount == 1:
-                self.logger.debug(
-                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
-                )
-            else:
-                self.logger.warning(
-                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
-                )
-
+        
+        # 如果启用批量插入，将item添加到缓冲区
+        if self.use_batch:
+            self.batch_buffer.append(dict(item))
+            
+            # 如果缓冲区达到批量大小，执行批量插入
+            if len(self.batch_buffer) >= self.batch_size:
+                await self._flush_batch(spider_name)
+                
             return item
+        else:
+            # 单条插入逻辑
+            try:
+                await self._ensure_pool()
+                item_dict = dict(item)
+                sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
 
-        except Exception as e:
-            self.logger.error(f"处理item时发生错误: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
+                rowcount = await self._execute_sql(sql=sql)
+                if rowcount > 1:
+                    self.logger.info(
+                        f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
+                    )
+                elif rowcount == 1:
+                    self.logger.debug(
+                        f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
+                    )
+                else:
+                    self.logger.warning(
+                        f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
+                    )
+
+                # 统计计数移到这里，与AiomysqlMySQLPipeline保持一致
+                self.crawler.stats.inc_value('mysql/insert_success')
+                return item
+
+            except Exception as e:
+                self.logger.error(f"处理item时发生错误: {e}")
+                self.crawler.stats.inc_value('mysql/insert_failed')
+                raise ItemDiscard(f"处理失败: {e}")
 
     async def _execute_sql(self, sql: str, values: list = None) -> int:
         """执行SQL语句并处理结果"""
@@ -96,15 +116,59 @@ class AsyncmyMySQLPipeline:
                         rowcount = await cursor.execute(sql)
 
                     await conn.commit()
-                    self.crawler.stats.inc_value('mysql/insert_success')
+                    # 移除这里的统计计数
                     return rowcount
                 except Exception as e:
                     await conn.rollback()
-                    self.crawler.stats.inc_value('mysql/insert_failed')
+                    # 移除这里的统计计数
                     raise ItemDiscard(f"MySQL插入失败: {e}")
 
+    async def _flush_batch(self, spider_name: str):
+        """刷新批量缓冲区并执行批量插入"""
+        if not self.batch_buffer:
+            return
+
+        try:
+            await self._ensure_pool()
+            
+            # 使用批量SQL生成函数
+            batch_result = make_batch_sql(table=self.table_name, datas=self.batch_buffer)
+            if batch_result is None:
+                self.logger.warning("批量插入数据为空")
+                self.batch_buffer.clear()
+                return
+
+            sql, values_list = batch_result
+
+            async with self.pool.acquire() as conn:
+                async with conn.cursor() as cursor:
+                    try:
+                        # 执行批量插入
+                        rowcount = await cursor.executemany(sql, values_list)
+                        await conn.commit()
+                        
+                        self.logger.info(
+                            f"爬虫 {spider_name} 批量插入 {rowcount} 条记录到表 {self.table_name}"
+                        )
+                        # 更新统计计数
+                        self.crawler.stats.inc_value('mysql/insert_success', rowcount)
+                        self.batch_buffer.clear()
+                    except Exception as e:
+                        await conn.rollback()
+                        self.crawler.stats.inc_value('mysql/insert_failed', len(self.batch_buffer))
+                        self.logger.error(f"批量插入失败: {e}")
+                        raise ItemDiscard(f"批量插入失败: {e}")
+        except Exception as e:
+            self.logger.error(f"批量插入过程中发生错误: {e}")
+            raise ItemDiscard(f"批量插入处理失败: {e}")
+
     async def spider_closed(self):
         """关闭爬虫时清理资源"""
+        # 在关闭前刷新剩余的批量数据
+        if self.use_batch and self.batch_buffer:
+            spider_name = getattr(self.crawler.spider, 'name', 'unknown')
+            await self._flush_batch(spider_name)
+            
         if self.pool:
             self.pool.close()
             await self.pool.wait_closed()
@@ -127,6 +191,11 @@ class AiomysqlMySQLPipeline:
                 f"{crawler.spider.name}_items"
         )
 
+        # 批量插入配置
+        self.batch_size = self.settings.get_int('MYSQL_BATCH_SIZE', 100)
+        self.use_batch = self.settings.get_bool('MYSQL_USE_BATCH', False)
+        self.batch_buffer: List[Dict] = []  # 批量缓冲区
+
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
     @classmethod
@@ -160,36 +229,89 @@ class AiomysqlMySQLPipeline:
 
     async def process_item(self, item, spider) -> Optional[dict]:
         """处理item方法"""
+        # 如果启用批量插入，将item添加到缓冲区
+        if self.use_batch:
+            self.batch_buffer.append(dict(item))
+            
+            # 如果缓冲区达到批量大小，执行批量插入
+            if len(self.batch_buffer) >= self.batch_size:
+                spider_name = getattr(spider, 'name', 'unknown')
+                await self._flush_batch(spider_name)
+                
+            return item
+        else:
+            # 单条插入逻辑
+            try:
+                await self._init_pool()
+
+                item_dict = dict(item)
+                # 使用make_insert_sql工具函数生成SQL
+                sql = make_insert_sql(table=self.table_name, data=item_dict)
+
+                async with self.pool.acquire() as conn:
+                    async with conn.cursor() as cursor:
+                        try:
+                            await cursor.execute(sql)
+                            await conn.commit()
+                            self.crawler.stats.inc_value('mysql/insert_success')
+                        except aiomysql.Error as e:
+                            await conn.rollback()
+                            self.crawler.stats.inc_value('mysql/insert_failed')
+                            raise ItemDiscard(f"MySQL错误: {e.args[1]}")
+
+                return item
+
+            except Exception as e:
+                self.logger.error(f"Pipeline处理异常: {e}")
+                raise ItemDiscard(f"处理失败: {e}")
+
+    async def _flush_batch(self, spider_name: str):
+        """刷新批量缓冲区并执行批量插入"""
+        if not self.batch_buffer:
+            return
+
         try:
             await self._init_pool()
+            
+            # 使用批量SQL生成函数
+            batch_result = make_batch_sql(table=self.table_name, datas=self.batch_buffer)
+            if batch_result is None:
+                self.logger.warning("批量插入数据为空")
+                self.batch_buffer.clear()
+                return
 
-            item_dict = dict(item)
-            sql = f"""
-            INSERT INTO `{self.table_name}` 
-            ({', '.join([f'`{k}`' for k in item_dict.keys()])})
-            VALUES ({', '.join(['%s'] * len(item_dict))})
-            """
+            sql, values_list = batch_result
 
             async with self.pool.acquire() as conn:
                 async with conn.cursor() as cursor:
                     try:
-                        await cursor.execute(sql, list(item_dict.values()))
+                        # 执行批量插入
+                        rowcount = await cursor.executemany(sql, values_list)
                         await conn.commit()
-                        self.crawler.stats.inc_value('mysql/insert_success')
-                    except aiomysql.Error as e:
+                        
+                        self.logger.info(
+                            f"爬虫 {spider_name} 批量插入 {rowcount} 条记录到表 {self.table_name}"
+                        )
+                        # 更新统计计数
+                        self.crawler.stats.inc_value('mysql/insert_success', rowcount)
+                        self.batch_buffer.clear()
+                    except Exception as e:
                         await conn.rollback()
-                        self.crawler.stats.inc_value('mysql/insert_failed')
-                        raise ItemDiscard(f"MySQL错误: {e.args[1]}")
-
-            return item
-
+                        self.crawler.stats.inc_value('mysql/insert_failed', len(self.batch_buffer))
+                        self.logger.error(f"批量插入失败: {e}")
+                        raise ItemDiscard(f"批量插入失败: {e}")
         except Exception as e:
-            self.logger.error(f"Pipeline处理异常: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
+            self.logger.error(f"批量插入过程中发生错误: {e}")
+            raise ItemDiscard(f"批量插入处理失败: {e}")
 
     async def spider_closed(self):
         """资源清理"""
+        # 在关闭前刷新剩余的批量数据
+        if self.use_batch and self.batch_buffer:
+            spider_name = getattr(self.crawler.spider, 'name', 'unknown')
+            await self._flush_batch(spider_name)
+            
         if self.pool:
             self.pool.close()
             await self.pool.wait_closed()
-            self.logger.info("aiomysql连接池已释放")
+            self.logger.info("aiomysql连接池已释放")

crawlo/pipelines/redis_dedup_pipeline.py

@@ -0,0 +1,163 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于 Redis 的数据项去重管道
+========================
+提供分布式环境下的数据项去重功能，防止保存重复的数据记录。
+
+特点:
+- 分布式支持: 多节点共享去重数据
+- 高性能: 使用 Redis 集合进行快速查找
+- 可配置: 支持自定义 Redis 连接参数
+- 容错设计: 网络异常时不会丢失数据
+"""
+
+import hashlib
+from typing import Dict, Any, Optional
+import redis
+
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import DropItem
+
+
+class RedisDedupPipeline:
+    """基于 Redis 的数据项去重管道"""
+
+    def __init__(
+            self,
+            redis_host: str = 'localhost',
+            redis_port: int = 6379,
+            redis_db: int = 0,
+            redis_password: Optional[str] = None,
+            redis_key: str = 'crawlo:item_fingerprints',
+            log_level: str = "INFO"
+    ):
+        """
+        初始化 Redis 去重管道
+        
+        :param redis_host: Redis 主机地址
+        :param redis_port: Redis 端口
+        :param redis_db: Redis 数据库编号
+        :param redis_password: Redis 密码
+        :param redis_key: 存储指纹的 Redis 键名
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 初始化 Redis 连接
+        try:
+            self.redis_client = redis.Redis(
+                host=redis_host,
+                port=redis_port,
+                db=redis_db,
+                password=redis_password,
+                decode_responses=True,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            # 测试连接
+            self.redis_client.ping()
+            self.logger.info(f"Redis 连接成功: {redis_host}:{redis_port}/{redis_db}")
+        except Exception as e:
+            self.logger.error(f"Redis 连接失败: {e}")
+            raise RuntimeError(f"Redis 连接失败: {e}")
+
+        self.redis_key = redis_key
+        self.dropped_count = 0
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        return cls(
+            redis_host=settings.get('REDIS_HOST', 'localhost'),
+            redis_port=settings.getint('REDIS_PORT', 6379),
+            redis_db=settings.getint('REDIS_DB', 0),
+            redis_password=settings.get('REDIS_PASSWORD') or None,
+            redis_key=settings.get('REDIS_DEDUP_KEY', 'crawlo:item_fingerprints'),
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 使用 Redis 的 SADD 命令检查并添加指纹
+            # 如果指纹已存在，SADD 返回 0；如果指纹是新的，SADD 返回 1
+            is_new = self.redis_client.sadd(self.redis_key, fingerprint)
+            
+            if not is_new:
+                # 如果指纹已存在，丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.debug(f"丢弃重复数据项: {fingerprint[:20]}...")
+                raise DropItem(f"重复的数据项: {fingerprint}")
+            else:
+                # 如果是新数据项，继续处理
+                self.logger.debug(f"处理新数据项: {fingerprint[:20]}...")
+                return item
+                
+        except redis.RedisError as e:
+            self.logger.error(f"Redis 错误: {e}")
+            # 在 Redis 错误时继续处理，避免丢失数据
+            return item
+        except Exception as e:
+            self.logger.error(f"处理数据项时出错: {e}")
+            # 在其他错误时继续处理
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        # 将数据项转换为可序列化的字典
+        try:
+            item_dict = item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            item_dict = dict(item)
+        
+        # 对字典进行排序以确保一致性
+        sorted_items = sorted(item_dict.items())
+        
+        # 生成指纹字符串
+        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
+        
+        # 使用 SHA256 生成固定长度的指纹
+        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        try:
+            # 获取去重统计信息
+            total_items = self.redis_client.scard(self.redis_key)
+            self.logger.info(f"爬虫 {spider.name} 关闭:")
+            self.logger.info(f"  - 丢弃的重复数据项: {self.dropped_count}")
+            self.logger.info(f"  - Redis 中存储的指纹数: {total_items}")
+            
+            # 注意：默认情况下不清理 Redis 中的指纹
+            # 如果需要清理，可以在设置中配置
+            if spider.crawler.settings.getbool('REDIS_DEDUP_CLEANUP', False):
+                deleted = self.redis_client.delete(self.redis_key)
+                self.logger.info(f"  - 清理的指纹数: {deleted}")
+        except Exception as e:
+            self.logger.error(f"关闭爬虫时出错: {e}")

crawlo/queue/queue_manager.py

@@ -7,6 +7,8 @@
 from typing import Optional, Dict, Any, Union
 from enum import Enum
 import asyncio
+import traceback
+import os
 
 from crawlo.utils.log import get_logger
 from crawlo.utils.request_serializer import RequestSerializer
@@ -106,7 +108,9 @@ class QueueManager:
             return True
             
         except Exception as e:
+            # 记录详细的错误信息和堆栈跟踪
             self.logger.error(f"❌ 队列初始化失败: {e}")
+            self.logger.error(f"详细错误信息:\n{traceback.format_exc()}")
             self._health_status = "error"
             return False
     

crawlo/queue/redis_priority_queue.py

@@ -3,6 +3,8 @@ import time
 import asyncio
 from typing import Optional
 import redis.asyncio as aioredis
+import traceback
+import os
 
 from crawlo import Request
 from crawlo.utils.log import get_logger
@@ -19,7 +21,7 @@ class RedisPriorityQueue:
 
     def __init__(
             self,
-            redis_url: str = "redis://localhost:6379/0",
+            redis_url: str = None,
             queue_name: str = "crawlo:requests",
             processing_queue: str = "crawlo:processing",
             failed_queue: str = "crawlo:failed",
@@ -27,6 +29,18 @@ class RedisPriorityQueue:
             timeout: int = 300,  # 任务处理超时时间（秒）
             max_connections: int = 10,  # 连接池大小
     ):
+        # 如果没有提供 redis_url，则从环境变量构造
+        if redis_url is None:
+            redis_host = os.getenv('REDIS_HOST', 'localhost')
+            redis_port = os.getenv('REDIS_PORT', '6379')
+            redis_db = os.getenv('REDIS_DB', '0')
+            redis_password = os.getenv('REDIS_PASSWORD', '')
+            
+            if redis_password:
+                redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+            else:
+                redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
+        
         self.redis_url = redis_url
         self.queue_name = queue_name
         self.processing_queue = processing_queue
@@ -59,6 +73,7 @@ class RedisPriorityQueue:
                     return self._redis
                 except Exception as e:
                     logger.warning(f"⚠️ Redis 连接失败 (尝试 {attempt + 1}/{max_retries}): {e}")
+                    logger.warning(f"详细错误信息:\n{traceback.format_exc()}")
                     if attempt < max_retries - 1:
                         await asyncio.sleep(delay)
                     else:
@@ -70,8 +85,8 @@ class RedisPriorityQueue:
             await self.connect()
         try:
             await self._redis.ping()
-        except Exception:
-            logger.warning("🔄 Redis 连接失效，尝试重连...")
+        except Exception as e:
+            logger.warning(f"🔄 Redis 连接失效，尝试重连...: {e}")
             self._redis = None
             await self.connect()
 
@@ -95,6 +110,7 @@ class RedisPriorityQueue:
             return result[0] > 0
         except Exception as e:
             logger.error(f"❌ 放入队列失败: {e}")
+            logger.error(f"详细错误信息:\n{traceback.format_exc()}")
             return False
 
     async def get(self, timeout: float = 5.0) -> Optional[Request]:
@@ -134,6 +150,7 @@ class RedisPriorityQueue:
 
             except Exception as e:
                 logger.error(f"❌ 获取队列任务失败: {e}")
+                logger.error(f"详细错误信息:\n{traceback.format_exc()}")
                 return None
 
     async def ack(self, request: Request):