crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/pipelines/pipeline_manager.py

@@ -1,84 +1,100 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-from asyncio import create_task
-
-from crawlo.utils.log import get_logger
-from crawlo.event import item_successful, item_discard
-from crawlo.utils.class_loader import load_class
-from crawlo.project import common_call
-from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError, DropItem
-
-
-class PipelineManager:
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.pipelines: List = []
-        self.methods: List = []
-
-        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
-        pipelines = self.crawler.settings.get_list('PIPELINES')
-        dedup_pipeline = self.crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
-
-        # 添加调试信息
-        self.logger.debug(f"PIPELINES from settings: {pipelines}")
-        self.logger.debug(f"DEFAULT_DEDUP_PIPELINE from settings: {dedup_pipeline}")
-
-        # 确保DEFAULT_DEDUP_PIPELINE被添加到管道列表开头
-        if dedup_pipeline:
-            # 移除所有去重管道实例（如果存在）
-            pipelines = [item for item in pipelines if item != dedup_pipeline]
-            # 在开头插入去重管道
-            self.logger.debug(f"{dedup_pipeline} insert successful")
-            pipelines.insert(0, dedup_pipeline)
-
-        self._add_pipelines(pipelines)
-        self._add_methods()
-
-    @classmethod
-    def from_crawler(cls, *args, **kwargs):
-        o = cls(*args, **kwargs)
-        return o
-
-    def _add_pipelines(self, pipelines):
-        for pipeline in pipelines:
-            try:
-                pipeline_cls = load_class(pipeline)
-                if not hasattr(pipeline_cls, 'from_crawler'):
-                    raise PipelineInitError(
-                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `from_crawler` method"
-                    )
-                self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
-            except Exception as e:
-                self.logger.error(f"Failed to load pipeline {pipeline}: {e}")
-                # 可以选择继续加载其他管道或抛出异常
-                raise
-        if pipelines:
-            # 恢复INFO级别日志，保留关键的启用信息
-            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
-
-    def _add_methods(self):
-        for pipeline in self.pipelines:
-            if hasattr(pipeline, 'process_item'):
-                self.methods.append(pipeline.process_item)
-
-    async def process_item(self, item):
-        try:
-            for i, method in enumerate(self.methods):
-                self.logger.debug(f"Processing item with pipeline method {i}: {method.__qualname__}")
-                try:
-                    item = await common_call(method, item, self.crawler.spider)
-                    if item is None:
-                        raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-                except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
-                    self.logger.debug(f"Item discarded by pipeline: {exc}")
-                    create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
-                    # 重新抛出异常，确保上层调用者也能捕获到，并停止执行后续管道
-                    raise
-        except (ItemDiscard, DropItem):
-            # 异常已经被处理和通知，这里只需要重新抛出
-            raise
-        else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+from asyncio import create_task
+
+from crawlo.utils.log import get_logger
+from crawlo.event import item_successful, item_discard
+from crawlo.utils.class_loader import load_class
+from crawlo.project import common_call
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError, DropItem
+
+
+def get_dedup_pipeline_classes():
+    """获取所有已知的去重管道类名"""
+    return [
+        'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+        'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
+        'crawlo.pipelines.bloom_dedup_pipeline.BloomDedupPipeline',
+        'crawlo.pipelines.database_dedup_pipeline.DatabaseDedupPipeline'
+    ]
+
+
+def remove_dedup_pipelines(pipelines: List[str]) -> List[str]:
+    """从管道列表中移除所有去重管道"""
+    dedup_classes = get_dedup_pipeline_classes()
+    return [pipeline for pipeline in pipelines if pipeline not in dedup_classes]
+
+
+class PipelineManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.pipelines: List = []
+        self.methods: List = []
+
+        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
+        pipelines = self.crawler.settings.get_list('PIPELINES')
+        dedup_pipeline = self.crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
+
+        # 添加调试信息
+        self.logger.debug(f"PIPELINES from settings: {pipelines}")
+        self.logger.debug(f"DEFAULT_DEDUP_PIPELINE from settings: {dedup_pipeline}")
+
+        # 确保DEFAULT_DEDUP_PIPELINE被添加到管道列表开头
+        if dedup_pipeline:
+            # 移除所有去重管道实例（如果存在）
+            pipelines = remove_dedup_pipelines(pipelines)
+            # 在开头插入去重管道
+            self.logger.debug(f"{dedup_pipeline} insert successful")
+            pipelines.insert(0, dedup_pipeline)
+
+        self._add_pipelines(pipelines)
+        self._add_methods()
+
+    @classmethod
+    def from_crawler(cls, *args, **kwargs):
+        o = cls(*args, **kwargs)
+        return o
+
+    def _add_pipelines(self, pipelines):
+        for pipeline in pipelines:
+            try:
+                pipeline_cls = load_class(pipeline)
+                if not hasattr(pipeline_cls, 'from_crawler'):
+                    raise PipelineInitError(
+                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `from_crawler` method"
+                    )
+                self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load pipeline {pipeline}: {e}")
+                # 可以选择继续加载其他管道或抛出异常
+                raise
+        if pipelines:
+            # 恢复INFO级别日志，保留关键的启用信息
+            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
+
+    def _add_methods(self):
+        for pipeline in self.pipelines:
+            if hasattr(pipeline, 'process_item'):
+                self.methods.append(pipeline.process_item)
+
+    async def process_item(self, item):
+        try:
+            for i, method in enumerate(self.methods):
+                self.logger.debug(f"Processing item with pipeline method {i}: {method.__qualname__}")
+                try:
+                    item = await common_call(method, item, self.crawler.spider)
+                    if item is None:
+                        raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
+                except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
+                    self.logger.debug(f"Item discarded by pipeline: {exc}")
+                    create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+                    # 重新抛出异常，确保上层调用者也能捕获到，并停止执行后续管道
+                    raise
+        except (ItemDiscard, DropItem):
+            # 异常已经被处理和通知，这里只需要重新抛出
+            raise
+        else:
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))

crawlo/pipelines/redis_dedup_pipeline.py

@@ -1,157 +1,157 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-基于 Redis 的数据项去重管道
-========================
-提供分布式环境下的数据项去重功能，防止保存重复的数据记录。
-
-特点:
-- 分布式支持: 多节点共享去重数据
-- 高性能: 使用 Redis 集合进行快速查找
-- 可配置: 支持自定义 Redis 连接参数
-- 容错设计: 网络异常时不会丢失数据
-"""
-import redis
-import hashlib
-from typing import Optional
-
-from crawlo import Item
-from crawlo.spider import Spider
-from crawlo.exceptions import DropItem, ItemDiscard
-from crawlo.utils.fingerprint import FingerprintGenerator
-from crawlo.utils.log import get_logger
-
-
-class RedisDedupPipeline:
-    """基于 Redis 的数据项去重管道"""
-
-    def __init__(
-            self,
-            redis_host: str = 'localhost',
-            redis_port: int = 6379,
-            redis_db: int = 0,
-            redis_password: Optional[str] = None,
-            redis_key: str = 'crawlo:item_fingerprints',
-            log_level: str = "INFO"
-    ):
-        """
-        初始化 Redis 去重管道
-        
-        :param redis_host: Redis 主机地址
-        :param redis_port: Redis 端口
-        :param redis_db: Redis 数据库编号
-        :param redis_password: Redis 密码
-        :param redis_key: 存储指纹的 Redis 键名
-        :param log_level: 日志级别
-        """
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        
-        # 初始化 Redis 连接
-        try:
-            self.redis_client = redis.Redis(
-                host=redis_host,
-                port=redis_port,
-                db=redis_db,
-                password=redis_password,
-                decode_responses=True,
-                socket_connect_timeout=5,
-                socket_timeout=5
-            )
-            # 测试连接
-            self.redis_client.ping()
-            # Change INFO level log to DEBUG level to avoid redundant output
-            # self.logger.debug(f"Redis connection successful: {redis_host}:{redis_port}/{redis_db}")  # 注释掉重复的日志
-        except Exception as e:
-            self.logger.error(f"Redis connection failed: {e}")
-            raise RuntimeError(f"Redis 连接失败: {e}")
-
-        self.redis_key = redis_key
-        self.dropped_count = 0
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        """从爬虫配置创建管道实例"""
-        settings = crawler.settings
-        
-        # 使用统一的Redis key命名规范: crawlo:{project_name}:item:fingerprint
-        project_name = settings.get('PROJECT_NAME', 'default')
-        redis_key = f"crawlo:{project_name}:item:fingerprint"
-        
-        return cls(
-            redis_host=settings.get('REDIS_HOST', 'localhost'),
-            redis_port=settings.get_int('REDIS_PORT', 6379),
-            redis_db=settings.get_int('REDIS_DB', 0),
-            redis_password=settings.get('REDIS_PASSWORD') or None,
-            redis_key=redis_key,
-            log_level=settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    def process_item(self, item: Item, spider: Spider) -> Item:
-        """
-        处理数据项，进行去重检查
-        
-        :param item: 要处理的数据项
-        :param spider: 爬虫实例
-        :return: 处理后的数据项或抛出 DropItem 异常
-        """
-        try:
-            # 生成数据项指纹
-            fingerprint = self._generate_item_fingerprint(item)
-            
-            # 使用 Redis 的 SADD 命令检查并添加指纹
-            # 如果指纹已存在，SADD 返回 0；如果指纹是新的，SADD 返回 1
-            is_new = self.redis_client.sadd(self.redis_key, fingerprint)
-            
-            if not is_new:
-                # 如果指纹已存在，丢弃这个数据项
-                self.dropped_count += 1
-                self.logger.info(f"Dropping duplicate item: {fingerprint}")
-                raise ItemDiscard(f"Duplicate item: {fingerprint}")
-            else:
-                # 如果是新数据项，继续处理
-                self.logger.debug(f"Processing new item: {fingerprint}")
-                return item
-                
-        except redis.RedisError as e:
-            self.logger.error(f"Redis error: {e}")
-            # 在 Redis 错误时继续处理，避免丢失数据
-            return item
-        except ItemDiscard:
-            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
-            raise
-        except Exception as e:
-            self.logger.error(f"Error processing item: {e}")
-            # 在其他错误时继续处理
-            return item
-
-    def _generate_item_fingerprint(self, item: Item) -> str:
-        """
-        生成数据项指纹
-        
-        基于数据项的所有字段生成唯一指纹，用于去重判断。
-        
-        :param item: 数据项
-        :return: 指纹字符串
-        """
-        return FingerprintGenerator.item_fingerprint(item)
-
-    def close_spider(self, spider: Spider) -> None:
-        """
-        爬虫关闭时的清理工作
-        
-        :param spider: 爬虫实例
-        """
-        try:
-            # 获取去重统计信息
-            total_items = self.redis_client.scard(self.redis_key)
-            self.logger.info(f"Spider {spider.name} closed:")
-            self.logger.info(f"  - Dropped duplicate items: {self.dropped_count}")
-            self.logger.info(f"  - Fingerprints stored in Redis: {total_items}")
-            
-            # 注意：默认情况下不清理 Redis 中的指纹
-            # 如果需要清理，可以在设置中配置
-            if spider.crawler.settings.getbool('REDIS_DEDUP_CLEANUP', False):
-                deleted = self.redis_client.delete(self.redis_key)
-                self.logger.info(f"  - Cleaned fingerprints: {deleted}")
-        except Exception as e:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于 Redis 的数据项去重管道
+========================
+提供分布式环境下的数据项去重功能，防止保存重复的数据记录。
+
+特点:
+- 分布式支持: 多节点共享去重数据
+- 高性能: 使用 Redis 集合进行快速查找
+- 可配置: 支持自定义 Redis 连接参数
+- 容错设计: 网络异常时不会丢失数据
+"""
+import redis
+import hashlib
+from typing import Optional
+
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.exceptions import DropItem, ItemDiscard
+from crawlo.utils.fingerprint import FingerprintGenerator
+from crawlo.utils.log import get_logger
+
+
+class RedisDedupPipeline:
+    """基于 Redis 的数据项去重管道"""
+
+    def __init__(
+            self,
+            redis_host: str = 'localhost',
+            redis_port: int = 6379,
+            redis_db: int = 0,
+            redis_password: Optional[str] = None,
+            redis_key: str = 'crawlo:item_fingerprints',
+            log_level: str = "INFO"
+    ):
+        """
+        初始化 Redis 去重管道
+        
+        :param redis_host: Redis 主机地址
+        :param redis_port: Redis 端口
+        :param redis_db: Redis 数据库编号
+        :param redis_password: Redis 密码
+        :param redis_key: 存储指纹的 Redis 键名
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 初始化 Redis 连接
+        try:
+            self.redis_client = redis.Redis(
+                host=redis_host,
+                port=redis_port,
+                db=redis_db,
+                password=redis_password,
+                decode_responses=True,
+                socket_connect_timeout=5,
+                socket_timeout=5
+            )
+            # 测试连接
+            self.redis_client.ping()
+            # Change INFO level log to DEBUG level to avoid redundant output
+            # self.logger.debug(f"Redis connection successful: {redis_host}:{redis_port}/{redis_db}")  # 注释掉重复的日志
+        except Exception as e:
+            self.logger.error(f"Redis connection failed: {e}")
+            raise RuntimeError(f"Redis 连接失败: {e}")
+
+        self.redis_key = redis_key
+        self.dropped_count = 0
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        # 使用统一的Redis key命名规范: crawlo:{project_name}:item:fingerprint
+        project_name = settings.get('PROJECT_NAME', 'default')
+        redis_key = f"crawlo:{project_name}:item:fingerprint"
+        
+        return cls(
+            redis_host=settings.get('REDIS_HOST', 'localhost'),
+            redis_port=settings.get_int('REDIS_PORT', 6379),
+            redis_db=settings.get_int('REDIS_DB', 0),
+            redis_password=settings.get('REDIS_PASSWORD') or None,
+            redis_key=redis_key,
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 使用 Redis 的 SADD 命令检查并添加指纹
+            # 如果指纹已存在，SADD 返回 0；如果指纹是新的，SADD 返回 1
+            is_new = self.redis_client.sadd(self.redis_key, fingerprint)
+            
+            if not is_new:
+                # 如果指纹已存在，丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.info(f"Dropping duplicate item: {fingerprint}")
+                raise ItemDiscard(f"Duplicate item: {fingerprint}")
+            else:
+                # 如果是新数据项，继续处理
+                self.logger.debug(f"Processing new item: {fingerprint}")
+                return item
+                
+        except redis.RedisError as e:
+            self.logger.error(f"Redis error: {e}")
+            # 在 Redis 错误时继续处理，避免丢失数据
+            return item
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
+        except Exception as e:
+            self.logger.error(f"Error processing item: {e}")
+            # 在其他错误时继续处理
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        return FingerprintGenerator.item_fingerprint(item)
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        try:
+            # 获取去重统计信息
+            total_items = self.redis_client.scard(self.redis_key)
+            self.logger.info(f"Spider {spider.name} closed:")
+            self.logger.info(f"  - Dropped duplicate items: {self.dropped_count}")
+            self.logger.info(f"  - Fingerprints stored in Redis: {total_items}")
+            
+            # 注意：默认情况下不清理 Redis 中的指纹
+            # 如果需要清理，可以在设置中配置
+            if spider.crawler.settings.getbool('REDIS_DEDUP_CLEANUP', False):
+                deleted = self.redis_client.delete(self.redis_key)
+                self.logger.info(f"  - Cleaned fingerprints: {deleted}")
+        except Exception as e:
             self.logger.error(f"Error closing spider: {e}")