crawlo 1.1.2__py3-none-any.whl → 1.1.3__py3-none-any.whl

crawlo/pipelines/memory_dedup_pipeline.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于内存的数据项去重管道
+======================
+提供单节点环境下的数据项去重功能，防止保存重复的数据记录。
+
+特点:
+- 高性能: 使用内存集合进行快速查找
+- 简单易用: 无需外部依赖
+- 轻量级: 适用于小规模数据采集
+- 低延迟: 内存操作无网络开销
+"""
+
+import hashlib
+from typing import Dict, Any, Set
+
+from crawlo import Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import DropItem
+
+
+class MemoryDedupPipeline:
+    """基于内存的数据项去重管道"""
+
+    def __init__(self, log_level: str = "INFO"):
+        """
+        初始化内存去重管道
+        
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 使用集合存储已见过的数据项指纹
+        self.seen_items: Set[str] = set()
+        self.dropped_count = 0
+        
+        self.logger.info("内存去重管道初始化完成")
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        return cls(
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 检查指纹是否已存在
+            if fingerprint in self.seen_items:
+                # 如果已存在，丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.debug(f"丢弃重复数据项: {fingerprint[:20]}...")
+                raise DropItem(f"重复的数据项: {fingerprint}")
+            else:
+                # 记录新数据项的指纹
+                self.seen_items.add(fingerprint)
+                self.logger.debug(f"处理新数据项: {fingerprint[:20]}...")
+                return item
+                
+        except Exception as e:
+            self.logger.error(f"处理数据项时出错: {e}")
+            # 在错误时继续处理，避免丢失数据
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        # 将数据项转换为可序列化的字典
+        try:
+            item_dict = item.to_dict()
+        except AttributeError:
+            # 兼容没有to_dict方法的Item实现
+            item_dict = dict(item)
+        
+        # 对字典进行排序以确保一致性
+        sorted_items = sorted(item_dict.items())
+        
+        # 生成指纹字符串
+        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
+        
+        # 使用 SHA256 生成固定长度的指纹
+        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        self.logger.info(f"爬虫 {spider.name} 关闭:")
+        self.logger.info(f"  - 丢弃的重复数据项: {self.dropped_count}")
+        self.logger.info(f"  - 内存中存储的指纹数: {len(self.seen_items)}")
+        
+        # 清理内存
+        self.seen_items.clear()
+        self.dropped_count = 0

crawlo/pipelines/mongo_pipeline.py

@@ -1,117 +1,117 @@
-# -*- coding: utf-8 -*-
-from typing import Optional
-from motor.motor_asyncio import AsyncIOMotorClient
-from pymongo.errors import PyMongoError
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import ItemDiscard
-
-
-class MongoPipeline:
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-
-        # 初始化连接参数
-        self.client = None
-        self.db = None
-        self.collection = None
-
-        # 配置默认值
-        self.mongo_uri = self.settings.get('MONGO_URI', 'mongodb://localhost:27017')
-        self.db_name = self.settings.get('MONGO_DATABASE', 'scrapy_db')
-        self.collection_name = self.settings.get('MONGO_COLLECTION', crawler.spider.name)
-
-        # 注册关闭事件
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    async def _ensure_connection(self):
-        """确保连接已建立"""
-        if self.client is None:
-            self.client = AsyncIOMotorClient(self.mongo_uri)
-            self.db = self.client[self.db_name]
-            self.collection = self.db[self.collection_name]
-            self.logger.info(f"MongoDB连接建立 (集合: {self.collection_name})")
-
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item的核心方法"""
-        try:
-            await self._ensure_connection()
-
-            item_dict = dict(item)
-            result = await self.collection.insert_one(item_dict)
-
-            # 统计计数
-            self.crawler.stats.inc_value('mongodb/inserted')
-            self.logger.debug(f"插入文档ID: {result.inserted_id}")
-
-            return item
-
-        except Exception as e:
-            self.crawler.stats.inc_value('mongodb/failed')
-            self.logger.error(f"MongoDB插入失败: {e}")
-            raise ItemDiscard(f"MongoDB操作失败: {e}")
-
-    async def spider_closed(self):
-        """关闭爬虫时清理资源"""
-        if self.client:
-            self.client.close()
-            self.logger.info("MongoDB连接已关闭")
-
-
-class MongoPoolPipeline:
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-
-        # 连接池配置
-        self.client = AsyncIOMotorClient(
-            self.settings.get('MONGO_URI', 'mongodb://localhost:27017'),
-            maxPoolSize=self.settings.getint('MONGO_MAX_POOL_SIZE', 100),
-            minPoolSize=self.settings.getint('MONGO_MIN_POOL_SIZE', 10),
-            connectTimeoutMS=5000,
-            socketTimeoutMS=30000
-        )
-
-        self.db = self.client[self.settings.get('MONGO_DATABASE', 'scrapy_db')]
-        self.collection = self.db[self.settings.get('MONGO_COLLECTION', crawler.spider.name)]
-
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-        self.logger.info(f"MongoDB连接池已初始化 (集合: {self.collection.name})")
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls(crawler)
-
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item方法（带重试机制）"""
-        try:
-            item_dict = dict(item)
-
-            # 带重试的插入操作
-            for attempt in range(3):
-                try:
-                    result = await self.collection.insert_one(item_dict)
-                    self.crawler.stats.inc_value('mongodb/insert_success')
-                    self.logger.debug(f"插入成功 [attempt {attempt + 1}]: {result.inserted_id}")
-                    return item
-                except PyMongoError as e:
-                    if attempt == 2:  # 最后一次尝试仍失败
-                        raise
-                    self.logger.warning(f"插入重试中 [attempt {attempt + 1}]: {e}")
-
-        except Exception as e:
-            self.crawler.stats.inc_value('mongodb/insert_failed')
-            self.logger.error(f"MongoDB操作最终失败: {e}")
-            raise ItemDiscard(f"MongoDB操作失败: {e}")
-
-    async def spider_closed(self):
-        """资源清理"""
-        if hasattr(self, 'client'):
-            self.client.close()
+# -*- coding: utf-8 -*-
+from typing import Optional
+from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo.errors import PyMongoError
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import ItemDiscard
+
+
+class MongoPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 初始化连接参数
+        self.client = None
+        self.db = None
+        self.collection = None
+
+        # 配置默认值
+        self.mongo_uri = self.settings.get('MONGO_URI', 'mongodb://localhost:27017')
+        self.db_name = self.settings.get('MONGO_DATABASE', 'scrapy_db')
+        self.collection_name = self.settings.get('MONGO_COLLECTION', crawler.spider.name)
+
+        # 注册关闭事件
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    async def _ensure_connection(self):
+        """确保连接已建立"""
+        if self.client is None:
+            self.client = AsyncIOMotorClient(self.mongo_uri)
+            self.db = self.client[self.db_name]
+            self.collection = self.db[self.collection_name]
+            self.logger.info(f"MongoDB连接建立 (集合: {self.collection_name})")
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item的核心方法"""
+        try:
+            await self._ensure_connection()
+
+            item_dict = dict(item)
+            result = await self.collection.insert_one(item_dict)
+
+            # 统计计数
+            self.crawler.stats.inc_value('mongodb/inserted')
+            self.logger.debug(f"插入文档ID: {result.inserted_id}")
+
+            return item
+
+        except Exception as e:
+            self.crawler.stats.inc_value('mongodb/failed')
+            self.logger.error(f"MongoDB插入失败: {e}")
+            raise ItemDiscard(f"MongoDB操作失败: {e}")
+
+    async def spider_closed(self):
+        """关闭爬虫时清理资源"""
+        if self.client:
+            self.client.close()
+            self.logger.info("MongoDB连接已关闭")
+
+
+class MongoPoolPipeline:
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+
+        # 连接池配置
+        self.client = AsyncIOMotorClient(
+            self.settings.get('MONGO_URI', 'mongodb://localhost:27017'),
+            maxPoolSize=self.settings.getint('MONGO_MAX_POOL_SIZE', 100),
+            minPoolSize=self.settings.getint('MONGO_MIN_POOL_SIZE', 10),
+            connectTimeoutMS=5000,
+            socketTimeoutMS=30000
+        )
+
+        self.db = self.client[self.settings.get('MONGO_DATABASE', 'scrapy_db')]
+        self.collection = self.db[self.settings.get('MONGO_COLLECTION', crawler.spider.name)]
+
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+        self.logger.info(f"MongoDB连接池已初始化 (集合: {self.collection.name})")
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls(crawler)
+
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法（带重试机制）"""
+        try:
+            item_dict = dict(item)
+
+            # 带重试的插入操作
+            for attempt in range(3):
+                try:
+                    result = await self.collection.insert_one(item_dict)
+                    self.crawler.stats.inc_value('mongodb/insert_success')
+                    self.logger.debug(f"插入成功 [attempt {attempt + 1}]: {result.inserted_id}")
+                    return item
+                except PyMongoError as e:
+                    if attempt == 2:  # 最后一次尝试仍失败
+                        raise
+                    self.logger.warning(f"插入重试中 [attempt {attempt + 1}]: {e}")
+
+        except Exception as e:
+            self.crawler.stats.inc_value('mongodb/insert_failed')
+            self.logger.error(f"MongoDB操作最终失败: {e}")
+            raise ItemDiscard(f"MongoDB操作失败: {e}")
+
+    async def spider_closed(self):
+        """资源清理"""
+        if hasattr(self, 'client'):
+            self.client.close()
             self.logger.info("MongoDB连接池已释放")