crawlo 1.4.2__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/pipelines/json_pipeline.py

@@ -1,219 +1,219 @@
-# -*- coding: utf-8 -*-
-import json
-import asyncio
-from pathlib import Path
-from typing import Optional
-from datetime import datetime
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import ItemDiscard
-
-
-class JsonPipeline:
-    """JSON文件输出管道"""
-    
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-        
-        # 配置文件路径
-        self.file_path = self._get_file_path()
-        self.file_handle = None
-        self.lock = asyncio.Lock()  # 异步锁保证线程安全
-        
-        # 注册关闭事件
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-        
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-    
-    def _get_file_path(self) -> Path:
-        """获取输出文件路径"""
-        # 优先级：设置 > 爬虫属性 > 默认路径
-        file_path = (
-            self.settings.get('JSON_FILE') or
-            getattr(self.crawler.spider, 'json_file', None) or
-            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
-        )
-        
-        path = Path(file_path)
-        path.parent.mkdir(parents=True, exist_ok=True)
-        return path
-    
-    async def _ensure_file_open(self):
-        """确保文件已打开"""
-        if self.file_handle is None:
-            self.file_handle = open(self.file_path, 'w', encoding='utf-8')
-            self.logger.info(f"JSON文件已创建: {self.file_path}")
-    
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item的核心方法"""
-        try:
-            async with self.lock:
-                await self._ensure_file_open()
-                
-                # 转换为字典并序列化
-                item_dict = dict(item)
-                json_line = json.dumps(item_dict, ensure_ascii=False, indent=None)
-                
-                # 写入文件（每行一个JSON对象）
-                self.file_handle.write(json_line + '\n')
-                self.file_handle.flush()  # 立即刷新到磁盘
-                
-                # 统计
-                self.crawler.stats.inc_value('json_pipeline/items_written')
-                self.logger.debug(f"写入JSON项: {len(item_dict)} 字段")
-                
-            return item
-            
-        except Exception as e:
-            self.crawler.stats.inc_value('json_pipeline/items_failed')
-            self.logger.error(f"JSON写入失败: {e}")
-            raise ItemDiscard(f"JSON Pipeline处理失败: {e}")
-    
-    async def spider_closed(self):
-        """关闭爬虫时清理资源"""
-        if self.file_handle:
-            self.file_handle.close()
-            self.logger.info(f"JSON文件已关闭: {self.file_path}")
-
-
-class JsonLinesPipeline:
-    """JSON Lines格式输出管道（每行一个JSON对象）"""
-    
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-        
-        self.file_path = self._get_file_path()
-        self.file_handle = None
-        self.items_count = 0
-        self.lock = asyncio.Lock()
-        
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-        
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-    
-    def _get_file_path(self) -> Path:
-        """获取输出文件路径"""
-        file_path = (
-            self.settings.get('JSONLINES_FILE') or
-            getattr(self.crawler.spider, 'jsonlines_file', None) or
-            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
-        )
-        
-        path = Path(file_path)
-        path.parent.mkdir(parents=True, exist_ok=True)
-        return path
-    
-    async def _ensure_file_open(self):
-        """确保文件已打开"""
-        if self.file_handle is None:
-            self.file_handle = open(self.file_path, 'w', encoding='utf-8')
-            self.logger.info(f"JSONL文件已创建: {self.file_path}")
-    
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item方法"""
-        try:
-            async with self.lock:
-                await self._ensure_file_open()
-                
-                item_dict = dict(item)
-                
-                # 添加元数据
-                if self.settings.get_bool('JSON_ADD_METADATA', False):
-                    item_dict['_crawl_time'] = datetime.now().isoformat()
-                    item_dict['_spider_name'] = spider.name
-                
-                # 写入JSONL格式
-                json_line = json.dumps(item_dict, ensure_ascii=False, separators=(',', ':'))
-                self.file_handle.write(json_line + '\n')
-                self.file_handle.flush()
-                
-                self.items_count += 1
-                
-                # 定期日志输出
-                if self.items_count % 100 == 0:
-                    self.logger.info(f"已写入 {self.items_count} 个JSON对象")
-                
-                self.crawler.stats.inc_value('jsonlines_pipeline/items_written')
-                
-            return item
-            
-        except Exception as e:
-            self.crawler.stats.inc_value('jsonlines_pipeline/items_failed')
-            self.logger.error(f"JSONL写入失败: {e}")
-            raise ItemDiscard(f"JSON Lines Pipeline处理失败: {e}")
-    
-    async def spider_closed(self):
-        """资源清理"""
-        if self.file_handle:
-            self.file_handle.close()
-            self.logger.info(f"JSONL文件已关闭，共写入 {self.items_count} 个项目: {self.file_path}")
-
-
-class JsonArrayPipeline:
-    """JSON数组格式输出管道（所有item组成一个JSON数组）"""
-    
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-        
-        self.file_path = self._get_file_path()
-        self.items = []  # 内存中暂存所有items
-        self.lock = asyncio.Lock()
-        
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-        
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-    
-    def _get_file_path(self) -> Path:
-        """获取输出文件路径"""
-        file_path = (
-            self.settings.get('JSON_ARRAY_FILE') or
-            getattr(self.crawler.spider, 'json_array_file', None) or
-            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}_array.json"
-        )
-        
-        path = Path(file_path)
-        path.parent.mkdir(parents=True, exist_ok=True)
-        return path
-    
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item方法"""
-        try:
-            async with self.lock:
-                item_dict = dict(item)
-                self.items.append(item_dict)
-                
-                self.crawler.stats.inc_value('json_array_pipeline/items_collected')
-                self.logger.debug(f"收集item，当前总数: {len(self.items)}")
-                
-            return item
-            
-        except Exception as e:
-            self.crawler.stats.inc_value('json_array_pipeline/items_failed')
-            self.logger.error(f"JSON Array收集失败: {e}")
-            raise ItemDiscard(f"JSON Array Pipeline处理失败: {e}")
-    
-    async def spider_closed(self):
-        """关闭时写入所有items到JSON数组文件"""
-        try:
-            if self.items:
-                with open(self.file_path, 'w', encoding='utf-8') as f:
-                    json.dump(self.items, f, ensure_ascii=False, indent=2)
-                
-                self.logger.info(f"JSON数组文件已保存，包含 {len(self.items)} 个项目: {self.file_path}")
-                self.crawler.stats.set_value('json_array_pipeline/total_items', len(self.items))
-            else:
-                self.logger.warning("没有items需要保存")
-                
-        except Exception as e:
+# -*- coding: utf-8 -*-
+import json
+import asyncio
+from pathlib import Path
+from typing import Optional
+from datetime import datetime
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import ItemDiscard
+
+
+class JsonPipeline:
+    """JSON文件输出管道"""
+    
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+        
+        # 配置文件路径
+        self.file_path = self._get_file_path()
+        self.file_handle = None
+        self.lock = asyncio.Lock()  # 异步锁保证线程安全
+        
+        # 注册关闭事件
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+        
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+    
+    def _get_file_path(self) -> Path:
+        """获取输出文件路径"""
+        # 优先级：设置 > 爬虫属性 > 默认路径
+        file_path = (
+            self.settings.get('JSON_FILE') or
+            getattr(self.crawler.spider, 'json_file', None) or
+            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
+        )
+        
+        path = Path(file_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        return path
+    
+    async def _ensure_file_open(self):
+        """确保文件已打开"""
+        if self.file_handle is None:
+            self.file_handle = open(self.file_path, 'w', encoding='utf-8')
+            self.logger.info(f"JSON文件已创建: {self.file_path}")
+    
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item的核心方法"""
+        try:
+            async with self.lock:
+                await self._ensure_file_open()
+                
+                # 转换为字典并序列化
+                item_dict = dict(item)
+                json_line = json.dumps(item_dict, ensure_ascii=False, indent=None)
+                
+                # 写入文件（每行一个JSON对象）
+                self.file_handle.write(json_line + '\n')
+                self.file_handle.flush()  # 立即刷新到磁盘
+                
+                # 统计
+                self.crawler.stats.inc_value('json_pipeline/items_written')
+                self.logger.debug(f"写入JSON项: {len(item_dict)} 字段")
+                
+            return item
+            
+        except Exception as e:
+            self.crawler.stats.inc_value('json_pipeline/items_failed')
+            self.logger.error(f"JSON写入失败: {e}")
+            raise ItemDiscard(f"JSON Pipeline处理失败: {e}")
+    
+    async def spider_closed(self):
+        """关闭爬虫时清理资源"""
+        if self.file_handle:
+            self.file_handle.close()
+            self.logger.info(f"JSON文件已关闭: {self.file_path}")
+
+
+class JsonLinesPipeline:
+    """JSON Lines格式输出管道（每行一个JSON对象）"""
+    
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+        
+        self.file_path = self._get_file_path()
+        self.file_handle = None
+        self.items_count = 0
+        self.lock = asyncio.Lock()
+        
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+        
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+    
+    def _get_file_path(self) -> Path:
+        """获取输出文件路径"""
+        file_path = (
+            self.settings.get('JSONLINES_FILE') or
+            getattr(self.crawler.spider, 'jsonlines_file', None) or
+            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.jsonl"
+        )
+        
+        path = Path(file_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        return path
+    
+    async def _ensure_file_open(self):
+        """确保文件已打开"""
+        if self.file_handle is None:
+            self.file_handle = open(self.file_path, 'w', encoding='utf-8')
+            self.logger.info(f"JSONL文件已创建: {self.file_path}")
+    
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法"""
+        try:
+            async with self.lock:
+                await self._ensure_file_open()
+                
+                item_dict = dict(item)
+                
+                # 添加元数据
+                if self.settings.get_bool('JSON_ADD_METADATA', False):
+                    item_dict['_crawl_time'] = datetime.now().isoformat()
+                    item_dict['_spider_name'] = spider.name
+                
+                # 写入JSONL格式
+                json_line = json.dumps(item_dict, ensure_ascii=False, separators=(',', ':'))
+                self.file_handle.write(json_line + '\n')
+                self.file_handle.flush()
+                
+                self.items_count += 1
+                
+                # 定期日志输出
+                if self.items_count % 100 == 0:
+                    self.logger.info(f"已写入 {self.items_count} 个JSON对象")
+                
+                self.crawler.stats.inc_value('jsonlines_pipeline/items_written')
+                
+            return item
+            
+        except Exception as e:
+            self.crawler.stats.inc_value('jsonlines_pipeline/items_failed')
+            self.logger.error(f"JSONL写入失败: {e}")
+            raise ItemDiscard(f"JSON Lines Pipeline处理失败: {e}")
+    
+    async def spider_closed(self):
+        """资源清理"""
+        if self.file_handle:
+            self.file_handle.close()
+            self.logger.info(f"JSONL文件已关闭，共写入 {self.items_count} 个项目: {self.file_path}")
+
+
+class JsonArrayPipeline:
+    """JSON数组格式输出管道（所有item组成一个JSON数组）"""
+    
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+        
+        self.file_path = self._get_file_path()
+        self.items = []  # 内存中暂存所有items
+        self.lock = asyncio.Lock()
+        
+        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
+        
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+    
+    def _get_file_path(self) -> Path:
+        """获取输出文件路径"""
+        file_path = (
+            self.settings.get('JSON_ARRAY_FILE') or
+            getattr(self.crawler.spider, 'json_array_file', None) or
+            f"output/{self.crawler.spider.name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}_array.json"
+        )
+        
+        path = Path(file_path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        return path
+    
+    async def process_item(self, item, spider) -> Optional[dict]:
+        """处理item方法"""
+        try:
+            async with self.lock:
+                item_dict = dict(item)
+                self.items.append(item_dict)
+                
+                self.crawler.stats.inc_value('json_array_pipeline/items_collected')
+                self.logger.debug(f"收集item，当前总数: {len(self.items)}")
+                
+            return item
+            
+        except Exception as e:
+            self.crawler.stats.inc_value('json_array_pipeline/items_failed')
+            self.logger.error(f"JSON Array收集失败: {e}")
+            raise ItemDiscard(f"JSON Array Pipeline处理失败: {e}")
+    
+    async def spider_closed(self):
+        """关闭时写入所有items到JSON数组文件"""
+        try:
+            if self.items:
+                with open(self.file_path, 'w', encoding='utf-8') as f:
+                    json.dump(self.items, f, ensure_ascii=False, indent=2)
+                
+                self.logger.info(f"JSON数组文件已保存，包含 {len(self.items)} 个项目: {self.file_path}")
+                self.crawler.stats.set_value('json_array_pipeline/total_items', len(self.items))
+            else:
+                self.logger.warning("没有items需要保存")
+                
+        except Exception as e:
             self.logger.error(f"保存JSON数组文件失败: {e}")

crawlo/pipelines/memory_dedup_pipeline.py

@@ -1,106 +1,106 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-基于内存的数据项去重管道
-======================
-提供单节点环境下的数据项去重功能，防止保存重复的数据记录。
-
-特点:
-- 高性能: 使用内存集合进行快速查找
-- 简单易用: 无需外部依赖
-- 轻量级: 适用于小规模数据采集
-- 低延迟: 内存操作无网络开销
-"""
-
-import hashlib
-from typing import Set
-
-from crawlo import Item
-from crawlo.exceptions import DropItem, ItemDiscard
-from crawlo.spider import Spider
-from crawlo.utils.fingerprint import FingerprintGenerator
-from crawlo.utils.log import get_logger
-
-
-class MemoryDedupPipeline:
-    """基于内存的数据项去重管道"""
-
-    def __init__(self, log_level: str = "INFO"):
-        """
-        初始化内存去重管道
-        
-        :param log_level: 日志级别
-        """
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        
-        # 使用集合存储已见过的数据项指纹
-        self.seen_items: Set[str] = set()
-        self.dropped_count = 0
-        
-        self.logger.info("Memory deduplication pipeline initialized")
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        """从爬虫配置创建管道实例"""
-        settings = crawler.settings
-        
-        return cls(
-            log_level=settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    def process_item(self, item: Item, spider: Spider) -> Item:
-        """
-        处理数据项，进行去重检查
-        
-        :param item: 要处理的数据项
-        :param spider: 爬虫实例
-        :return: 处理后的数据项或抛出 DropItem 异常
-        """
-        try:
-            # 生成数据项指纹
-            fingerprint = self._generate_item_fingerprint(item)
-            
-            # 检查指纹是否已存在
-            if fingerprint in self.seen_items:
-                # 如果已存在，丢弃这个数据项
-                self.dropped_count += 1
-                self.logger.debug(f"Dropping duplicate item: {fingerprint[:20]}...")
-                raise ItemDiscard(f"重复的数据项: {fingerprint}")
-            else:
-                # 记录新数据项的指纹
-                self.seen_items.add(fingerprint)
-                self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
-                return item
-                
-        except ItemDiscard:
-            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
-            raise
-        except Exception as e:
-            self.logger.error(f"Error processing item: {e}")
-            # 在错误时继续处理，避免丢失数据
-            return item
-
-    def _generate_item_fingerprint(self, item: Item) -> str:
-        """
-        生成数据项指纹
-        
-        基于数据项的所有字段生成唯一指纹，用于去重判断。
-        
-        :param item: 数据项
-        :return: 指纹字符串
-        """
-        return FingerprintGenerator.item_fingerprint(item)
-
-    def close_spider(self, spider: Spider) -> None:
-        """
-        爬虫关闭时的清理工作
-        
-        :param spider: 爬虫实例
-        """
-        self.logger.info(f"Spider {spider.name} closed:")
-        self.logger.info(f"  - Dropped duplicate items: {self.dropped_count}")
-        self.logger.info(f"  - Fingerprints stored in memory: {len(self.seen_items)}")
-        
-        # 清理内存
-        self.seen_items.clear()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+基于内存的数据项去重管道
+======================
+提供单节点环境下的数据项去重功能，防止保存重复的数据记录。
+
+特点:
+- 高性能: 使用内存集合进行快速查找
+- 简单易用: 无需外部依赖
+- 轻量级: 适用于小规模数据采集
+- 低延迟: 内存操作无网络开销
+"""
+
+import hashlib
+from typing import Set
+
+from crawlo import Item
+from crawlo.exceptions import DropItem, ItemDiscard
+from crawlo.spider import Spider
+from crawlo.utils.fingerprint import FingerprintGenerator
+from crawlo.utils.log import get_logger
+
+
+class MemoryDedupPipeline:
+    """基于内存的数据项去重管道"""
+
+    def __init__(self, log_level: str = "INFO"):
+        """
+        初始化内存去重管道
+        
+        :param log_level: 日志级别
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        
+        # 使用集合存储已见过的数据项指纹
+        self.seen_items: Set[str] = set()
+        self.dropped_count = 0
+        
+        self.logger.info("Memory deduplication pipeline initialized")
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        """从爬虫配置创建管道实例"""
+        settings = crawler.settings
+        
+        return cls(
+            log_level=settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    def process_item(self, item: Item, spider: Spider) -> Item:
+        """
+        处理数据项，进行去重检查
+        
+        :param item: 要处理的数据项
+        :param spider: 爬虫实例
+        :return: 处理后的数据项或抛出 DropItem 异常
+        """
+        try:
+            # 生成数据项指纹
+            fingerprint = self._generate_item_fingerprint(item)
+            
+            # 检查指纹是否已存在
+            if fingerprint in self.seen_items:
+                # 如果已存在，丢弃这个数据项
+                self.dropped_count += 1
+                self.logger.debug(f"Dropping duplicate item: {fingerprint[:20]}...")
+                raise ItemDiscard(f"重复的数据项: {fingerprint}")
+            else:
+                # 记录新数据项的指纹
+                self.seen_items.add(fingerprint)
+                self.logger.debug(f"Processing new item: {fingerprint[:20]}...")
+                return item
+                
+        except ItemDiscard:
+            # 重新抛出ItemDiscard异常，确保管道管理器能正确处理
+            raise
+        except Exception as e:
+            self.logger.error(f"Error processing item: {e}")
+            # 在错误时继续处理，避免丢失数据
+            return item
+
+    def _generate_item_fingerprint(self, item: Item) -> str:
+        """
+        生成数据项指纹
+        
+        基于数据项的所有字段生成唯一指纹，用于去重判断。
+        
+        :param item: 数据项
+        :return: 指纹字符串
+        """
+        return FingerprintGenerator.item_fingerprint(item)
+
+    def close_spider(self, spider: Spider) -> None:
+        """
+        爬虫关闭时的清理工作
+        
+        :param spider: 爬虫实例
+        """
+        self.logger.info(f"Spider {spider.name} closed:")
+        self.logger.info(f"  - Dropped duplicate items: {self.dropped_count}")
+        self.logger.info(f"  - Fingerprints stored in memory: {len(self.seen_items)}")
+        
+        # 清理内存
+        self.seen_items.clear()
         self.dropped_count = 0