crawlo 1.3.7__py3-none-any.whl → 1.3.9__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.3.7'
+__version__ = '1.3.9'

crawlo/pipelines/bloom_dedup_pipeline.py

@@ -39,7 +39,7 @@ except ImportError:
 from crawlo import Item
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
-from crawlo.exceptions import DropItem
+from crawlo.exceptions import DropItem, ItemDiscard
 
 
 class BloomDedupPipeline:
@@ -101,7 +101,7 @@ class BloomDedupPipeline:
                 # 如果可能已存在（Bloom Filter 可能有误判），丢弃这个数据项
                 self.dropped_count += 1
                 self.logger.debug(f"Possibly dropping duplicate item: {fingerprint[:20]}...")
-                raise DropItem(f"可能重复的数据项: {fingerprint}")
+                raise ItemDiscard(f"可能重复的数据项: {fingerprint}")
             else:
                 # 添加指纹到 Bloom Filter
                 self.bloom_filter.add(fingerprint)

crawlo/pipelines/database_dedup_pipeline.py

@@ -15,7 +15,7 @@ import hashlib
 import aiomysql
 
 from crawlo import Item
-from crawlo.exceptions import DropItem
+from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
 
@@ -133,7 +133,7 @@ class DatabaseDedupPipeline:
                 # 如果已存在，丢弃这个数据项
                 self.dropped_count += 1
                 self.logger.debug(f"Dropping duplicate item: {fingerprint[:20]}...")
-                raise DropItem(f"Duplicate item: {fingerprint}")
+                raise ItemDiscard(f"Duplicate item: {fingerprint}")
             else:
                 # 记录新数据项的指纹
                 await self._insert_fingerprint(fingerprint)
@@ -176,7 +176,7 @@ class DatabaseDedupPipeline:
                 except aiomysql.IntegrityError:
                     # 指纹已存在（并发情况下可能发生）
                     await conn.rollback()
-                    raise DropItem(f"重复的数据项: {fingerprint}")
+                    raise ItemDiscard(f"重复的数据项: {fingerprint}")
                 except Exception:
                     await conn.rollback()
                     raise
@@ -197,27 +197,4 @@ class DatabaseDedupPipeline:
             # 兼容没有to_dict方法的Item实现
             item_dict = dict(item)
         
-        # 对字典进行排序以确保一致性
-        sorted_items = sorted(item_dict.items())
-        
-        # 生成指纹字符串
-        fingerprint_string = '|'.join([f"{k}={v}" for k, v in sorted_items if v is not None])
-        
-        # 使用 SHA256 生成固定长度的指纹
-        return hashlib.sha256(fingerprint_string.encode('utf-8')).hexdigest()
-
-    async def close_spider(self, spider: Spider) -> None:
-        """
-        爬虫关闭时的清理工作
-        
-        :param spider: 爬虫实例
-        """
-        try:
-            if self.pool:
-                self.pool.close()
-                await self.pool.wait_closed()
-                
-            self.logger.info(f"Spider {spider.name} closed:")
-            self.logger.info(f"  - Dropped duplicate items: {self.dropped_count}")
-        except Exception as e:
-            self.logger.error(f"Error closing spider: {e}")
+        # 对字典进行排序以确保一致性

crawlo/pipelines/memory_dedup_pipeline.py

@@ -16,7 +16,7 @@ import hashlib
 from typing import Set
 
 from crawlo import Item
-from crawlo.exceptions import DropItem
+from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.spider import Spider
 from crawlo.utils.log import get_logger
 
@@ -64,7 +64,7 @@ class MemoryDedupPipeline:
                 # 如果已存在，丢弃这个数据项
                 self.dropped_count += 1
                 self.logger.debug(f"Dropping duplicate item: {fingerprint[:20]}...")
-                raise DropItem(f"重复的数据项: {fingerprint}")
+                raise ItemDiscard(f"重复的数据项: {fingerprint}")
             else:
                 # 记录新数据项的指纹
                 self.seen_items.add(fingerprint)

crawlo/pipelines/pipeline_manager.py

@@ -8,7 +8,7 @@ from crawlo.utils.log import get_logger
 from crawlo.event import item_successful, item_discard
 from crawlo.utils.class_loader import load_class
 from crawlo.project import common_call
-from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError, DropItem
 
 
 class PipelineManager:
@@ -70,7 +70,7 @@ class PipelineManager:
                 item = await common_call(method, item, self.crawler.spider)
                 if item is None:
                     raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-        except ItemDiscard as exc:
+        except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
             create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
         else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))

crawlo/pipelines/redis_dedup_pipeline.py

@@ -17,7 +17,7 @@ from typing import Optional
 
 from crawlo import Item
 from crawlo.spider import Spider
-from crawlo.exceptions import DropItem
+from crawlo.exceptions import DropItem, ItemDiscard
 from crawlo.utils.log import get_logger
 
 
@@ -105,7 +105,7 @@ class RedisDedupPipeline:
                 # 如果指纹已存在，丢弃这个数据项
                 self.dropped_count += 1
                 # self.logger.debug(f"Dropping duplicate item: {fingerprint[:20]}...")  # 注释掉重复的日志
-                raise DropItem(f"Duplicate item: {fingerprint}")
+                raise ItemDiscard(f"Duplicate item: {fingerprint}")
             else:
                 # 如果是新数据项，继续处理
                 # self.logger.debug(f"Processing new item: {fingerprint[:20]}...")  # 注释掉重复的日志

{crawlo-1.3.7.dist-info → crawlo-1.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.7
+Version: 1.3.9
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.3.7.dist-info → crawlo-1.3.9.dist-info}/RECORD

@@ -1,5 +1,5 @@
 crawlo/__init__.py,sha256=rCeDq1OoX6mmcBxuK60eUpEp1cIg5T8Zgic3FUQAOkA,2318
-crawlo/__version__.py,sha256=ejHyy8zORCf0PfUyyfPDzlV1k5vn5lI98S7TxKzblZc,22
+crawlo/__version__.py,sha256=w1HvwXrREPyUQwcUNaOv25LesFD0cwTBQjuG4ym_vww,22
 crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
 crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
 crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
@@ -78,16 +78,16 @@ crawlo/network/__init__.py,sha256=BLPERYPo22g1BXrW--wUnlolrdFUmOPjgOB8XQQJlck,39
 crawlo/network/request.py,sha256=9kV-gqb_d6aCsSBAwyzxnP9a70cAViwX8qvpyYV7Ym4,13799
 crawlo/network/response.py,sha256=EZiG4LjuIb7PxdGou4H-oSOQhec1ZdBRTkO-5fl8JTo,12701
 crawlo/pipelines/__init__.py,sha256=lrdVDjeHLNkA4_MAwI1auk_I9xfeU1SlBWXiammb6lc,616
-crawlo/pipelines/bloom_dedup_pipeline.py,sha256=omB_gHtoacbco0sn_c6HO6PHCh6xylSecK7UbJIeLq8,5661
+crawlo/pipelines/bloom_dedup_pipeline.py,sha256=AIejBxWpRWg2SSYuMPmPy2VrG_FhFmjXYaAcRhuF7h0,5677
 crawlo/pipelines/console_pipeline.py,sha256=KABkR3J-rqO0Awox7lizxKR2XuHfVhWPiVRgIybwwu4,1248
 crawlo/pipelines/csv_pipeline.py,sha256=6FBT2AoU6iNU-5NfgWRq7-JpF9dK2nBokjxx-y4jIas,12174
-crawlo/pipelines/database_dedup_pipeline.py,sha256=Ao_5jvVPl5QikxXhPeIrcB7_3tinR9bPNRV5Fu5zfDU,7978
+crawlo/pipelines/database_dedup_pipeline.py,sha256=qQ6w0eoGcUB26VO1lgI8GFhVjERG-GZhg5UCTYKyovg,7135
 crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZRHAY,8367
-crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
+crawlo/pipelines/memory_dedup_pipeline.py,sha256=9KuUA1S0uHWSB3qJntPdg9ifPdRXwc8ju4j9tWe8qTo,3853
 crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
 crawlo/pipelines/mysql_pipeline.py,sha256=_oRfIvlEiOsTKkr4v-yPTcL8nG9O9coRmke2ZSkkKII,13871
-crawlo/pipelines/pipeline_manager.py,sha256=rtKZEgDc9oMDYaTrSSQYCc7rVJ-a65TQw4p3dWHF1SM,3116
-crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
+crawlo/pipelines/pipeline_manager.py,sha256=AZPOjm7N1WcjyfIoyZpzVEchmAfZP0uFSZ_WicKL5co,3171
+crawlo/pipelines/redis_dedup_pipeline.py,sha256=2Esl-Yh6nhNzYlUsrpvT0fV8Wx4cNNU9jpwIxqOrgCM,6358
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlo/queue/pqueue.py,sha256=j2ISmyays5t1tuI36xM6EcELwSpq2xIjAScSBWSRZms,1220
 crawlo/queue/queue_manager.py,sha256=JfkjtOD04e_OZZvEEvp3O_W3lfGXhHslZHrCgw90amY,20693
@@ -209,6 +209,7 @@ tests/test_config_validator.py,sha256=5ivB71KstHGNi2BPzcclf9hBukXEgt_B8N4l1HRjBF
 tests/test_controlled_spider_mixin.py,sha256=7t6VGWr6Hxw0xtIFyToLH8_deSagUtsdqSJpibXHMY8,2785
 tests/test_crawlo_proxy_integration.py,sha256=SvdBuZjS6N2vuvFkTnc59U5n3dHV3E4dmFayxtmjCm4,2625
 tests/test_date_tools.py,sha256=CQdAmIS6bpAdwQH9ETDH__06l2gGL7EHUQuh7mdTF-A,3930
+tests/test_dedup_fix.py,sha256=9rFzzsDJKQbFaohzKJAlqo3Mm4wFy8-wAm3fWmw8jb4,8568
 tests/test_default_header_middleware.py,sha256=7kpONSsGMsmWgTX2pCpseme54_-82Baak0xVz6gclJk,5845
 tests/test_distributed.py,sha256=RQHUpDfRNG2x_1Cdr9DLk25IBcgapm_u0xSBMObE0Xc,1725
 tests/test_double_crawlo_fix.py,sha256=E5NxWHnQkwRTIrJGoag8G29fZqVMnsN6eCPuv17gGq0,7652
@@ -285,8 +286,8 @@ tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3
 tests/verify_log_fix.py,sha256=TD7M1R22NxLqQPufvgE-H33u9tUjyz-rSR2ayIXozRU,4225
 tests/scrapy_comparison/ofweek_scrapy.py,sha256=2Hvpi6DRTubUxBy6RyJApQxMQONPLc1zWjKTQO_i5U4,5652
 tests/scrapy_comparison/scrapy_test.py,sha256=5sw7jOHhaTmQ8bsUd1TiolAUTRQYQOe-f49HPfysqbI,5466
-crawlo-1.3.7.dist-info/METADATA,sha256=rHuHJj9-pUt4TJRUgQcNV48DM6LRk0Ag8Ek5pZZeqLk,33235
-crawlo-1.3.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.3.7.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.3.7.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.3.7.dist-info/RECORD,,
+crawlo-1.3.9.dist-info/METADATA,sha256=a6FUc4WoaMyek0LyyW55pQIMjeAaeRgxyDnWJXvA62I,33235
+crawlo-1.3.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.3.9.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.3.9.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.3.9.dist-info/RECORD,,

tests/test_dedup_fix.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+去重管道异常处理修复测试
+验证去重管道抛出的异常能被正确处理，防止重复数据传递到后续管道
+"""
+import sys
+import os
+import asyncio
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+from collections import namedtuple
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 导入相关模块
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+from crawlo.pipelines.pipeline_manager import PipelineManager
+from crawlo.exceptions import ItemDiscard, DropItem
+
+
+class TestDedupFix(unittest.TestCase):
+    """去重管道异常处理修复测试"""
+
+    def setUp(self):
+        """测试初始化"""
+        # 创建模拟的crawler对象
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value="INFO")
+        self.mock_crawler.settings.get_int = Mock(return_value=0)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 创建简单的测试数据项（使用namedtuple模拟Item）
+        self.TestItem = namedtuple('TestItem', ['title', 'url', 'content'])
+        self.test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+
+    def test_redis_dedup_pipeline_exception_type(self):
+        """测试Redis去重管道抛出正确的异常类型"""
+        # 创建Redis去重管道实例
+        with patch('redis.Redis') as mock_redis:
+            mock_redis_instance = Mock()
+            mock_redis_instance.sadd = Mock(return_value=0)  # 模拟已存在的指纹
+            mock_redis.return_value = mock_redis_instance
+            
+            pipeline = RedisDedupPipeline(
+                redis_host='localhost',
+                redis_port=6379,
+                redis_db=0,
+                redis_password=None,
+                redis_key='test:key',
+                log_level='INFO'
+            )
+            
+            # 验证抛出的是ItemDiscard异常而不是DropItem异常
+            with self.assertRaises(ItemDiscard) as context:
+                pipeline.process_item(self.test_item, Mock())
+            
+            # 验证异常消息
+            self.assertIn("Duplicate item:", str(context.exception))
+
+    def test_memory_dedup_pipeline_exception_type(self):
+        """测试内存去重管道抛出正确的异常类型"""
+        # 创建内存去重管道实例
+        pipeline = MemoryDedupPipeline(log_level='INFO')
+        
+        # 添加一个指纹到已见过的集合中
+        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
+        pipeline.seen_items.add(fingerprint)
+        
+        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        with self.assertRaises(ItemDiscard) as context:
+            pipeline.process_item(self.test_item, Mock())
+        
+        # 验证异常消息
+        self.assertIn("重复的数据项:", str(context.exception))
+
+    def test_bloom_dedup_pipeline_exception_type(self):
+        """测试Bloom去重管道抛出正确的异常类型"""
+        # 创建Bloom去重管道实例
+        pipeline = BloomDedupPipeline(log_level='INFO')
+        
+        # 添加一个指纹到Bloom过滤器中
+        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
+        pipeline.bloom_filter.add(fingerprint)
+        
+        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        with self.assertRaises(ItemDiscard) as context:
+            pipeline.process_item(self.test_item, Mock())
+        
+        # 验证异常消息
+        self.assertIn("可能重复的数据项:", str(context.exception))
+
+    async def test_pipeline_manager_exception_handling(self):
+        """测试管道管理器能正确处理两种异常类型"""
+        # 创建管道管理器实例
+        pipeline_manager = PipelineManager(self.mock_crawler)
+        
+        # 创建测试数据项
+        test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+        
+        # 模拟管道方法列表
+        pipeline_manager.methods = []
+        
+        # 创建模拟的去重管道方法（抛出ItemDiscard异常）
+        mock_dedup_method = Mock()
+        mock_dedup_method.side_effect = ItemDiscard("测试ItemDiscard异常")
+        
+        # 创建模拟的MySQL管道方法
+        mock_mysql_method = Mock()
+        mock_mysql_method.return_value = test_item
+        
+        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
+        
+        # 测试处理数据项
+        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
+             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
+            
+            # 设置common_call的副作用来模拟异常
+            async def mock_common_call_func(method, *args, **kwargs):
+                if method == mock_dedup_method:
+                    raise ItemDiscard("测试ItemDiscard异常")
+                return test_item
+                
+            mock_common_call.side_effect = mock_common_call_func
+            
+            # 调用处理方法
+            await pipeline_manager.process_item(test_item)
+            
+            # 验证ItemDiscard异常被正确处理
+            # 验证create_task被调用了一次（item_discard事件）
+            self.assertEqual(mock_create_task.call_count, 1)
+            
+            # 验证MySQL管道方法没有被调用
+            mock_mysql_method.assert_not_called()
+
+    async def test_pipeline_manager_dropitem_exception_handling(self):
+        """测试管道管理器能正确处理DropItem异常"""
+        # 创建管道管理器实例
+        pipeline_manager = PipelineManager(self.mock_crawler)
+        
+        # 创建测试数据项
+        test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+        
+        # 模拟管道方法列表
+        pipeline_manager.methods = []
+        
+        # 创建模拟的去重管道方法（抛出DropItem异常）
+        mock_dedup_method = Mock()
+        mock_dedup_method.side_effect = DropItem("测试DropItem异常")
+        
+        # 创建模拟的MySQL管道方法
+        mock_mysql_method = Mock()
+        mock_mysql_method.return_value = test_item
+        
+        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
+        
+        # 测试处理数据项
+        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
+             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
+            
+            # 设置common_call的副作用来模拟异常
+            async def mock_common_call_func(method, *args, **kwargs):
+                if method == mock_dedup_method:
+                    raise DropItem("测试DropItem异常")
+                return test_item
+                
+            mock_common_call.side_effect = mock_common_call_func
+            
+            # 调用处理方法
+            await pipeline_manager.process_item(test_item)
+            
+            # 验证DropItem异常被正确处理
+            # 验证create_task被调用了一次（item_discard事件）
+            self.assertEqual(mock_create_task.call_count, 1)
+            
+            # 验证MySQL管道方法没有被调用
+            mock_mysql_method.assert_not_called()
+
+
+async def main():
+    """主测试函数"""
+    print("开始去重管道异常处理修复测试...")
+    print("=" * 50)
+    
+    # 创建测试套件
+    test_suite = unittest.TestLoader().loadTestsFromTestCase(TestDedupFix)
+    
+    # 运行测试
+    runner = unittest.TextTestRunner(verbosity=2)
+    result = runner.run(test_suite)
+    
+    print("=" * 50)
+    if result.wasSuccessful():
+        print("所有测试通过！去重管道异常处理修复验证成功")
+        return 0
+    else:
+        print("部分测试失败，请检查实现")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)