crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_dedup_fix.py

@@ -1,221 +1,221 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-去重管道异常处理修复测试
-验证去重管道抛出的异常能被正确处理，防止重复数据传递到后续管道
-"""
-import sys
-import os
-import asyncio
-import unittest
-from unittest.mock import Mock, patch, MagicMock
-from collections import namedtuple
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-# 导入相关模块
-from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
-from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
-from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
-from crawlo.pipelines.pipeline_manager import PipelineManager
-from crawlo.exceptions import ItemDiscard, DropItem
-
-
-class TestDedupFix(unittest.TestCase):
-    """去重管道异常处理修复测试"""
-
-    def setUp(self):
-        """测试初始化"""
-        # 创建模拟的crawler对象
-        self.mock_crawler = Mock()
-        self.mock_crawler.settings = Mock()
-        self.mock_crawler.settings.get = Mock(return_value="INFO")
-        self.mock_crawler.settings.get_int = Mock(return_value=0)
-        self.mock_crawler.settings.get_bool = Mock(return_value=False)
-        self.mock_crawler.subscriber = Mock()
-        self.mock_crawler.subscriber.subscribe = Mock()
-        
-        # 创建简单的测试数据项（使用namedtuple模拟Item）
-        self.TestItem = namedtuple('TestItem', ['title', 'url', 'content'])
-        self.test_item = self.TestItem(
-            title="Test Title",
-            url="http://example.com",
-            content="Test content"
-        )
-
-    def test_redis_dedup_pipeline_exception_type(self):
-        """测试Redis去重管道抛出正确的异常类型"""
-        # 创建Redis去重管道实例
-        with patch('redis.Redis') as mock_redis:
-            mock_redis_instance = Mock()
-            mock_redis_instance.sadd = Mock(return_value=0)  # 模拟已存在的指纹
-            mock_redis.return_value = mock_redis_instance
-            
-            pipeline = RedisDedupPipeline(
-                redis_host='localhost',
-                redis_port=6379,
-                redis_db=0,
-                redis_password=None,
-                redis_key='test:key',
-                log_level='INFO'
-            )
-            
-            # 验证抛出的是ItemDiscard异常而不是DropItem异常
-            with self.assertRaises(ItemDiscard) as context:
-                pipeline.process_item(self.test_item, Mock())
-            
-            # 验证异常消息
-            self.assertIn("Duplicate item:", str(context.exception))
-
-    def test_memory_dedup_pipeline_exception_type(self):
-        """测试内存去重管道抛出正确的异常类型"""
-        # 创建内存去重管道实例
-        pipeline = MemoryDedupPipeline(log_level='INFO')
-        
-        # 添加一个指纹到已见过的集合中
-        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
-        pipeline.seen_items.add(fingerprint)
-        
-        # 验证抛出的是ItemDiscard异常而不是DropItem异常
-        with self.assertRaises(ItemDiscard) as context:
-            pipeline.process_item(self.test_item, Mock())
-        
-        # 验证异常消息
-        self.assertIn("重复的数据项:", str(context.exception))
-
-    def test_bloom_dedup_pipeline_exception_type(self):
-        """测试Bloom去重管道抛出正确的异常类型"""
-        # 创建Bloom去重管道实例
-        pipeline = BloomDedupPipeline(log_level='INFO')
-        
-        # 添加一个指纹到Bloom过滤器中
-        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
-        pipeline.bloom_filter.add(fingerprint)
-        
-        # 验证抛出的是ItemDiscard异常而不是DropItem异常
-        with self.assertRaises(ItemDiscard) as context:
-            pipeline.process_item(self.test_item, Mock())
-        
-        # 验证异常消息
-        self.assertIn("可能重复的数据项:", str(context.exception))
-
-    async def test_pipeline_manager_exception_handling(self):
-        """测试管道管理器能正确处理两种异常类型"""
-        # 创建管道管理器实例
-        pipeline_manager = PipelineManager(self.mock_crawler)
-        
-        # 创建测试数据项
-        test_item = self.TestItem(
-            title="Test Title",
-            url="http://example.com",
-            content="Test content"
-        )
-        
-        # 模拟管道方法列表
-        pipeline_manager.methods = []
-        
-        # 创建模拟的去重管道方法（抛出ItemDiscard异常）
-        mock_dedup_method = Mock()
-        mock_dedup_method.side_effect = ItemDiscard("测试ItemDiscard异常")
-        
-        # 创建模拟的MySQL管道方法
-        mock_mysql_method = Mock()
-        mock_mysql_method.return_value = test_item
-        
-        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
-        
-        # 测试处理数据项
-        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
-             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
-            
-            # 设置common_call的副作用来模拟异常
-            async def mock_common_call_func(method, *args, **kwargs):
-                if method == mock_dedup_method:
-                    raise ItemDiscard("测试ItemDiscard异常")
-                return test_item
-                
-            mock_common_call.side_effect = mock_common_call_func
-            
-            # 调用处理方法
-            await pipeline_manager.process_item(test_item)
-            
-            # 验证ItemDiscard异常被正确处理
-            # 验证create_task被调用了一次（item_discard事件）
-            self.assertEqual(mock_create_task.call_count, 1)
-            
-            # 验证MySQL管道方法没有被调用
-            mock_mysql_method.assert_not_called()
-
-    async def test_pipeline_manager_dropitem_exception_handling(self):
-        """测试管道管理器能正确处理DropItem异常"""
-        # 创建管道管理器实例
-        pipeline_manager = PipelineManager(self.mock_crawler)
-        
-        # 创建测试数据项
-        test_item = self.TestItem(
-            title="Test Title",
-            url="http://example.com",
-            content="Test content"
-        )
-        
-        # 模拟管道方法列表
-        pipeline_manager.methods = []
-        
-        # 创建模拟的去重管道方法（抛出DropItem异常）
-        mock_dedup_method = Mock()
-        mock_dedup_method.side_effect = DropItem("测试DropItem异常")
-        
-        # 创建模拟的MySQL管道方法
-        mock_mysql_method = Mock()
-        mock_mysql_method.return_value = test_item
-        
-        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
-        
-        # 测试处理数据项
-        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
-             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
-            
-            # 设置common_call的副作用来模拟异常
-            async def mock_common_call_func(method, *args, **kwargs):
-                if method == mock_dedup_method:
-                    raise DropItem("测试DropItem异常")
-                return test_item
-                
-            mock_common_call.side_effect = mock_common_call_func
-            
-            # 调用处理方法
-            await pipeline_manager.process_item(test_item)
-            
-            # 验证DropItem异常被正确处理
-            # 验证create_task被调用了一次（item_discard事件）
-            self.assertEqual(mock_create_task.call_count, 1)
-            
-            # 验证MySQL管道方法没有被调用
-            mock_mysql_method.assert_not_called()
-
-
-async def main():
-    """主测试函数"""
-    print("开始去重管道异常处理修复测试...")
-    print("=" * 50)
-    
-    # 创建测试套件
-    test_suite = unittest.TestLoader().loadTestsFromTestCase(TestDedupFix)
-    
-    # 运行测试
-    runner = unittest.TextTestRunner(verbosity=2)
-    result = runner.run(test_suite)
-    
-    print("=" * 50)
-    if result.wasSuccessful():
-        print("所有测试通过！去重管道异常处理修复验证成功")
-        return 0
-    else:
-        print("部分测试失败，请检查实现")
-        return 1
-
-
-if __name__ == "__main__":
-    exit_code = asyncio.run(main())
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+去重管道异常处理修复测试
+验证去重管道抛出的异常能被正确处理，防止重复数据传递到后续管道
+"""
+import sys
+import os
+import asyncio
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+from collections import namedtuple
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 导入相关模块
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+from crawlo.pipelines.pipeline_manager import PipelineManager
+from crawlo.exceptions import ItemDiscard, DropItem
+
+
+class TestDedupFix(unittest.TestCase):
+    """去重管道异常处理修复测试"""
+
+    def setUp(self):
+        """测试初始化"""
+        # 创建模拟的crawler对象
+        self.mock_crawler = Mock()
+        self.mock_crawler.settings = Mock()
+        self.mock_crawler.settings.get = Mock(return_value="INFO")
+        self.mock_crawler.settings.get_int = Mock(return_value=0)
+        self.mock_crawler.settings.get_bool = Mock(return_value=False)
+        self.mock_crawler.subscriber = Mock()
+        self.mock_crawler.subscriber.subscribe = Mock()
+        
+        # 创建简单的测试数据项（使用namedtuple模拟Item）
+        self.TestItem = namedtuple('TestItem', ['title', 'url', 'content'])
+        self.test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+
+    def test_redis_dedup_pipeline_exception_type(self):
+        """测试Redis去重管道抛出正确的异常类型"""
+        # 创建Redis去重管道实例
+        with patch('redis.Redis') as mock_redis:
+            mock_redis_instance = Mock()
+            mock_redis_instance.sadd = Mock(return_value=0)  # 模拟已存在的指纹
+            mock_redis.return_value = mock_redis_instance
+            
+            pipeline = RedisDedupPipeline(
+                redis_host='localhost',
+                redis_port=6379,
+                redis_db=0,
+                redis_password=None,
+                redis_key='test:key',
+                log_level='INFO'
+            )
+            
+            # 验证抛出的是ItemDiscard异常而不是DropItem异常
+            with self.assertRaises(ItemDiscard) as context:
+                pipeline.process_item(self.test_item, Mock())
+            
+            # 验证异常消息
+            self.assertIn("Duplicate item:", str(context.exception))
+
+    def test_memory_dedup_pipeline_exception_type(self):
+        """测试内存去重管道抛出正确的异常类型"""
+        # 创建内存去重管道实例
+        pipeline = MemoryDedupPipeline(log_level='INFO')
+        
+        # 添加一个指纹到已见过的集合中
+        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
+        pipeline.seen_items.add(fingerprint)
+        
+        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        with self.assertRaises(ItemDiscard) as context:
+            pipeline.process_item(self.test_item, Mock())
+        
+        # 验证异常消息
+        self.assertIn("重复的数据项:", str(context.exception))
+
+    def test_bloom_dedup_pipeline_exception_type(self):
+        """测试Bloom去重管道抛出正确的异常类型"""
+        # 创建Bloom去重管道实例
+        pipeline = BloomDedupPipeline(log_level='INFO')
+        
+        # 添加一个指纹到Bloom过滤器中
+        fingerprint = pipeline._generate_item_fingerprint(self.test_item)
+        pipeline.bloom_filter.add(fingerprint)
+        
+        # 验证抛出的是ItemDiscard异常而不是DropItem异常
+        with self.assertRaises(ItemDiscard) as context:
+            pipeline.process_item(self.test_item, Mock())
+        
+        # 验证异常消息
+        self.assertIn("可能重复的数据项:", str(context.exception))
+
+    async def test_pipeline_manager_exception_handling(self):
+        """测试管道管理器能正确处理两种异常类型"""
+        # 创建管道管理器实例
+        pipeline_manager = PipelineManager(self.mock_crawler)
+        
+        # 创建测试数据项
+        test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+        
+        # 模拟管道方法列表
+        pipeline_manager.methods = []
+        
+        # 创建模拟的去重管道方法（抛出ItemDiscard异常）
+        mock_dedup_method = Mock()
+        mock_dedup_method.side_effect = ItemDiscard("测试ItemDiscard异常")
+        
+        # 创建模拟的MySQL管道方法
+        mock_mysql_method = Mock()
+        mock_mysql_method.return_value = test_item
+        
+        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
+        
+        # 测试处理数据项
+        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
+             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
+            
+            # 设置common_call的副作用来模拟异常
+            async def mock_common_call_func(method, *args, **kwargs):
+                if method == mock_dedup_method:
+                    raise ItemDiscard("测试ItemDiscard异常")
+                return test_item
+                
+            mock_common_call.side_effect = mock_common_call_func
+            
+            # 调用处理方法
+            await pipeline_manager.process_item(test_item)
+            
+            # 验证ItemDiscard异常被正确处理
+            # 验证create_task被调用了一次（item_discard事件）
+            self.assertEqual(mock_create_task.call_count, 1)
+            
+            # 验证MySQL管道方法没有被调用
+            mock_mysql_method.assert_not_called()
+
+    async def test_pipeline_manager_dropitem_exception_handling(self):
+        """测试管道管理器能正确处理DropItem异常"""
+        # 创建管道管理器实例
+        pipeline_manager = PipelineManager(self.mock_crawler)
+        
+        # 创建测试数据项
+        test_item = self.TestItem(
+            title="Test Title",
+            url="http://example.com",
+            content="Test content"
+        )
+        
+        # 模拟管道方法列表
+        pipeline_manager.methods = []
+        
+        # 创建模拟的去重管道方法（抛出DropItem异常）
+        mock_dedup_method = Mock()
+        mock_dedup_method.side_effect = DropItem("测试DropItem异常")
+        
+        # 创建模拟的MySQL管道方法
+        mock_mysql_method = Mock()
+        mock_mysql_method.return_value = test_item
+        
+        pipeline_manager.methods = [mock_dedup_method, mock_mysql_method]
+        
+        # 测试处理数据项
+        with patch('crawlo.pipelines.pipeline_manager.common_call') as mock_common_call, \
+             patch('crawlo.pipelines.pipeline_manager.create_task') as mock_create_task:
+            
+            # 设置common_call的副作用来模拟异常
+            async def mock_common_call_func(method, *args, **kwargs):
+                if method == mock_dedup_method:
+                    raise DropItem("测试DropItem异常")
+                return test_item
+                
+            mock_common_call.side_effect = mock_common_call_func
+            
+            # 调用处理方法
+            await pipeline_manager.process_item(test_item)
+            
+            # 验证DropItem异常被正确处理
+            # 验证create_task被调用了一次（item_discard事件）
+            self.assertEqual(mock_create_task.call_count, 1)
+            
+            # 验证MySQL管道方法没有被调用
+            mock_mysql_method.assert_not_called()
+
+
+async def main():
+    """主测试函数"""
+    print("开始去重管道异常处理修复测试...")
+    print("=" * 50)
+    
+    # 创建测试套件
+    test_suite = unittest.TestLoader().loadTestsFromTestCase(TestDedupFix)
+    
+    # 运行测试
+    runner = unittest.TextTestRunner(verbosity=2)
+    result = runner.run(test_suite)
+    
+    print("=" * 50)
+    if result.wasSuccessful():
+        print("所有测试通过！去重管道异常处理修复验证成功")
+        return 0
+    else:
+        print("部分测试失败，请检查实现")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
     sys.exit(exit_code)

tests/test_dedup_pipeline_consistency.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试去重管道配置一致性
+验证在不同模式下，去重管道配置的正确性
+"""
+
+import sys
+import os
+import unittest
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.pipelines.pipeline_manager import remove_dedup_pipelines, get_dedup_pipeline_classes
+
+
+class TestDedupPipelineConsistency(unittest.TestCase):
+    """测试去重管道配置一致性"""
+
+    def test_get_dedup_pipeline_classes(self):
+        """测试获取所有去重管道类名"""
+        dedup_classes = get_dedup_pipeline_classes()
+        
+        # 验证返回的是列表
+        self.assertIsInstance(dedup_classes, list)
+        
+        # 验证包含所有已知的去重管道类
+        expected_classes = [
+            'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+            'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
+            'crawlo.pipelines.bloom_dedup_pipeline.BloomDedupPipeline',
+            'crawlo.pipelines.database_dedup_pipeline.DatabaseDedupPipeline'
+        ]
+        
+        for expected_class in expected_classes:
+            self.assertIn(expected_class, dedup_classes, f"缺失去重管道类: {expected_class}")
+        
+        print("✅ 获取去重管道类名测试通过")
+
+    def test_remove_dedup_pipelines_empty_list(self):
+        """测试从空列表中移除去重管道"""
+        pipelines = []
+        result = remove_dedup_pipelines(pipelines)
+        self.assertEqual(result, [], "空列表处理错误")
+        print("✅ 空列表处理测试通过")
+
+    def test_remove_dedup_pipelines_no_dedup(self):
+        """测试从不包含去重管道的列表中移除去重管道"""
+        pipelines = [
+            'crawlo.pipelines.console_pipeline.ConsolePipeline',
+            'crawlo.pipelines.csv_pipeline.CSVPipeline',
+            'crawlo.pipelines.mysql_pipeline.MySQLPipeline'
+        ]
+        result = remove_dedup_pipelines(pipelines)
+        self.assertEqual(result, pipelines, "不应该修改不包含去重管道的列表")
+        print("✅ 不包含去重管道的列表处理测试通过")
+
+    def test_remove_dedup_pipelines_with_dedup(self):
+        """测试从包含去重管道的列表中移除去重管道"""
+        pipelines = [
+            'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+            'crawlo.pipelines.console_pipeline.ConsolePipeline',
+            'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
+            'crawlo.pipelines.csv_pipeline.CSVPipeline',
+            'crawlo.pipelines.bloom_dedup_pipeline.BloomDedupPipeline'
+        ]
+        
+        expected = [
+            'crawlo.pipelines.console_pipeline.ConsolePipeline',
+            'crawlo.pipelines.csv_pipeline.CSVPipeline'
+        ]
+        
+        result = remove_dedup_pipelines(pipelines)
+        self.assertEqual(result, expected, "去重管道移除错误")
+        print("✅ 包含去重管道的列表处理测试通过")
+
+    def test_remove_dedup_pipelines_all_dedup(self):
+        """测试从只包含去重管道的列表中移除去重管道"""
+        pipelines = [
+            'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+            'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
+            'crawlo.pipelines.bloom_dedup_pipeline.BloomDedupPipeline',
+            'crawlo.pipelines.database_dedup_pipeline.DatabaseDedupPipeline'
+        ]
+        
+        result = remove_dedup_pipelines(pipelines)
+        self.assertEqual(result, [], "应该返回空列表")
+        print("✅ 只包含去重管道的列表处理测试通过")
+
+    def test_remove_dedup_pipelines_mixed_order(self):
+        """测试混合顺序的管道列表"""
+        pipelines = [
+            'crawlo.pipelines.csv_pipeline.CSVPipeline',
+            'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+            'crawlo.pipelines.console_pipeline.ConsolePipeline',
+            'crawlo.pipelines.bloom_dedup_pipeline.BloomDedupPipeline',
+            'crawlo.pipelines.mysql_pipeline.MySQLPipeline',
+            'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'
+        ]
+        
+        expected = [
+            'crawlo.pipelines.csv_pipeline.CSVPipeline',
+            'crawlo.pipelines.console_pipeline.ConsolePipeline',
+            'crawlo.pipelines.mysql_pipeline.MySQLPipeline'
+        ]
+        
+        result = remove_dedup_pipelines(pipelines)
+        self.assertEqual(result, expected, "混合顺序处理错误")
+        print("✅ 混合顺序的管道列表处理测试通过")
+
+
+if __name__ == "__main__":
+    print("开始测试去重管道配置一致性...")
+    
+    try:
+        # 运行测试
+        unittest.main(verbosity=2)
+        
+        print("\n🎉 所有测试通过！去重管道配置一致性已正确实现。")
+        
+    except Exception as e:
+        print(f"\n❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()