crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_final_validation.py

@@ -1,154 +1,154 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
-"""
-import asyncio
-import pickle
-import sys
-sys.path.insert(0, "..")
-
-from crawlo.network.request import Request
-from crawlo.spider import Spider
-from crawlo.core.scheduler import Scheduler
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-from crawlo.utils.log import get_logger
-from unittest.mock import Mock
-
-
-class TestSpider(Spider):
-    """测试爬虫"""
-    name = "validation_spider"
-    
-    def __init__(self):
-        super().__init__()
-        # 故意添加多个 logger 来测试清理
-        self.custom_logger = get_logger("custom")
-        self.debug_logger = get_logger("debug")
-        self.nested_data = {
-            'logger': get_logger("nested"),
-            'sub': {
-                'logger_ref': get_logger("sub_logger")
-            }
-        }
-    
-    def parse(self, response):
-        # 验证主 logger 还在
-        self.logger.info(f"主 logger 工作正常: {response.url}")
-        return {"url": response.url, "status": "success"}
-
-
-def test_scheduler_cleaning():
-    """测试调度器的 logger 清理"""
-    print("测试调度器 logger 清理...")
-    
-    spider = TestSpider()
-    request = Request(
-        url="https://scheduler-test.com",
-        callback=spider.parse,
-        meta={"logger": get_logger("meta_logger")}
-    )
-    
-    # Mock crawler 和 scheduler
-    class MockCrawler:
-        def __init__(self):
-            self.spider = spider
-    
-    class MockScheduler(Scheduler):
-        def __init__(self):
-            self.crawler = MockCrawler()
-            self.logger = get_logger("MockScheduler")
-    
-    scheduler = MockScheduler()
-    
-    # 清理前检查
-    print(f"   清理前 - spider.logger: {spider.logger is not None}")
-    print(f"   清理前 - spider.custom_logger: {spider.custom_logger is not None}")
-    print(f"   清理前 - request.callback: {request.callback is not None}")
-    
-    # 执行清理
-    cleaned_request = scheduler._deep_clean_loggers(request)
-    
-    # 清理后检查
-    print(f"   清理后 - spider.logger: {spider.logger is not None}")
-    print(f"   清理后 - spider.custom_logger: {spider.custom_logger is None}")
-    print(f"   清理后 - request.callback: {cleaned_request.callback is None}")
-    
-    # 序列化测试
-    try:
-        serialized = pickle.dumps(cleaned_request)
-        print(f"   调度器清理后序列化成功，大小: {len(serialized)} bytes")
-        return True
-    except Exception as e:
-        print(f"   调度器清理后序列化失败: {e}")
-        return False
-
-
-async def test_redis_queue_cleaning():
-    """测试 Redis 队列的 logger 清理"""
-    print("\\n测试 Redis 队列 logger 清理...")
-    
-    spider = TestSpider()
-    request = Request(
-        url="https://redis-test.com",
-        callback=spider.parse,
-        meta={"logger": get_logger("meta_logger")}
-    )
-    
-    try:
-        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
-        await queue.connect()
-        
-        # 入队测试
-        success = await queue.put(request, priority=0)
-        print(f"   Redis 队列入队成功: {success}")
-        
-        if success:
-            # 出队测试
-            retrieved = await queue.get(timeout=2.0)
-            if retrieved:
-                print(f"   Redis 队列出队成功: {retrieved.url}")
-                print(f"   callback 信息保存: {'_callback_info' in retrieved.meta}")
-                await queue.close()
-                return True
-            else:
-                print("   出队失败")
-                await queue.close()
-                return False
-        else:
-            await queue.close()
-            return False
-            
-    except Exception as e:
-        print(f"   Redis 队列测试失败: {e}")
-        return False
-
-
-async def main():
-    """主测试函数"""
-    print("开始最终验证测试...")
-    print("=" * 60)
-    
-    # 测试 1: 调度器清理
-    scheduler_ok = test_scheduler_cleaning()
-    
-    # 测试 2: Redis 队列清理
-    redis_ok = await test_redis_queue_cleaning()
-    
-    print("\\n" + "=" * 60)
-    print("测试结果汇总:")
-    print(f"   调度器 logger 清理: {'通过' if scheduler_ok else '失败'}")
-    print(f"   Redis 队列清理: {'通过' if redis_ok else '失败'}")
-    
-    if scheduler_ok and redis_ok:
-        print("\\n所有测试通过！")
-        print("分布式队列的 logger 序列化问题已完全修复！")
-        print("Crawlo 现在可以正常使用 Redis 分布式队列了！")
-        return True
-    else:
-        print("\\n部分测试失败，需要进一步修复")
-        return False
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
+"""
+import asyncio
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.utils.log import get_logger
+from unittest.mock import Mock
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = "validation_spider"
+    
+    def __init__(self):
+        super().__init__()
+        # 故意添加多个 logger 来测试清理
+        self.custom_logger = get_logger("custom")
+        self.debug_logger = get_logger("debug")
+        self.nested_data = {
+            'logger': get_logger("nested"),
+            'sub': {
+                'logger_ref': get_logger("sub_logger")
+            }
+        }
+    
+    def parse(self, response):
+        # 验证主 logger 还在
+        self.logger.info(f"主 logger 工作正常: {response.url}")
+        return {"url": response.url, "status": "success"}
+
+
+def test_scheduler_cleaning():
+    """测试调度器的 logger 清理"""
+    print("测试调度器 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://scheduler-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    # Mock crawler 和 scheduler
+    class MockCrawler:
+        def __init__(self):
+            self.spider = spider
+    
+    class MockScheduler(Scheduler):
+        def __init__(self):
+            self.crawler = MockCrawler()
+            self.logger = get_logger("MockScheduler")
+    
+    scheduler = MockScheduler()
+    
+    # 清理前检查
+    print(f"   清理前 - spider.logger: {spider.logger is not None}")
+    print(f"   清理前 - spider.custom_logger: {spider.custom_logger is not None}")
+    print(f"   清理前 - request.callback: {request.callback is not None}")
+    
+    # 执行清理
+    cleaned_request = scheduler._deep_clean_loggers(request)
+    
+    # 清理后检查
+    print(f"   清理后 - spider.logger: {spider.logger is not None}")
+    print(f"   清理后 - spider.custom_logger: {spider.custom_logger is None}")
+    print(f"   清理后 - request.callback: {cleaned_request.callback is None}")
+    
+    # 序列化测试
+    try:
+        serialized = pickle.dumps(cleaned_request)
+        print(f"   调度器清理后序列化成功，大小: {len(serialized)} bytes")
+        return True
+    except Exception as e:
+        print(f"   调度器清理后序列化失败: {e}")
+        return False
+
+
+async def test_redis_queue_cleaning():
+    """测试 Redis 队列的 logger 清理"""
+    print("\\n测试 Redis 队列 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://redis-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    try:
+        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
+        await queue.connect()
+        
+        # 入队测试
+        success = await queue.put(request, priority=0)
+        print(f"   Redis 队列入队成功: {success}")
+        
+        if success:
+            # 出队测试
+            retrieved = await queue.get(timeout=2.0)
+            if retrieved:
+                print(f"   Redis 队列出队成功: {retrieved.url}")
+                print(f"   callback 信息保存: {'_callback_info' in retrieved.meta}")
+                await queue.close()
+                return True
+            else:
+                print("   出队失败")
+                await queue.close()
+                return False
+        else:
+            await queue.close()
+            return False
+            
+    except Exception as e:
+        print(f"   Redis 队列测试失败: {e}")
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("开始最终验证测试...")
+    print("=" * 60)
+    
+    # 测试 1: 调度器清理
+    scheduler_ok = test_scheduler_cleaning()
+    
+    # 测试 2: Redis 队列清理
+    redis_ok = await test_redis_queue_cleaning()
+    
+    print("\\n" + "=" * 60)
+    print("测试结果汇总:")
+    print(f"   调度器 logger 清理: {'通过' if scheduler_ok else '失败'}")
+    print(f"   Redis 队列清理: {'通过' if redis_ok else '失败'}")
+    
+    if scheduler_ok and redis_ok:
+        print("\\n所有测试通过！")
+        print("分布式队列的 logger 序列化问题已完全修复！")
+        print("Crawlo 现在可以正常使用 Redis 分布式队列了！")
+        return True
+    else:
+        print("\\n部分测试失败，需要进一步修复")
+        return False
+
+
+if __name__ == "__main__":
     asyncio.run(main())

tests/test_fingerprint_consistency.py

@@ -1,136 +1,136 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-指纹一致性测试
-==============
-验证框架中各组件对相同数据生成一致的指纹
-"""
-
-import unittest
-from unittest.mock import Mock
-
-from crawlo import Item
-from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
-from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
-from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
-from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
-from crawlo.tools.distributed_coordinator import DeduplicationTool
-from crawlo.utils.fingerprint import FingerprintGenerator
-
-
-class TestItem(Item):
-    """测试用数据项类"""
-    
-    def __init__(self, **kwargs):
-        super().__init__()
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-    
-    def to_dict(self):
-        """转换为字典"""
-        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
-
-
-class FingerprintConsistencyTest(unittest.TestCase):
-    """指纹一致性测试"""
-    
-    def setUp(self):
-        """测试初始化"""
-        # 创建测试数据项
-        self.test_item = TestItem(
-            title="Test Title",
-            url="https://example.com",
-            content="Test content",
-            price=99.99
-        )
-        
-        # 创建各去重管道实例
-        self.memory_pipeline = MemoryDedupPipeline()
-        self.redis_pipeline = RedisDedupPipeline(
-            redis_host='localhost',
-            redis_port=6379,
-            redis_db=0,
-            redis_key='test:fingerprints'
-        )
-        self.bloom_pipeline = BloomDedupPipeline()
-        self.database_pipeline = DatabaseDedupPipeline()
-        
-        # 创建去重工具实例
-        self.dedup_tool = DeduplicationTool()
-    
-    def test_item_fingerprint_consistency(self):
-        """测试数据项指纹一致性"""
-        # 使用各管道生成指纹
-        memory_fingerprint = self.memory_pipeline._generate_item_fingerprint(self.test_item)
-        redis_fingerprint = self.redis_pipeline._generate_item_fingerprint(self.test_item)
-        bloom_fingerprint = self.bloom_pipeline._generate_item_fingerprint(self.test_item)
-        database_fingerprint = self.database_pipeline._generate_item_fingerprint(self.test_item)
-        
-        # 使用指纹生成器直接生成指纹
-        direct_fingerprint = FingerprintGenerator.item_fingerprint(self.test_item)
-        
-        # 验证所有指纹一致
-        self.assertEqual(memory_fingerprint, redis_fingerprint)
-        self.assertEqual(memory_fingerprint, bloom_fingerprint)
-        self.assertEqual(memory_fingerprint, database_fingerprint)
-        self.assertEqual(memory_fingerprint, direct_fingerprint)
-        
-        print(f"Memory Pipeline Fingerprint: {memory_fingerprint}")
-        print(f"Redis Pipeline Fingerprint: {redis_fingerprint}")
-        print(f"Bloom Pipeline Fingerprint: {bloom_fingerprint}")
-        print(f"Database Pipeline Fingerprint: {database_fingerprint}")
-        print(f"Direct Fingerprint: {direct_fingerprint}")
-    
-    def test_data_fingerprint_consistency(self):
-        """测试通用数据指纹一致性"""
-        # 测试字典数据
-        test_data = {
-            "name": "test",
-            "value": 123,
-            "nested": {
-                "inner": "value"
-            }
-        }
-        
-        # 使用去重工具生成指纹
-        tool_fingerprint = self.dedup_tool.generate_fingerprint(test_data)
-        
-        # 使用指纹生成器生成指纹
-        generator_fingerprint = FingerprintGenerator.data_fingerprint(test_data)
-        
-        # 验证指纹一致
-        self.assertEqual(tool_fingerprint, generator_fingerprint)
-        
-        print(f"DeduplicationTool Fingerprint: {tool_fingerprint}")
-        print(f"FingerprintGenerator Fingerprint: {generator_fingerprint}")
-    
-    def test_fingerprint_stability(self):
-        """测试指纹稳定性"""
-        # 创建相同的测试数据项多次
-        item1 = TestItem(
-            title="Test Title",
-            url="https://example.com",
-            content="Test content",
-            price=99.99
-        )
-        
-        item2 = TestItem(
-            title="Test Title",
-            url="https://example.com",
-            content="Test content",
-            price=99.99
-        )
-        
-        # 生成指纹
-        fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
-        fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
-        
-        # 验证相同数据生成相同指纹
-        self.assertEqual(fingerprint1, fingerprint2)
-        
-        print(f"First fingerprint: {fingerprint1}")
-        print(f"Second fingerprint: {fingerprint2}")
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+指纹一致性测试
+==============
+验证框架中各组件对相同数据生成一致的指纹
+"""
+
+import unittest
+from unittest.mock import Mock
+
+from crawlo import Item
+from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
+from crawlo.tools.distributed_coordinator import DeduplicationTool
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+class TestItem(Item):
+    """测试用数据项类"""
+    
+    def __init__(self, **kwargs):
+        super().__init__()
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+    
+    def to_dict(self):
+        """转换为字典"""
+        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
+
+
+class FingerprintConsistencyTest(unittest.TestCase):
+    """指纹一致性测试"""
+    
+    def setUp(self):
+        """测试初始化"""
+        # 创建测试数据项
+        self.test_item = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        # 创建各去重管道实例
+        self.memory_pipeline = MemoryDedupPipeline()
+        self.redis_pipeline = RedisDedupPipeline(
+            redis_host='localhost',
+            redis_port=6379,
+            redis_db=0,
+            redis_key='test:fingerprints'
+        )
+        self.bloom_pipeline = BloomDedupPipeline()
+        self.database_pipeline = DatabaseDedupPipeline()
+        
+        # 创建去重工具实例
+        self.dedup_tool = DeduplicationTool()
+    
+    def test_item_fingerprint_consistency(self):
+        """测试数据项指纹一致性"""
+        # 使用各管道生成指纹
+        memory_fingerprint = self.memory_pipeline._generate_item_fingerprint(self.test_item)
+        redis_fingerprint = self.redis_pipeline._generate_item_fingerprint(self.test_item)
+        bloom_fingerprint = self.bloom_pipeline._generate_item_fingerprint(self.test_item)
+        database_fingerprint = self.database_pipeline._generate_item_fingerprint(self.test_item)
+        
+        # 使用指纹生成器直接生成指纹
+        direct_fingerprint = FingerprintGenerator.item_fingerprint(self.test_item)
+        
+        # 验证所有指纹一致
+        self.assertEqual(memory_fingerprint, redis_fingerprint)
+        self.assertEqual(memory_fingerprint, bloom_fingerprint)
+        self.assertEqual(memory_fingerprint, database_fingerprint)
+        self.assertEqual(memory_fingerprint, direct_fingerprint)
+        
+        print(f"Memory Pipeline Fingerprint: {memory_fingerprint}")
+        print(f"Redis Pipeline Fingerprint: {redis_fingerprint}")
+        print(f"Bloom Pipeline Fingerprint: {bloom_fingerprint}")
+        print(f"Database Pipeline Fingerprint: {database_fingerprint}")
+        print(f"Direct Fingerprint: {direct_fingerprint}")
+    
+    def test_data_fingerprint_consistency(self):
+        """测试通用数据指纹一致性"""
+        # 测试字典数据
+        test_data = {
+            "name": "test",
+            "value": 123,
+            "nested": {
+                "inner": "value"
+            }
+        }
+        
+        # 使用去重工具生成指纹
+        tool_fingerprint = self.dedup_tool.generate_fingerprint(test_data)
+        
+        # 使用指纹生成器生成指纹
+        generator_fingerprint = FingerprintGenerator.data_fingerprint(test_data)
+        
+        # 验证指纹一致
+        self.assertEqual(tool_fingerprint, generator_fingerprint)
+        
+        print(f"DeduplicationTool Fingerprint: {tool_fingerprint}")
+        print(f"FingerprintGenerator Fingerprint: {generator_fingerprint}")
+    
+    def test_fingerprint_stability(self):
+        """测试指纹稳定性"""
+        # 创建相同的测试数据项多次
+        item1 = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        item2 = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        # 生成指纹
+        fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
+        fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
+        
+        # 验证相同数据生成相同指纹
+        self.assertEqual(fingerprint1, fingerprint2)
+        
+        print(f"First fingerprint: {fingerprint1}")
+        print(f"Second fingerprint: {fingerprint2}")
+
+
+if __name__ == '__main__':
     unittest.main()