crawlo 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

tests/test_redis_queue.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis 分布式队列测试脚本
+用于诊断和修复分布式队列问题
+"""
+import asyncio
+import sys
+import traceback
+import time
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.network.request import Request
+
+
+async def test_redis_connection():
+    """测试 Redis 连接"""
+    print("🔍 1. 测试 Redis 连接...")
+    
+    # 测试不同的 Redis URL 格式
+    test_urls = [
+        "redis://localhost:6379/0",
+        "redis://:oscar&0503@127.0.0.1:6379/0",  # 带密码
+        "redis://127.0.0.1:6379/0",               # 无密码
+    ]
+    
+    for redis_url in test_urls:
+        try:
+            print(f"   尝试连接: {redis_url}")
+            queue = RedisPriorityQueue(redis_url=redis_url)
+            await queue.connect()
+            print(f"   ✅ 连接成功: {redis_url}")
+            await queue.close()
+            return redis_url
+        except Exception as e:
+            print(f"   ❌ 连接失败: {redis_url} - {e}")
+    
+    raise ConnectionError("所有 Redis URL 都连接失败")
+
+
+async def test_queue_operations(redis_url):
+    """测试队列基本操作"""
+    print("🔍 2. 测试队列基本操作...")
+    
+    queue = RedisPriorityQueue(
+        redis_url=redis_url,
+        queue_name="test:crawlo:requests",
+        max_retries=2
+    )
+    
+    try:
+        await queue.connect()
+        
+        # 测试 put 操作
+        test_request = Request(url="https://example.com", priority=5)
+        print(f"   📤 插入请求: {test_request.url}")
+        
+        success = await queue.put(test_request, priority=5)
+        if success:
+            print("   ✅ 插入成功")
+        else:
+            print("   ❌ 插入失败")
+            return False
+            
+        # 测试队列大小
+        size = await queue.qsize()
+        print(f"   📊 队列大小: {size}")
+        
+        # 测试 get 操作
+        print("   📥 获取请求...")
+        retrieved_request = await queue.get(timeout=2.0)
+        
+        if retrieved_request:
+            print(f"   ✅ 获取成功: {retrieved_request.url}")
+            # 测试 ack
+            await queue.ack(retrieved_request)
+            print("   ✅ ACK 成功")
+        else:
+            print("   ❌ 获取失败（超时）")
+            return False
+            
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 队列操作失败: {e}")
+        traceback.print_exc()
+        return False
+    finally:
+        await queue.close()
+
+
+async def test_serialization():
+    """测试序列化问题"""
+    print("🔍 3. 测试 Request 序列化...")
+    
+    try:
+        import pickle
+        from crawlo.network.request import Request
+        
+        # 创建测试请求
+        request = Request(
+            url="https://example.com",
+            method="GET",
+            headers={"User-Agent": "Test"},
+            meta={"test": "data"},
+            priority=5
+        )
+        
+        # 测试序列化
+        serialized = pickle.dumps(request)
+        print(f"   ✅ 序列化成功，大小: {len(serialized)} bytes")
+        
+        # 测试反序列化
+        deserialized = pickle.loads(serialized)
+        print(f"   ✅ 反序列化成功: {deserialized.url}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 序列化失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_concurrent_operations(redis_url):
+    """测试并发操作"""
+    print("🔍 4. 测试并发操作...")
+    
+    async def producer(queue, start_id):
+        """生产者"""
+        try:
+            for i in range(5):
+                request = Request(url=f"https://example{start_id + i}.com", priority=i)
+                await queue.put(request, priority=i)
+                await asyncio.sleep(0.1)
+            print(f"   ✅ 生产者 {start_id} 完成")
+        except Exception as e:
+            print(f"   ❌ 生产者 {start_id} 失败: {e}")
+    
+    async def consumer(queue, consumer_id):
+        """消费者"""
+        consumed = 0
+        try:
+            for _ in range(3):  # 每个消费者处理3个请求
+                request = await queue.get(timeout=5.0)
+                if request:
+                    await queue.ack(request)
+                    consumed += 1
+                    await asyncio.sleep(0.05)
+                else:
+                    break
+            print(f"   ✅ 消费者 {consumer_id} 处理了 {consumed} 个请求")
+        except Exception as e:
+            print(f"   ❌ 消费者 {consumer_id} 失败: {e}")
+    
+    queue = RedisPriorityQueue(
+        redis_url=redis_url,
+        queue_name="test:concurrent:requests"
+    )
+    
+    try:
+        await queue.connect()
+        
+        # 并发运行生产者和消费者
+        tasks = [
+            producer(queue, 0),
+            producer(queue, 10),
+            consumer(queue, 1),
+            consumer(queue, 2),
+        ]
+        
+        await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # 检查剩余队列大小
+        final_size = await queue.qsize()
+        print(f"   📊 最终队列大小: {final_size}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 并发测试失败: {e}")
+        return False
+    finally:
+        await queue.close()
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始 Redis 分布式队列诊断...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试连接
+        redis_url = await test_redis_connection()
+        
+        # 2. 测试序列化
+        if not await test_serialization():
+            return
+            
+        # 3. 测试基本操作
+        if not await test_queue_operations(redis_url):
+            return
+            
+        # 4. 测试并发操作
+        if not await test_concurrent_operations(redis_url):
+            return
+            
+        print("=" * 50)
+        print("🎉 所有测试通过！Redis 队列工作正常")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 诊断失败: {e}")
+        traceback.print_exc()
+        
+        # 提供解决建议
+        print("\n🔧 可能的解决方案:")
+        print("1. 检查 Redis 服务是否启动: redis-server")
+        print("2. 检查 Redis 密码配置")
+        print("3. 检查防火墙和端口 6379")
+        print("4. 安装 Redis: pip install redis")
+        print("5. 检查 Redis 配置文件中的 bind 设置")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_request_serialization.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 Request 序列化问题修复
+"""
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.core.scheduler import Scheduler
+from unittest.mock import Mock
+
+# 模拟一个带 logger 的 Request
+class TestRequest(Request):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 添加一个 logger 属性模拟问题
+        from crawlo.utils.log import get_logger
+        self.logger = get_logger("test_request")
+        self.meta['spider_logger'] = get_logger("spider_logger")
+
+def test_request_serialization():
+    """测试 Request 序列化"""
+    print("🔍 测试 Request 序列化修复...")
+    
+    # 创建一个带 logger 的请求
+    request = TestRequest(
+        url="https://example.com",
+        meta={"test": "data"}  # 移除 Mock 对象
+    )
+    
+    print(f"   📦 原始请求: {request}")
+    print(f"   🔧 请求有 logger: {hasattr(request, 'logger')}")
+    print(f"   🔧 meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 创建一个 mock scheduler 来测试清理
+    class MockScheduler:
+        def _deep_clean_loggers(self, request):
+            return Scheduler._deep_clean_loggers(self, request)
+        def _remove_logger_from_dict(self, d):
+            return Scheduler._remove_logger_from_dict(self, d)
+    
+    scheduler = MockScheduler()
+    
+    # 执行清理
+    scheduler._deep_clean_loggers(request)
+    
+    print(f"   🧹 清理后有 logger: {hasattr(request, 'logger')}")
+    print(f"   🧹 清理后 meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 测试序列化
+    try:
+        serialized = pickle.dumps(request)
+        print(f"   ✅ 序列化成功，大小: {len(serialized)} bytes")
+        
+        # 测试反序列化
+        deserialized = pickle.loads(serialized)
+        print(f"   ✅ 反序列化成功: {deserialized}")
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 序列化失败: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = test_request_serialization()
+    if success:
+        print("🎉 Request 序列化修复成功！")
+    else:
+        print("❌ 序列化问题仍未解决")

tests/test_scheduler.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试修复后的 Scheduler 分布式队列功能
+"""
+import asyncio
+import sys
+from unittest.mock import Mock
+from crawlo.core.scheduler import Scheduler
+from crawlo.network.request import Request
+from crawlo.utils.log import get_logger
+
+
+class MockCrawler:
+    """模拟 Crawler 对象"""
+    def __init__(self, use_redis=True):
+        self.settings = MockSettings(use_redis)
+        self.stats = Mock()
+
+
+class MockSettings:
+    """模拟 Settings 对象"""
+    def __init__(self, use_redis=True):
+        self.use_redis = use_redis
+        
+    def get(self, key, default=None):
+        config = {
+            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'LOG_LEVEL': 'INFO',
+            'DEPTH_PRIORITY': 1,
+            'SCHEDULER_MAX_QUEUE_SIZE': 100,
+            'SCHEDULER_QUEUE_NAME': 'test:crawlo:requests',
+            'FILTER_DEBUG': False,
+            'PROJECT_NAME': 'test',
+        }
+        if self.use_redis:
+            config['REDIS_URL'] = 'redis://localhost:6379/0'
+        
+        return config.get(key, default)
+    
+    def get_int(self, key, default=0):
+        value = self.get(key, default)
+        return int(value) if value is not None else default
+        
+    def get_bool(self, key, default=False):
+        value = self.get(key, default)
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            return value.lower() in ('true', '1', 'yes')
+        return bool(value) if value is not None else default
+
+
+class MockFilter:
+    """模拟去重过滤器"""
+    def __init__(self):
+        self.seen = set()
+        
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()
+    
+    async def requested(self, request):
+        if request.url in self.seen:
+            return True
+        self.seen.add(request.url)
+        return False
+    
+    def log_stats(self, request):
+        pass
+
+
+async def test_memory_scheduler():
+    """测试内存调度器"""
+    print("🔍 测试内存调度器...")
+    
+    crawler = MockCrawler(use_redis=False)
+    scheduler = Scheduler.create_instance(crawler)
+    
+    # 模拟去重过滤器
+    scheduler.dupe_filter = MockFilter()
+    
+    scheduler.open()
+    
+    # 测试入队
+    request1 = Request(url="https://example1.com")
+    request2 = Request(url="https://example2.com")
+    
+    success1 = await scheduler.enqueue_request(request1)
+    success2 = await scheduler.enqueue_request(request2)
+    
+    print(f"   📤 入队结果: {success1}, {success2}")
+    print(f"   📊 队列大小: {len(scheduler)}")
+    
+    # 测试出队
+    req1 = await scheduler.next_request()
+    req2 = await scheduler.next_request()
+    
+    print(f"   📥 出队结果: {req1.url if req1 else None}, {req2.url if req2 else None}")
+    print(f"   📊 剩余大小: {len(scheduler)}")
+    
+    await scheduler.close()
+    print("   ✅ 内存调度器测试完成")
+
+
+async def test_redis_scheduler():
+    """测试 Redis 调度器"""
+    print("🔍 测试 Redis 调度器...")
+    
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        
+        # 模拟去重过滤器
+        scheduler.dupe_filter = MockFilter()
+        
+        scheduler.open()
+        
+        # 测试入队
+        request1 = Request(url="https://redis-test1.com", priority=5)
+        request2 = Request(url="https://redis-test2.com", priority=3)
+        request3 = Request(url="https://redis-test3.com", priority=8)
+        
+        success1 = await scheduler.enqueue_request(request1)
+        success2 = await scheduler.enqueue_request(request2)
+        success3 = await scheduler.enqueue_request(request3)
+        
+        print(f"   📤 入队结果: {success1}, {success2}, {success3}")
+        print(f"   📊 队列大小: {len(scheduler)}")
+        
+        # 等待一小段时间让 Redis 操作完成
+        await asyncio.sleep(0.5)
+        
+        # 测试出队（应该按优先级排序）
+        req1 = await scheduler.next_request()
+        req2 = await scheduler.next_request()
+        req3 = await scheduler.next_request()
+        
+        print("   📥 出队结果（按优先级）:")
+        if req1:
+            print(f"      {req1.url} (优先级: {getattr(req1, 'priority', 0)})")
+        if req2:
+            print(f"      {req2.url} (优先级: {getattr(req2, 'priority', 0)})")
+        if req3:
+            print(f"      {req3.url} (优先级: {getattr(req3, 'priority', 0)})")
+            
+        print(f"   📊 剩余大小: {len(scheduler)}")
+        
+        await scheduler.close()
+        print("   ✅ Redis 调度器测试完成")
+        
+    except Exception as e:
+        print(f"   ❌ Redis 调度器测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def test_concurrent_redis():
+    """测试并发 Redis 操作"""
+    print("🔍 测试并发 Redis 操作...")
+    
+    async def producer(scheduler, name, count):
+        """生产者"""
+        for i in range(count):
+            request = Request(url=f"https://{name}-{i}.com", priority=i % 10)
+            await scheduler.enqueue_request(request)
+            await asyncio.sleep(0.01)
+        print(f"   ✅ 生产者 {name} 完成 ({count} 个请求)")
+    
+    async def consumer(scheduler, name, count):
+        """消费者"""
+        consumed = 0
+        for _ in range(count):
+            request = await scheduler.next_request()
+            if request:
+                consumed += 1
+                await asyncio.sleep(0.005)
+            else:
+                break
+        print(f"   ✅ 消费者 {name} 处理了 {consumed} 个请求")
+    
+    try:
+        crawler = MockCrawler(use_redis=True)
+        scheduler = Scheduler.create_instance(crawler)
+        scheduler.dupe_filter = MockFilter()
+        scheduler.open()
+        
+        # 并发运行生产者和消费者
+        tasks = [
+            producer(scheduler, "producer-1", 5),
+            producer(scheduler, "producer-2", 5),
+            consumer(scheduler, "consumer-1", 3),
+            consumer(scheduler, "consumer-2", 3),
+            consumer(scheduler, "consumer-3", 4),
+        ]
+        
+        await asyncio.gather(*tasks, return_exceptions=True)
+        
+        print(f"   📊 最终队列大小: {len(scheduler)}")
+        
+        await scheduler.close()
+        print("   ✅ 并发测试完成")
+        
+    except Exception as e:
+        print(f"   ❌ 并发测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始测试修复后的 Scheduler...")
+    print("=" * 50)
+    
+    try:
+        # 1. 测试内存调度器
+        await test_memory_scheduler()
+        print()
+        
+        # 2. 测试 Redis 调度器
+        await test_redis_scheduler()
+        print()
+        
+        # 3. 测试并发操作
+        await test_concurrent_redis()
+        
+        print("=" * 50)
+        print("🎉 所有 Scheduler 测试完成！")
+        
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # 设置日志级别避免过多输出
+    import logging
+    logging.getLogger('crawlo').setLevel(logging.WARNING)
+    
+    asyncio.run(main())