crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

tests/test_priority_behavior.py

@@ -1,212 +1,212 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试请求优先级行为
-验证优先级值越小越优先的规则
-"""
-
-import sys
-import os
-import asyncio
-from unittest.mock import Mock, patch
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.network.request import Request
-from crawlo.queue.pqueue import SpiderPriorityQueue
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-
-
-async def test_memory_queue_priority():
-    """测试内存队列的优先级行为"""
-    print("=== 测试内存队列优先级行为 ===")
-    
-    queue = SpiderPriorityQueue()
-    
-    # 创建不同优先级的请求
-    request_low_priority = Request(url="https://low-priority.com", priority=100)   # 低优先级（数值大）
-    request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 高优先级（数值小）
-    request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 正常优先级
-    
-    # 按照"数值小优先级高"的原则入队
-    await queue.put((-100, request_high_priority))  # 高优先级先入队
-    await queue.put((0, request_normal_priority))   # 正常优先级
-    await queue.put((100, request_low_priority))    # 低优先级最后入队
-    
-    print(f"  队列大小: {queue.qsize()}")
-    
-    # 出队顺序应该按照优先级从高到低
-    item1 = await queue.get(timeout=1.0)
-    item2 = await queue.get(timeout=1.0)
-    item3 = await queue.get(timeout=1.0)
-    
-    assert item1 is not None and item1[1].url == "https://high-priority.com", "高优先级应该先出队"
-    assert item2 is not None and item2[1].url == "https://normal-priority.com", "正常优先级应该第二个出队"
-    assert item3 is not None and item3[1].url == "https://low-priority.com", "低优先级应该最后出队"
-    
-    print("  ✅ 内存队列优先级测试通过")
-
-
-async def test_redis_queue_priority():
-    """测试Redis队列的优先级行为"""
-    print("\n=== 测试Redis队列优先级行为 ===")
-    
-    # 使用测试专用的Redis数据库
-    queue = RedisPriorityQueue(
-        redis_url="redis://127.0.0.1:6379/15",
-        queue_name="test:priority:queue"
-    )
-    
-    try:
-        await queue.connect()
-        
-        # 清理之前的测试数据
-        await queue._redis.delete(queue.queue_name)
-        await queue._redis.delete(f"{queue.queue_name}:data")
-        
-        # 创建不同优先级的请求
-        # 注意：Request构造函数会将传入的priority值取反存储
-        # 所以priority=100的请求实际存储为-100，priority=-100的请求实际存储为100
-        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 实际存储为-100（高优先级）
-        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 实际存储为100（低优先级）
-        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 实际存储为0（正常优先级）
-        
-        # 按照正确的顺序入队以验证优先级行为
-        # 使用实际存储的priority值
-        await queue.put(request_low_priority, priority=request_low_priority.priority)    # 实际score=-100
-        await queue.put(request_normal_priority, priority=request_normal_priority.priority)   # 实际score=0
-        await queue.put(request_high_priority, priority=request_high_priority.priority)  # 实际score=100
-        
-        print(f"  队列大小: {await queue.qsize()}")
-        
-        # 出队顺序应该按照score从小到大（priority从小到大）
-        # 所以request_low_priority先出队（score=-100），request_normal_priority第二个出队（score=0），request_high_priority最后出队（score=100）
-        item1 = await queue.get(timeout=2.0)
-        item2 = await queue.get(timeout=2.0)
-        item3 = await queue.get(timeout=2.0)
-        
-        # 验证出队顺序
-        print(f"  第一个出队: {item1.url if item1 else None}")
-        print(f"  第二个出队: {item2.url if item2 else None}")
-        print(f"  第三个出队: {item3.url if item3 else None}")
-        
-        # Redis队列中，score小的先出队，所以priority小的先出队
-        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级请求应该先出队，实际: {item1.url if item1 else None}"
-        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级请求应该第二个出队，实际: {item2.url if item2 else None}"
-        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级请求应该最后出队，实际: {item3.url if item3 else None}"
-        
-        print("  ✅ Redis队列优先级测试通过（确认了score越小越优先的规则）")
-        print("  注意：Redis队列中score = priority，所以priority值小的请求score小，会先出队")
-        
-    except Exception as e:
-        print(f"  ❌ Redis队列优先级测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-    finally:
-        await queue.close()
-
-
-async def test_priority_values():
-    """测试优先级常量值"""
-    print("\n=== 测试优先级常量值 ===")
-    
-    from crawlo.network.request import RequestPriority
-    
-    # 检查优先级常量值
-    print(f"  URGENT: {RequestPriority.URGENT}")
-    print(f"  HIGH: {RequestPriority.HIGH}")
-    print(f"  NORMAL: {RequestPriority.NORMAL}")
-    print(f"  LOW: {RequestPriority.LOW}")
-    print(f"  BACKGROUND: {RequestPriority.BACKGROUND}")
-    
-    # 验证数值越小优先级越高
-    assert RequestPriority.URGENT < RequestPriority.HIGH < RequestPriority.NORMAL < RequestPriority.LOW < RequestPriority.BACKGROUND
-    print("  ✅ 优先级常量值测试通过")
-
-
-async def test_retry_middleware_priority():
-    """测试重试中间件中的优先级调整"""
-    print("\n=== 测试重试中间件优先级调整 ===")
-    
-    from crawlo.middleware.retry import RetryMiddleware
-    from crawlo.stats_collector import StatsCollector
-    from crawlo.settings.setting_manager import SettingManager
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    settings.set('RETRY_HTTP_CODES', [500])
-    settings.set('IGNORE_HTTP_CODES', [])
-    settings.set('MAX_RETRY_TIMES', 3)
-    settings.set('RETRY_EXCEPTIONS', [])
-    settings.set('RETRY_PRIORITY', -1)  # 重试时降低优先级
-    
-    # 创建统计收集器
-    class MockCrawler:
-        def __init__(self):
-            self.settings = settings
-    
-    crawler = MockCrawler()
-    stats = StatsCollector(crawler)
-    crawler.stats = stats
-    
-    class MockCrawlerWithStats:
-        def __init__(self):
-            self.settings = settings
-            self.stats = stats
-    
-    crawler_with_stats = MockCrawlerWithStats()
-    
-    # 创建重试中间件
-    middleware = RetryMiddleware.create_instance(crawler_with_stats)
-    
-    # 创建请求和爬虫（注意：这里设置优先级为-10，因为Request构造函数会将其转换为10）
-    request = Request(url="https://example.com", priority=-10)  # 实际priority将为10
-    spider = Mock()
-    
-    print(f"  原始请求优先级: {request.priority}")  # 应该是10
-    
-    # 模拟500错误响应
-    class MockResponse:
-        def __init__(self, status_code=200):
-            self.status_code = status_code
-    
-    response = MockResponse(500)
-    result = middleware.process_response(request, response, spider)
-    
-    # 应该返回重试的请求，优先级应该降低
-    assert result is not None
-    assert result is request  # 同一个对象
-    assert result.priority == 9  # 原始优先级10 + RETRY_PRIORITY(-1) = 9
-    print(f"  重试后请求优先级: {result.priority}")
-    print("  ✅ 重试中间件优先级调整测试通过")
-
-
-async def main():
-    print("开始测试请求优先级行为...")
-    
-    try:
-        # 运行所有测试
-        await test_priority_values()
-        await test_memory_queue_priority()
-        await test_redis_queue_priority()
-        await test_retry_middleware_priority()
-        
-        print("\n🎉 所有优先级行为测试通过！")
-        print("\n总结:")
-        print("1. 请求优先级遵循'数值越小越优先'的原则")
-        print("2. 内存队列: 直接使用(priority, request)元组，priority小的先出队")
-        print("3. Redis队列: 使用score = priority，score小的先出队，所以priority小的先出队")
-        print("   现在内存队列和Redis队列行为一致")
-        print("4. 重试中间件会根据RETRY_PRIORITY配置调整请求优先级")
-        print("5. 系统内置的优先级常量: URGENT(-200) < HIGH(-100) < NORMAL(0) < LOW(100) < BACKGROUND(200)")
-        print("6. Request对象构造时会将传入的priority值取反存储")
-        
-    except Exception as e:
-        print(f"\n❌ 测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试请求优先级行为
+验证优先级值越小越优先的规则
+"""
+
+import sys
+import os
+import asyncio
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.request import Request
+from crawlo.queue.pqueue import SpiderPriorityQueue
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+
+
+async def test_memory_queue_priority():
+    """测试内存队列的优先级行为"""
+    print("=== 测试内存队列优先级行为 ===")
+    
+    queue = SpiderPriorityQueue()
+    
+    # 创建不同优先级的请求
+    request_low_priority = Request(url="https://low-priority.com", priority=100)   # 低优先级（数值大）
+    request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 高优先级（数值小）
+    request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 正常优先级
+    
+    # 按照"数值小优先级高"的原则入队
+    await queue.put((-100, request_high_priority))  # 高优先级先入队
+    await queue.put((0, request_normal_priority))   # 正常优先级
+    await queue.put((100, request_low_priority))    # 低优先级最后入队
+    
+    print(f"  队列大小: {queue.qsize()}")
+    
+    # 出队顺序应该按照优先级从高到低
+    item1 = await queue.get(timeout=1.0)
+    item2 = await queue.get(timeout=1.0)
+    item3 = await queue.get(timeout=1.0)
+    
+    assert item1 is not None and item1[1].url == "https://high-priority.com", "高优先级应该先出队"
+    assert item2 is not None and item2[1].url == "https://normal-priority.com", "正常优先级应该第二个出队"
+    assert item3 is not None and item3[1].url == "https://low-priority.com", "低优先级应该最后出队"
+    
+    print("  ✅ 内存队列优先级测试通过")
+
+
+async def test_redis_queue_priority():
+    """测试Redis队列的优先级行为"""
+    print("\n=== 测试Redis队列优先级行为 ===")
+    
+    # 使用测试专用的Redis数据库
+    queue = RedisPriorityQueue(
+        redis_url="redis://127.0.0.1:6379/15",
+        queue_name="test:priority:queue"
+    )
+    
+    try:
+        await queue.connect()
+        
+        # 清理之前的测试数据
+        await queue._redis.delete(queue.queue_name)
+        await queue._redis.delete(f"{queue.queue_name}:data")
+        
+        # 创建不同优先级的请求
+        # 注意：Request构造函数会将传入的priority值取反存储
+        # 所以priority=100的请求实际存储为-100，priority=-100的请求实际存储为100
+        request_low_priority = Request(url="https://low-priority.com", priority=100)   # 实际存储为-100（高优先级）
+        request_high_priority = Request(url="https://high-priority.com", priority=-100)  # 实际存储为100（低优先级）
+        request_normal_priority = Request(url="https://normal-priority.com", priority=0)  # 实际存储为0（正常优先级）
+        
+        # 按照正确的顺序入队以验证优先级行为
+        # 使用实际存储的priority值
+        await queue.put(request_low_priority, priority=request_low_priority.priority)    # 实际score=-100
+        await queue.put(request_normal_priority, priority=request_normal_priority.priority)   # 实际score=0
+        await queue.put(request_high_priority, priority=request_high_priority.priority)  # 实际score=100
+        
+        print(f"  队列大小: {await queue.qsize()}")
+        
+        # 出队顺序应该按照score从小到大（priority从小到大）
+        # 所以request_low_priority先出队（score=-100），request_normal_priority第二个出队（score=0），request_high_priority最后出队（score=100）
+        item1 = await queue.get(timeout=2.0)
+        item2 = await queue.get(timeout=2.0)
+        item3 = await queue.get(timeout=2.0)
+        
+        # 验证出队顺序
+        print(f"  第一个出队: {item1.url if item1 else None}")
+        print(f"  第二个出队: {item2.url if item2 else None}")
+        print(f"  第三个出队: {item3.url if item3 else None}")
+        
+        # Redis队列中，score小的先出队，所以priority小的先出队
+        assert item1 is not None and item1.url == "https://low-priority.com", f"低优先级请求应该先出队，实际: {item1.url if item1 else None}"
+        assert item2 is not None and item2.url == "https://normal-priority.com", f"正常优先级请求应该第二个出队，实际: {item2.url if item2 else None}"
+        assert item3 is not None and item3.url == "https://high-priority.com", f"高优先级请求应该最后出队，实际: {item3.url if item3 else None}"
+        
+        print("  ✅ Redis队列优先级测试通过（确认了score越小越优先的规则）")
+        print("  注意：Redis队列中score = priority，所以priority值小的请求score小，会先出队")
+        
+    except Exception as e:
+        print(f"  ❌ Redis队列优先级测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        await queue.close()
+
+
+async def test_priority_values():
+    """测试优先级常量值"""
+    print("\n=== 测试优先级常量值 ===")
+    
+    from crawlo.network.request import RequestPriority
+    
+    # 检查优先级常量值
+    print(f"  URGENT: {RequestPriority.URGENT}")
+    print(f"  HIGH: {RequestPriority.HIGH}")
+    print(f"  NORMAL: {RequestPriority.NORMAL}")
+    print(f"  LOW: {RequestPriority.LOW}")
+    print(f"  BACKGROUND: {RequestPriority.BACKGROUND}")
+    
+    # 验证数值越小优先级越高
+    assert RequestPriority.URGENT < RequestPriority.HIGH < RequestPriority.NORMAL < RequestPriority.LOW < RequestPriority.BACKGROUND
+    print("  ✅ 优先级常量值测试通过")
+
+
+async def test_retry_middleware_priority():
+    """测试重试中间件中的优先级调整"""
+    print("\n=== 测试重试中间件优先级调整 ===")
+    
+    from crawlo.middleware.retry import RetryMiddleware
+    from crawlo.stats_collector import StatsCollector
+    from crawlo.settings.setting_manager import SettingManager
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set('RETRY_HTTP_CODES', [500])
+    settings.set('IGNORE_HTTP_CODES', [])
+    settings.set('MAX_RETRY_TIMES', 3)
+    settings.set('RETRY_EXCEPTIONS', [])
+    settings.set('RETRY_PRIORITY', -1)  # 重试时降低优先级
+    
+    # 创建统计收集器
+    class MockCrawler:
+        def __init__(self):
+            self.settings = settings
+    
+    crawler = MockCrawler()
+    stats = StatsCollector(crawler)
+    crawler.stats = stats
+    
+    class MockCrawlerWithStats:
+        def __init__(self):
+            self.settings = settings
+            self.stats = stats
+    
+    crawler_with_stats = MockCrawlerWithStats()
+    
+    # 创建重试中间件
+    middleware = RetryMiddleware.create_instance(crawler_with_stats)
+    
+    # 创建请求和爬虫（注意：这里设置优先级为-10，因为Request构造函数会将其转换为10）
+    request = Request(url="https://example.com", priority=-10)  # 实际priority将为10
+    spider = Mock()
+    
+    print(f"  原始请求优先级: {request.priority}")  # 应该是10
+    
+    # 模拟500错误响应
+    class MockResponse:
+        def __init__(self, status_code=200):
+            self.status_code = status_code
+    
+    response = MockResponse(500)
+    result = middleware.process_response(request, response, spider)
+    
+    # 应该返回重试的请求，优先级应该降低
+    assert result is not None
+    assert result is request  # 同一个对象
+    assert result.priority == 9  # 原始优先级10 + RETRY_PRIORITY(-1) = 9
+    print(f"  重试后请求优先级: {result.priority}")
+    print("  ✅ 重试中间件优先级调整测试通过")
+
+
+async def main():
+    print("开始测试请求优先级行为...")
+    
+    try:
+        # 运行所有测试
+        await test_priority_values()
+        await test_memory_queue_priority()
+        await test_redis_queue_priority()
+        await test_retry_middleware_priority()
+        
+        print("\n🎉 所有优先级行为测试通过！")
+        print("\n总结:")
+        print("1. 请求优先级遵循'数值越小越优先'的原则")
+        print("2. 内存队列: 直接使用(priority, request)元组，priority小的先出队")
+        print("3. Redis队列: 使用score = priority，score小的先出队，所以priority小的先出队")
+        print("   现在内存队列和Redis队列行为一致")
+        print("4. 重试中间件会根据RETRY_PRIORITY配置调整请求优先级")
+        print("5. 系统内置的优先级常量: URGENT(-200) < HIGH(-100) < NORMAL(0) < LOW(100) < BACKGROUND(200)")
+        print("6. Request对象构造时会将传入的priority值取反存储")
+        
+    except Exception as e:
+        print(f"\n❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
     asyncio.run(main())