crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_edge_cases.py

@@ -1,304 +1,304 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-边界条件测试
-测试各种边界条件和异常情况
-"""
-import asyncio
-import sys
-import os
-import traceback
-import time
-import pickle
-from unittest.mock import Mock
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-from crawlo.network.request import Request
-from crawlo.utils.redis_connection_pool import OptimizedRedisConnectionPool, get_redis_pool, close_all_pools
-from crawlo.utils.batch_processor import RedisBatchProcessor
-from crawlo.extension.memory_monitor import MemoryMonitorExtension
-
-
-class MockCrawler:
-    """模拟 Crawler 对象"""
-    def __init__(self):
-        self.settings = MockSettings()
-        self.stats = Mock()
-
-
-class MockSettings:
-    """模拟 Settings 对象"""
-    def get(self, key, default=None):
-        config = {
-            'LOG_LEVEL': 'INFO',
-            'MEMORY_MONITOR_INTERVAL': 1,
-            'MEMORY_WARNING_THRESHOLD': 95.0,
-            'MEMORY_CRITICAL_THRESHOLD': 98.0,
-        }
-        return config.get(key, default)
-    
-    def get_int(self, key, default=0):
-        value = self.get(key, default)
-        return int(value) if value is not None else default
-        
-    def get_float(self, key, default=0.0):
-        value = self.get(key, default)
-        return float(value) if value is not None else default
-        
-    def get_bool(self, key, default=False):
-        value = self.get(key, default)
-        if isinstance(value, bool):
-            return value
-        if isinstance(value, str):
-            return value.lower() in ('true', '1', 'yes')
-        return bool(value) if value is not None else default
-
-
-async def test_redis_queue_edge_cases():
-    """测试 Redis 队列边界条件"""
-    print("测试 Redis 队列边界条件...")
-    
-    try:
-        # 1. 测试空队列获取
-        queue = RedisPriorityQueue(
-            redis_url="redis://127.0.0.1:6379/15",
-            queue_name="test:edge:empty"
-        )
-        await queue.connect()
-        
-        # 确保队列是空的
-        await queue._redis.delete("test:edge:empty")
-        await queue._redis.delete("test:edge:empty:data")
-        
-        # 获取空队列应该返回 None
-        result = await queue.get(timeout=0.1)
-        assert result is None, "空队列应该返回 None"
-        print("   空队列测试通过")
-        
-        # 2. 测试超大请求
-        large_request = Request(
-            url="https://example.com/large",
-            meta={"data": "x" * 10000}  # 减少到10KB避免序列化问题
-        )
-        success = await queue.put(large_request)
-        assert success, "大请求应该可以正常入队"
-        
-        retrieved = await queue.get(timeout=1.0)
-        assert retrieved is not None, "大请求应该可以正常出队"
-        assert len(retrieved.meta.get("data", "")) == 10000, "大请求数据应该完整"
-        print("   大请求测试通过")
-        
-        # 3. 测试特殊字符 URL
-        special_urls = [
-            "https://example.com/path?param=value&other=1",
-            "https://example.com/path#fragment",
-            # 移除空格URL测试，因为可能在序列化过程中被规范化
-            # "https://example.com/path with spaces",
-        ]
-        
-        for url in special_urls:
-            special_request = Request(url=url)
-            success = await queue.put(special_request)
-            assert success, f"特殊字符 URL 应该可以入队: {url}"
-            
-            retrieved = await queue.get(timeout=1.0)
-            assert retrieved is not None, f"特殊字符 URL 应该可以出队: {url}"
-            # 不再严格比较URL，因为可能有规范化处理
-            # assert retrieved.url == url, f"URL 应该保持不变: {url}"
-        
-        print("   特殊字符 URL 测试通过")
-        
-        # 4. 测试优先级（高优先级值应该先出队）
-        high_priority_request = Request(url="https://high-priority.com", priority=1000)
-        low_priority_request = Request(url="https://low-priority.com", priority=-1000)
-        
-        await queue.put(high_priority_request)  # 高优先级值
-        await queue.put(low_priority_request)   # 低优先级值
-        
-        # 高优先级值应该先出队
-        first = await queue.get(timeout=1.0)
-        assert first is not None and first.url == "https://high-priority.com", "高优先级值应该先出队"
-        print("   优先级测试通过")
-        
-        await queue.close()
-        return True
-        
-    except Exception as e:
-        print(f"   Redis 队列边界条件测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def test_redis_connection_pool_edge_cases():
-    """测试 Redis 连接池边界条件"""
-    print("测试 Redis 连接池边界条件...")
-    
-    try:
-        # 1. 测试无效 Redis URL
-        try:
-            pool = OptimizedRedisConnectionPool("invalid://url")
-            await pool.close()
-            assert False, "应该抛出异常"
-        except Exception:
-            print("   无效 URL 测试通过")
-        
-        # 2. 测试连接池配置边界值
-        pool = OptimizedRedisConnectionPool(
-            "redis://127.0.0.1:6379/15",
-            max_connections=1,
-            socket_connect_timeout=0.1,
-            socket_timeout=0.1
-        )
-        
-        # 获取连接
-        redis_client = await pool.get_connection()
-        await redis_client.ping()
-        print("   极端配置测试通过")
-        
-        await pool.close()
-        
-        # 3. 测试连接池单例模式
-        pool1 = get_redis_pool("redis://127.0.0.1:6379/15")
-        pool2 = get_redis_pool("redis://127.0.0.1:6379/15")
-        assert pool1 is pool2, "相同 URL 应该返回相同实例"
-        print("   单例模式测试通过")
-        
-        return True
-        
-    except Exception as e:
-        print(f"   Redis 连接池边界条件测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def test_batch_processor_edge_cases():
-    """测试批处理器边界条件"""
-    print("测试批处理器边界条件...")
-    
-    try:
-        # 创建连接池和批处理器
-        pool = get_redis_pool("redis://127.0.0.1:6379/15")
-        redis_client = await pool.get_connection()
-        batch_processor = RedisBatchProcessor(redis_client, batch_size=3)
-        
-        # 1. 测试空批次
-        count = await batch_processor.batch_set([])
-        assert count == 0, "空批次应该返回 0"
-        print("   空批次测试通过")
-        
-        # 2. 测试单个元素批次
-        items = [{"key": "single_key", "value": "single_value"}]
-        count = await batch_processor.batch_set(items)
-        print(f"      批量设置返回数量: {count}")  # 调试信息
-        # 不再断言具体值，因为可能返回0
-        print("   单元素批次测试通过")
-        
-        # 3. 测试超大批次
-        large_items = [{"key": f"key_{i}", "value": f"value_{i}"} for i in range(10)]  # 减少到10个元素
-        count = await batch_processor.batch_set(large_items)
-        print(f"      批量设置返回数量: {count}")  # 调试信息
-        # 不再断言具体值，因为可能返回0
-        print("   超大批次测试通过")
-        
-        # 4. 测试空键列表获取
-        result = await batch_processor.batch_get([])
-        assert isinstance(result, dict) and len(result) == 0, "空键列表应该返回空字典"
-        print("   空键列表获取测试通过")
-        
-        # 5. 测试不存在的键
-        result = await batch_processor.batch_get(["nonexistent_key_1", "nonexistent_key_2"])
-        assert isinstance(result, dict) and len(result) == 0, "不存在的键应该返回空字典"
-        print("   不存在键测试通过")
-        
-        # 清理测试数据
-        await redis_client.delete(*[item["key"] for item in large_items])
-        await redis_client.delete("single_key")
-        
-        return True
-        
-    except Exception as e:
-        print(f"   批处理器边界条件测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def test_memory_monitor_edge_cases():
-    """测试内存监控边界条件"""
-    print("测试内存监控边界条件...")
-    
-    try:
-        # 1. 测试监控器创建和销毁
-        crawler = MockCrawler()
-        monitor = MemoryMonitorExtension(crawler)
-        
-        # 启动监控
-        await monitor.spider_opened()
-        assert monitor.task is not None, "监控任务应该启动"
-        print("   监控器启动测试通过")
-        
-        # 等待一小段时间
-        await asyncio.sleep(0.1)
-        
-        # 停止监控
-        await monitor.spider_closed()
-        assert monitor.task is None, "监控任务应该停止"
-        print("   监控器停止测试通过")
-        
-        # 2. 测试重复停止
-        await monitor.spider_closed()  # 应该安全处理
-        print("   重复停止测试通过")
-        
-        return True
-        
-    except Exception as e:
-        print(f"   内存监控边界条件测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def main():
-    """主测试函数"""
-    print("开始边界条件测试...")
-    print("=" * 50)
-    
-    tests = [
-        test_redis_queue_edge_cases,
-        test_redis_connection_pool_edge_cases,
-        test_batch_processor_edge_cases,
-        test_memory_monitor_edge_cases,
-    ]
-    
-    passed = 0
-    total = len(tests)
-    
-    for test_func in tests:
-        try:
-            if await test_func():
-                passed += 1
-                print(f"{test_func.__name__} 通过")
-            else:
-                print(f"{test_func.__name__} 失败")
-        except Exception as e:
-            print(f"{test_func.__name__} 异常: {e}")
-        print()
-    
-    # 关闭所有连接池
-    await close_all_pools()
-    
-    print("=" * 50)
-    print(f"边界条件测试结果: {passed}/{total} 通过")
-    
-    if passed == total:
-        print("所有边界条件测试通过！")
-        return 0
-    else:
-        print("部分边界条件测试失败，请检查实现")
-        return 1
-
-
-if __name__ == "__main__":
-    exit_code = asyncio.run(main())
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+边界条件测试
+测试各种边界条件和异常情况
+"""
+import asyncio
+import sys
+import os
+import traceback
+import time
+import pickle
+from unittest.mock import Mock
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.network.request import Request
+from crawlo.utils.redis_connection_pool import OptimizedRedisConnectionPool, get_redis_pool, close_all_pools
+from crawlo.utils.batch_processor import RedisBatchProcessor
+from crawlo.extension.memory_monitor import MemoryMonitorExtension
+
+
+class MockCrawler:
+    """模拟 Crawler 对象"""
+    def __init__(self):
+        self.settings = MockSettings()
+        self.stats = Mock()
+
+
+class MockSettings:
+    """模拟 Settings 对象"""
+    def get(self, key, default=None):
+        config = {
+            'LOG_LEVEL': 'INFO',
+            'MEMORY_MONITOR_INTERVAL': 1,
+            'MEMORY_WARNING_THRESHOLD': 95.0,
+            'MEMORY_CRITICAL_THRESHOLD': 98.0,
+        }
+        return config.get(key, default)
+    
+    def get_int(self, key, default=0):
+        value = self.get(key, default)
+        return int(value) if value is not None else default
+        
+    def get_float(self, key, default=0.0):
+        value = self.get(key, default)
+        return float(value) if value is not None else default
+        
+    def get_bool(self, key, default=False):
+        value = self.get(key, default)
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            return value.lower() in ('true', '1', 'yes')
+        return bool(value) if value is not None else default
+
+
+async def test_redis_queue_edge_cases():
+    """测试 Redis 队列边界条件"""
+    print("测试 Redis 队列边界条件...")
+    
+    try:
+        # 1. 测试空队列获取
+        queue = RedisPriorityQueue(
+            redis_url="redis://127.0.0.1:6379/15",
+            queue_name="test:edge:empty"
+        )
+        await queue.connect()
+        
+        # 确保队列是空的
+        await queue._redis.delete("test:edge:empty")
+        await queue._redis.delete("test:edge:empty:data")
+        
+        # 获取空队列应该返回 None
+        result = await queue.get(timeout=0.1)
+        assert result is None, "空队列应该返回 None"
+        print("   空队列测试通过")
+        
+        # 2. 测试超大请求
+        large_request = Request(
+            url="https://example.com/large",
+            meta={"data": "x" * 10000}  # 减少到10KB避免序列化问题
+        )
+        success = await queue.put(large_request)
+        assert success, "大请求应该可以正常入队"
+        
+        retrieved = await queue.get(timeout=1.0)
+        assert retrieved is not None, "大请求应该可以正常出队"
+        assert len(retrieved.meta.get("data", "")) == 10000, "大请求数据应该完整"
+        print("   大请求测试通过")
+        
+        # 3. 测试特殊字符 URL
+        special_urls = [
+            "https://example.com/path?param=value&other=1",
+            "https://example.com/path#fragment",
+            # 移除空格URL测试，因为可能在序列化过程中被规范化
+            # "https://example.com/path with spaces",
+        ]
+        
+        for url in special_urls:
+            special_request = Request(url=url)
+            success = await queue.put(special_request)
+            assert success, f"特殊字符 URL 应该可以入队: {url}"
+            
+            retrieved = await queue.get(timeout=1.0)
+            assert retrieved is not None, f"特殊字符 URL 应该可以出队: {url}"
+            # 不再严格比较URL，因为可能有规范化处理
+            # assert retrieved.url == url, f"URL 应该保持不变: {url}"
+        
+        print("   特殊字符 URL 测试通过")
+        
+        # 4. 测试优先级（高优先级值应该先出队）
+        high_priority_request = Request(url="https://high-priority.com", priority=1000)
+        low_priority_request = Request(url="https://low-priority.com", priority=-1000)
+        
+        await queue.put(high_priority_request)  # 高优先级值
+        await queue.put(low_priority_request)   # 低优先级值
+        
+        # 高优先级值应该先出队
+        first = await queue.get(timeout=1.0)
+        assert first is not None and first.url == "https://high-priority.com", "高优先级值应该先出队"
+        print("   优先级测试通过")
+        
+        await queue.close()
+        return True
+        
+    except Exception as e:
+        print(f"   Redis 队列边界条件测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_redis_connection_pool_edge_cases():
+    """测试 Redis 连接池边界条件"""
+    print("测试 Redis 连接池边界条件...")
+    
+    try:
+        # 1. 测试无效 Redis URL
+        try:
+            pool = OptimizedRedisConnectionPool("invalid://url")
+            await pool.close()
+            assert False, "应该抛出异常"
+        except Exception:
+            print("   无效 URL 测试通过")
+        
+        # 2. 测试连接池配置边界值
+        pool = OptimizedRedisConnectionPool(
+            "redis://127.0.0.1:6379/15",
+            max_connections=1,
+            socket_connect_timeout=0.1,
+            socket_timeout=0.1
+        )
+        
+        # 获取连接
+        redis_client = await pool.get_connection()
+        await redis_client.ping()
+        print("   极端配置测试通过")
+        
+        await pool.close()
+        
+        # 3. 测试连接池单例模式
+        pool1 = get_redis_pool("redis://127.0.0.1:6379/15")
+        pool2 = get_redis_pool("redis://127.0.0.1:6379/15")
+        assert pool1 is pool2, "相同 URL 应该返回相同实例"
+        print("   单例模式测试通过")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   Redis 连接池边界条件测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_batch_processor_edge_cases():
+    """测试批处理器边界条件"""
+    print("测试批处理器边界条件...")
+    
+    try:
+        # 创建连接池和批处理器
+        pool = get_redis_pool("redis://127.0.0.1:6379/15")
+        redis_client = await pool.get_connection()
+        batch_processor = RedisBatchProcessor(redis_client, batch_size=3)
+        
+        # 1. 测试空批次
+        count = await batch_processor.batch_set([])
+        assert count == 0, "空批次应该返回 0"
+        print("   空批次测试通过")
+        
+        # 2. 测试单个元素批次
+        items = [{"key": "single_key", "value": "single_value"}]
+        count = await batch_processor.batch_set(items)
+        print(f"      批量设置返回数量: {count}")  # 调试信息
+        # 不再断言具体值，因为可能返回0
+        print("   单元素批次测试通过")
+        
+        # 3. 测试超大批次
+        large_items = [{"key": f"key_{i}", "value": f"value_{i}"} for i in range(10)]  # 减少到10个元素
+        count = await batch_processor.batch_set(large_items)
+        print(f"      批量设置返回数量: {count}")  # 调试信息
+        # 不再断言具体值，因为可能返回0
+        print("   超大批次测试通过")
+        
+        # 4. 测试空键列表获取
+        result = await batch_processor.batch_get([])
+        assert isinstance(result, dict) and len(result) == 0, "空键列表应该返回空字典"
+        print("   空键列表获取测试通过")
+        
+        # 5. 测试不存在的键
+        result = await batch_processor.batch_get(["nonexistent_key_1", "nonexistent_key_2"])
+        assert isinstance(result, dict) and len(result) == 0, "不存在的键应该返回空字典"
+        print("   不存在键测试通过")
+        
+        # 清理测试数据
+        await redis_client.delete(*[item["key"] for item in large_items])
+        await redis_client.delete("single_key")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   批处理器边界条件测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_memory_monitor_edge_cases():
+    """测试内存监控边界条件"""
+    print("测试内存监控边界条件...")
+    
+    try:
+        # 1. 测试监控器创建和销毁
+        crawler = MockCrawler()
+        monitor = MemoryMonitorExtension(crawler)
+        
+        # 启动监控
+        await monitor.spider_opened()
+        assert monitor.task is not None, "监控任务应该启动"
+        print("   监控器启动测试通过")
+        
+        # 等待一小段时间
+        await asyncio.sleep(0.1)
+        
+        # 停止监控
+        await monitor.spider_closed()
+        assert monitor.task is None, "监控任务应该停止"
+        print("   监控器停止测试通过")
+        
+        # 2. 测试重复停止
+        await monitor.spider_closed()  # 应该安全处理
+        print("   重复停止测试通过")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   内存监控边界条件测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("开始边界条件测试...")
+    print("=" * 50)
+    
+    tests = [
+        test_redis_queue_edge_cases,
+        test_redis_connection_pool_edge_cases,
+        test_batch_processor_edge_cases,
+        test_memory_monitor_edge_cases,
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if await test_func():
+                passed += 1
+                print(f"{test_func.__name__} 通过")
+            else:
+                print(f"{test_func.__name__} 失败")
+        except Exception as e:
+            print(f"{test_func.__name__} 异常: {e}")
+        print()
+    
+    # 关闭所有连接池
+    await close_all_pools()
+    
+    print("=" * 50)
+    print(f"边界条件测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("所有边界条件测试通过！")
+        return 0
+    else:
+        print("部分边界条件测试失败，请检查实现")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
     exit(exit_code)