crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

tests/test_redis_connection_pool.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis连接池优化工具测试
+"""
+import asyncio
+import sys
+import os
+import traceback
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.redis_connection_pool import (
+    OptimizedRedisConnectionPool, 
+    RedisBatchOperationHelper,
+    get_redis_pool,
+    close_all_pools
+)
+
+
+async def test_connection_pool():
+    """测试连接池基本功能"""
+    print("🔍 1. 测试连接池基本功能...")
+    
+    try:
+        # 创建连接池
+        pool = OptimizedRedisConnectionPool(
+            "redis://127.0.0.1:6379/15",  # 使用测试数据库
+            max_connections=10,
+            socket_connect_timeout=5,
+            socket_timeout=30
+        )
+        
+        # 获取连接
+        redis_client = await pool.get_connection()
+        
+        # 测试连接
+        await redis_client.ping()
+        print("   ✅ Redis连接测试成功")
+        
+        # 获取统计信息
+        stats = pool.get_stats()
+        print(f"   📊 连接池统计: {stats}")
+        
+        # 关闭连接池
+        await pool.close()
+        print("   ✅ 连接池关闭成功")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 连接池测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_batch_operations():
+    """测试批量操作功能"""
+    print("🔍 2. 测试批量操作功能...")
+    
+    try:
+        # 创建连接池和批量操作助手
+        pool = get_redis_pool("redis://127.0.0.1:6379/15")
+        redis_client = await pool.get_connection()
+        helper = RedisBatchOperationHelper(redis_client, batch_size=5)
+        
+        # 准备测试数据
+        test_key = "test_batch_key"
+        await redis_client.delete(test_key)  # 清理旧数据
+        
+        # 测试批量执行
+        operations = [
+            ("set", f"{test_key}:1", "value1"),
+            ("set", f"{test_key}:2", "value2"),
+            ("set", f"{test_key}:3", "value3"),
+            ("set", f"{test_key}:4", "value4"),
+            ("set", f"{test_key}:5", "value5"),
+            ("set", f"{test_key}:6", "value6"),
+            ("set", f"{test_key}:7", "value7"),
+        ]
+        
+        results = await helper.batch_execute(operations)
+        print(f"   ✅ 批量执行完成，结果数量: {len(results)}")
+        
+        # 验证结果
+        for i in range(1, 8):
+            value = await redis_client.get(f"{test_key}:{i}")
+            assert value == f"value{i}", f"值不匹配: {value} != value{i}"
+        
+        print("   ✅ 批量执行结果验证成功")
+        
+        # 测试批量Hash操作
+        hash_key = "test_batch_hash"
+        await redis_client.delete(hash_key)  # 清理旧数据
+        
+        # 批量设置Hash字段
+        hash_items = {
+            "field1": "value1",
+            "field2": "value2",
+            "field3": "value3",
+            "field4": "value4",
+            "field5": "value5",
+            "field6": "value6",
+        }
+        
+        count = await helper.batch_set_hash(hash_key, hash_items)
+        print(f"   ✅ 批量设置Hash字段，设置数量: {count}")
+        
+        # 批量获取Hash字段
+        fields = ["field1", "field3", "field5", "field7"]  # 包含一个不存在的字段
+        hash_values = await helper.batch_get_hash(hash_key, fields)
+        print(f"   ✅ 批量获取Hash字段，获取数量: {len(hash_values)}")
+        
+        # 验证获取的值
+        assert hash_values["field1"] == "value1"
+        assert hash_values["field3"] == "value3"
+        assert hash_values["field5"] == "value5"
+        assert "field7" not in hash_values  # 不存在的字段
+        
+        print("   ✅ 批量Hash操作验证成功")
+        
+        # 清理测试数据
+        await redis_client.delete(test_key, hash_key)
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 批量操作测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_connection_context():
+    """测试连接上下文管理器"""
+    print("🔍 3. 测试连接上下文管理器...")
+    
+    try:
+        # 创建连接池
+        pool = get_redis_pool("redis://127.0.0.1:6379/15")
+        
+        # 使用上下文管理器
+        async with pool.connection_context() as redis_client:
+            # 测试连接
+            await redis_client.ping()
+            print("   ✅ 连接上下文管理器测试成功")
+            
+            # 设置测试值
+            await redis_client.set("context_test_key", "context_test_value")
+            
+            # 获取测试值
+            value = await redis_client.get("context_test_key")
+            assert value == "context_test_value"
+            
+            print("   ✅ 上下文管理器操作验证成功")
+        
+        # 检查统计信息
+        stats = pool.get_stats()
+        print(f"   📊 连接池统计: {stats}")
+        
+        # 清理测试数据
+        redis_client = await pool.get_connection()
+        await redis_client.delete("context_test_key")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 连接上下文管理器测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_singleton_pattern():
+    """测试单例模式"""
+    print("🔍 4. 测试单例模式...")
+    
+    try:
+        # 获取同一个URL的连接池
+        pool1 = get_redis_pool("redis://127.0.0.1:6379/15")
+        pool2 = get_redis_pool("redis://127.0.0.1:6379/15")
+        
+        # 验证是否为同一个实例
+        assert pool1 is pool2, "单例模式失败"
+        print("   ✅ 单例模式测试成功")
+        
+        # 获取不同URL的连接池
+        pool3 = get_redis_pool("redis://127.0.0.1:6379/14")
+        assert pool1 is not pool3, "不同URL应该创建不同实例"
+        print("   ✅ 不同URL创建不同实例测试成功")
+        
+        return True
+        
+    except Exception as e:
+        print(f"   ❌ 单例模式测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def test_concurrent_access():
+    """测试并发访问"""
+    print("🔍 5. 测试并发访问...")
+    
+    try:
+        # 创建连接池
+        pool = get_redis_pool("redis://127.0.0.1:6379/15", max_connections=20)
+        
+        async def worker(worker_id: int):
+            """工作协程"""
+            try:
+                # 获取连接
+                redis_client = await pool.get_connection()
+                
+                # 执行一些操作
+                key = f"concurrent_test:{worker_id}"
+                await redis_client.set(key, f"value_{worker_id}")
+                value = await redis_client.get(key)
+                
+                # 验证操作结果
+                assert value == f"value_{worker_id}"
+                
+                # 等待一小段时间
+                await asyncio.sleep(0.1)
+                
+                # 删除测试键
+                await redis_client.delete(key)
+                
+                return True
+            except Exception as e:
+                print(f"   ❌ 工作协程 {worker_id} 失败: {e}")
+                return False
+        
+        # 创建多个并发任务
+        tasks = [worker(i) for i in range(10)]
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+        
+        # 检查结果
+        success_count = sum(1 for result in results if result is True)
+        print(f"   ✅ 并发访问测试完成，成功: {success_count}/10")
+        
+        # 检查连接池统计
+        stats = pool.get_stats()
+        print(f"   📊 连接池统计: {stats}")
+        
+        return success_count >= 8  # 允许少量失败
+        
+    except Exception as e:
+        print(f"   ❌ 并发访问测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始Redis连接池优化工具测试...")
+    print("=" * 50)
+    
+    tests = [
+        test_connection_pool,
+        test_batch_operations,
+        test_connection_context,
+        test_singleton_pattern,
+        test_concurrent_access
+    ]
+    
+    passed = 0
+    total = len(tests)
+    
+    for test_func in tests:
+        try:
+            if await test_func():
+                passed += 1
+                print(f"✅ {test_func.__name__} 通过")
+            else:
+                print(f"❌ {test_func.__name__} 失败")
+        except Exception as e:
+            print(f"❌ {test_func.__name__} 异常: {e}")
+        print()
+    
+    # 关闭所有连接池
+    await close_all_pools()
+    
+    print("=" * 50)
+    print(f"📊 测试结果: {passed}/{total} 通过")
+    
+    if passed == total:
+        print("🎉 所有测试通过！Redis连接池优化工具工作正常")
+        return 0
+    else:
+        print("❌ 部分测试失败，请检查实现")
+        return 1
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    exit(exit_code)

tests/test_redis_key_naming.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 命名规范测试脚本
+用于验证新的统一Redis key命名规范是否正确实现
+"""
+import asyncio
+import sys
+import os
+import traceback
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.filters.aioredis_filter import AioRedisFilter
+import redis.asyncio as aioredis
+
+
+class MockSettings:
+    """模拟设置类"""
+    def __init__(self, project_name="test_project"):
+        self.project_name = project_name
+        self.REDIS_URL = "redis://127.0.0.1:6379/15"  # 使用测试数据库
+        self.REDIS_TTL = 0
+        self.CLEANUP_FP = 0
+        self.FILTER_DEBUG = True
+        self.LOG_LEVEL = "INFO"
+        self.DECODE_RESPONSES = True
+    
+    def get(self, key, default=None):
+        if key == 'PROJECT_NAME':
+            return self.project_name
+        elif key == 'REDIS_URL':
+            return self.REDIS_URL
+        elif key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'LOG_LEVEL':
+            return self.LOG_LEVEL
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        return default
+    
+    def get_bool(self, key, default=False):
+        if key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'CLEANUP_FP':
+            return self.CLEANUP_FP
+        return default
+    
+    def get_int(self, key, default=0):
+        if key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        return default
+
+
+class MockCrawler:
+    """模拟爬虫类"""
+    def __init__(self, project_name="test_project"):
+        self.settings = MockSettings(project_name)
+        self.stats = {}
+
+
+async def test_redis_key_naming():
+    """测试Redis key命名规范"""
+    print("🔍 测试Redis key命名规范...")
+    
+    project_name = "test_redis_naming"
+    
+    try:
+        # 1. 测试RedisPriorityQueue的key命名
+        print("   1. 测试RedisPriorityQueue的key命名...")
+        queue = RedisPriorityQueue(
+            redis_url="redis://127.0.0.1:6379/15",
+            module_name=project_name
+        )
+        
+        expected_queue_name = f"crawlo:{project_name}:queue:requests"
+        expected_processing_queue = f"crawlo:{project_name}:queue:processing"
+        expected_failed_queue = f"crawlo:{project_name}:queue:failed"
+        
+        assert queue.queue_name == expected_queue_name, f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
+        assert queue.processing_queue == expected_processing_queue, f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
+        assert queue.failed_queue == expected_failed_queue, f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
+        
+        print(f"      ✅ 队列名称: {queue.queue_name}")
+        print(f"      ✅ 处理中队列名称: {queue.processing_queue}")
+        print(f"      ✅ 失败队列名称: {queue.failed_queue}")
+        
+        # 2. 测试AioRedisFilter的key命名
+        print("   2. 测试AioRedisFilter的key命名...")
+        mock_crawler = MockCrawler(project_name)
+        filter_instance = AioRedisFilter.create_instance(mock_crawler)
+        
+        expected_filter_key = f"crawlo:{project_name}:filter:fingerprint"
+        assert filter_instance.redis_key == expected_filter_key, f"过滤器key不匹配: {filter_instance.redis_key} != {expected_filter_key}"
+        
+        print(f"      ✅ 过滤器key: {filter_instance.redis_key}")
+        
+        # 3. 测试实际的Redis连接和基本操作
+        print("   3. 测试实际的Redis连接...")
+        await queue.connect()
+        
+        # 为AioRedisFilter创建Redis连接
+        redis_client = aioredis.from_url(
+            "redis://127.0.0.1:6379/15",
+            decode_responses=False,
+            max_connections=20,
+            encoding='utf-8'
+        )
+        filter_instance.redis = redis_client
+        
+        # 确保连接正常
+        await queue._redis.ping()
+        await filter_instance.redis.ping()
+        
+        # 清理可能存在的旧数据
+        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
+        await filter_instance.redis.delete(filter_instance.redis_key)
+        
+        # 验证Redis中key的命名格式
+        print("   4. 验证Redis中key的命名格式...")
+        # 检查key是否符合命名规范
+        assert queue.queue_name.startswith("crawlo:"), "队列名称未使用crawlo前缀"
+        assert ":queue:requests" in queue.queue_name, "队列名称未包含queue:requests"
+        
+        assert queue.processing_queue.startswith("crawlo:"), "处理中队列名称未使用crawlo前缀"
+        assert ":queue:processing" in queue.processing_queue, "处理中队列名称未包含queue:processing"
+        
+        assert queue.failed_queue.startswith("crawlo:"), "失败队列名称未使用crawlo前缀"
+        assert ":queue:failed" in queue.failed_queue, "失败队列名称未包含queue:failed"
+        
+        assert filter_instance.redis_key.startswith("crawlo:"), "过滤器key未使用crawlo前缀"
+        assert ":filter:fingerprint" in filter_instance.redis_key, "过滤器key未包含filter:fingerprint"
+        
+        print("      ✅ 所有key都符合命名规范")
+        
+        # 5. 清理测试数据
+        print("   5. 清理测试数据...")
+        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
+        await filter_instance.redis.delete(filter_instance.redis_key)
+        await queue.close()
+        await filter_instance.redis.close()
+        
+        print("✅ Redis key命名规范测试通过！")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Redis key命名规范测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始Redis key命名规范测试...")
+    print("=" * 50)
+    
+    try:
+        success = await test_redis_key_naming()
+        
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！新的Redis key命名规范工作正常")
+        else:
+            print("❌ 测试失败，请检查实现")
+            return 1
+            
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
+    sys.exit(exit_code)

tests/test_redis_key_validator.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 验证工具测试脚本
+用于验证Redis Key验证工具的功能
+"""
+import sys
+import os
+import unittest
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.redis_key_validator import (
+    RedisKeyValidator, 
+    validate_redis_key_naming, 
+    validate_multiple_redis_keys,
+    get_redis_key_info,
+    print_validation_report
+)
+
+
+class TestRedisKeyValidator(unittest.TestCase):
+    """Redis Key 验证器测试类"""
+    
+    def setUp(self):
+        """测试前准备"""
+        self.validator = RedisKeyValidator()
+    
+    def test_valid_filter_key(self):
+        """测试有效的过滤器Key"""
+        key = "crawlo:test_project:filter:fingerprint"
+        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_valid_queue_keys(self):
+        """测试有效的队列Key"""
+        keys = [
+            "crawlo:test_project:queue:requests",
+            "crawlo:test_project:queue:processing",
+            "crawlo:test_project:queue:failed"
+        ]
+        
+        for key in keys:
+            self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_valid_item_key(self):
+        """测试有效的数据项Key"""
+        key = "crawlo:test_project:item:fingerprint"
+        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_invalid_key_format(self):
+        """测试无效的Key格式"""
+        invalid_keys = [
+            "invalid_format",  # 不以crawlo开头
+            "crawlo:test_project",  # 部分缺失
+            "crawlo:test_project:invalid_component:fingerprint",  # 无效组件
+            "crawlo:test_project:queue:invalid_subcomponent",  # 无效子组件
+            "",  # 空字符串
+            None  # None值
+        ]
+        
+        for key in invalid_keys:
+            self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_project_name_mismatch(self):
+        """测试项目名称不匹配"""
+        key = "crawlo:wrong_project:filter:fingerprint"
+        self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_convenience_functions(self):
+        """测试便利函数"""
+        # 测试单个Key验证
+        key = "crawlo:test_project:filter:fingerprint"
+        self.assertTrue(validate_redis_key_naming(key, "test_project"))
+        
+        # 测试多个Key验证
+        keys = [
+            "crawlo:test_project:filter:fingerprint",
+            "crawlo:test_project:queue:requests"
+        ]
+        is_valid, invalid_keys = validate_multiple_redis_keys(keys, "test_project")
+        self.assertTrue(is_valid)
+        self.assertEqual(len(invalid_keys), 0)
+        
+        # 测试Key信息获取
+        key = "crawlo:test_project:queue:requests"
+        info = get_redis_key_info(key)
+        self.assertTrue(info['valid'])
+        self.assertEqual(info['framework'], 'crawlo')
+        self.assertEqual(info['project'], 'test_project')
+        self.assertEqual(info['component'], 'queue')
+        self.assertEqual(info['sub_component'], 'requests')
+    
+    def test_multiple_key_validation(self):
+        """测试多个Key验证"""
+        keys = [
+            "crawlo:test_project:filter:fingerprint",  # 有效
+            "crawlo:test_project:queue:requests",      # 有效
+            "crawlo:test_project:queue:processing",    # 有效
+            "crawlo:test_project:queue:failed",        # 有效
+            "crawlo:test_project:item:fingerprint",    # 有效
+            "invalid_key_format"                       # 无效
+        ]
+        
+        is_valid, invalid_keys = self.validator.validate_multiple_keys(keys, "test_project")
+        self.assertFalse(is_valid)
+        self.assertEqual(len(invalid_keys), 1)
+        self.assertEqual(invalid_keys[0], "invalid_key_format")
+
+
+def main():
+    """主测试函数"""
+    print("🚀 开始Redis Key验证工具测试...")
+    print("=" * 50)
+    
+    # 运行测试
+    unittest.main(argv=['first-arg-is-ignored'], exit=False, verbosity=2)
+    
+    print("=" * 50)
+    print("✅ Redis Key验证工具测试完成")
+
+
+if __name__ == "__main__":
+    main()