crawlo 1.2.3__py3-none-any.whl → 1.2.5__py3-none-any.whl

tests/test_redis_key_naming.py

@@ -1,182 +1,182 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Redis Key 命名规范测试脚本
-用于验证新的统一Redis key命名规范是否正确实现
-"""
-import asyncio
-import sys
-import os
-import traceback
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-from crawlo.filters.aioredis_filter import AioRedisFilter
-import redis.asyncio as aioredis
-
-
-class MockSettings:
-    """模拟设置类"""
-    def __init__(self, project_name="test_project"):
-        self.project_name = project_name
-        self.REDIS_URL = "redis://127.0.0.1:6379/15"  # 使用测试数据库
-        self.REDIS_TTL = 0
-        self.CLEANUP_FP = 0
-        self.FILTER_DEBUG = True
-        self.LOG_LEVEL = "INFO"
-        self.DECODE_RESPONSES = True
-    
-    def get(self, key, default=None):
-        if key == 'PROJECT_NAME':
-            return self.project_name
-        elif key == 'REDIS_URL':
-            return self.REDIS_URL
-        elif key == 'FILTER_DEBUG':
-            return self.FILTER_DEBUG
-        elif key == 'LOG_LEVEL':
-            return self.LOG_LEVEL
-        elif key == 'DECODE_RESPONSES':
-            return self.DECODE_RESPONSES
-        return default
-    
-    def get_bool(self, key, default=False):
-        if key == 'FILTER_DEBUG':
-            return self.FILTER_DEBUG
-        elif key == 'DECODE_RESPONSES':
-            return self.DECODE_RESPONSES
-        elif key == 'CLEANUP_FP':
-            return self.CLEANUP_FP
-        return default
-    
-    def get_int(self, key, default=0):
-        if key == 'REDIS_TTL':
-            return self.REDIS_TTL
-        return default
-
-
-class MockCrawler:
-    """模拟爬虫类"""
-    def __init__(self, project_name="test_project"):
-        self.settings = MockSettings(project_name)
-        self.stats = {}
-
-
-async def test_redis_key_naming():
-    """测试Redis key命名规范"""
-    print("🔍 测试Redis key命名规范...")
-    
-    project_name = "test_redis_naming"
-    
-    try:
-        # 1. 测试RedisPriorityQueue的key命名
-        print("   1. 测试RedisPriorityQueue的key命名...")
-        queue = RedisPriorityQueue(
-            redis_url="redis://127.0.0.1:6379/15",
-            module_name=project_name
-        )
-        
-        expected_queue_name = f"crawlo:{project_name}:queue:requests"
-        expected_processing_queue = f"crawlo:{project_name}:queue:processing"
-        expected_failed_queue = f"crawlo:{project_name}:queue:failed"
-        
-        assert queue.queue_name == expected_queue_name, f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
-        assert queue.processing_queue == expected_processing_queue, f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
-        assert queue.failed_queue == expected_failed_queue, f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
-        
-        print(f"      ✅ 队列名称: {queue.queue_name}")
-        print(f"      ✅ 处理中队列名称: {queue.processing_queue}")
-        print(f"      ✅ 失败队列名称: {queue.failed_queue}")
-        
-        # 2. 测试AioRedisFilter的key命名
-        print("   2. 测试AioRedisFilter的key命名...")
-        mock_crawler = MockCrawler(project_name)
-        filter_instance = AioRedisFilter.create_instance(mock_crawler)
-        
-        expected_filter_key = f"crawlo:{project_name}:filter:fingerprint"
-        assert filter_instance.redis_key == expected_filter_key, f"过滤器key不匹配: {filter_instance.redis_key} != {expected_filter_key}"
-        
-        print(f"      ✅ 过滤器key: {filter_instance.redis_key}")
-        
-        # 3. 测试实际的Redis连接和基本操作
-        print("   3. 测试实际的Redis连接...")
-        await queue.connect()
-        
-        # 为AioRedisFilter创建Redis连接
-        redis_client = aioredis.from_url(
-            "redis://127.0.0.1:6379/15",
-            decode_responses=False,
-            max_connections=20,
-            encoding='utf-8'
-        )
-        filter_instance.redis = redis_client
-        
-        # 确保连接正常
-        await queue._redis.ping()
-        await filter_instance.redis.ping()
-        
-        # 清理可能存在的旧数据
-        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
-        await filter_instance.redis.delete(filter_instance.redis_key)
-        
-        # 验证Redis中key的命名格式
-        print("   4. 验证Redis中key的命名格式...")
-        # 检查key是否符合命名规范
-        assert queue.queue_name.startswith("crawlo:"), "队列名称未使用crawlo前缀"
-        assert ":queue:requests" in queue.queue_name, "队列名称未包含queue:requests"
-        
-        assert queue.processing_queue.startswith("crawlo:"), "处理中队列名称未使用crawlo前缀"
-        assert ":queue:processing" in queue.processing_queue, "处理中队列名称未包含queue:processing"
-        
-        assert queue.failed_queue.startswith("crawlo:"), "失败队列名称未使用crawlo前缀"
-        assert ":queue:failed" in queue.failed_queue, "失败队列名称未包含queue:failed"
-        
-        assert filter_instance.redis_key.startswith("crawlo:"), "过滤器key未使用crawlo前缀"
-        assert ":filter:fingerprint" in filter_instance.redis_key, "过滤器key未包含filter:fingerprint"
-        
-        print("      ✅ 所有key都符合命名规范")
-        
-        # 5. 清理测试数据
-        print("   5. 清理测试数据...")
-        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
-        await filter_instance.redis.delete(filter_instance.redis_key)
-        await queue.close()
-        await filter_instance.redis.close()
-        
-        print("✅ Redis key命名规范测试通过！")
-        return True
-        
-    except Exception as e:
-        print(f"❌ Redis key命名规范测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def main():
-    """主测试函数"""
-    print("🚀 开始Redis key命名规范测试...")
-    print("=" * 50)
-    
-    try:
-        success = await test_redis_key_naming()
-        
-        print("=" * 50)
-        if success:
-            print("🎉 所有测试通过！新的Redis key命名规范工作正常")
-        else:
-            print("❌ 测试失败，请检查实现")
-            return 1
-            
-    except Exception as e:
-        print("=" * 50)
-        print(f"❌ 测试过程中发生异常: {e}")
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == "__main__":
-    exit_code = asyncio.run(main())
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 命名规范测试脚本
+用于验证新的统一Redis key命名规范是否正确实现
+"""
+import asyncio
+import sys
+import os
+import traceback
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.filters.aioredis_filter import AioRedisFilter
+import redis.asyncio as aioredis
+
+
+class MockSettings:
+    """模拟设置类"""
+    def __init__(self, project_name="test_project"):
+        self.project_name = project_name
+        self.REDIS_URL = "redis://127.0.0.1:6379/15"  # 使用测试数据库
+        self.REDIS_TTL = 0
+        self.CLEANUP_FP = 0
+        self.FILTER_DEBUG = True
+        self.LOG_LEVEL = "INFO"
+        self.DECODE_RESPONSES = True
+    
+    def get(self, key, default=None):
+        if key == 'PROJECT_NAME':
+            return self.project_name
+        elif key == 'REDIS_URL':
+            return self.REDIS_URL
+        elif key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'LOG_LEVEL':
+            return self.LOG_LEVEL
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        return default
+    
+    def get_bool(self, key, default=False):
+        if key == 'FILTER_DEBUG':
+            return self.FILTER_DEBUG
+        elif key == 'DECODE_RESPONSES':
+            return self.DECODE_RESPONSES
+        elif key == 'CLEANUP_FP':
+            return self.CLEANUP_FP
+        return default
+    
+    def get_int(self, key, default=0):
+        if key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        return default
+
+
+class MockCrawler:
+    """模拟爬虫类"""
+    def __init__(self, project_name="test_project"):
+        self.settings = MockSettings(project_name)
+        self.stats = {}
+
+
+async def test_redis_key_naming():
+    """测试Redis key命名规范"""
+    print("🔍 测试Redis key命名规范...")
+    
+    project_name = "test_redis_naming"
+    
+    try:
+        # 1. 测试RedisPriorityQueue的key命名
+        print("   1. 测试RedisPriorityQueue的key命名...")
+        queue = RedisPriorityQueue(
+            redis_url="redis://127.0.0.1:6379/15",
+            module_name=project_name
+        )
+        
+        expected_queue_name = f"crawlo:{project_name}:queue:requests"
+        expected_processing_queue = f"crawlo:{project_name}:queue:processing"
+        expected_failed_queue = f"crawlo:{project_name}:queue:failed"
+        
+        assert queue.queue_name == expected_queue_name, f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
+        assert queue.processing_queue == expected_processing_queue, f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
+        assert queue.failed_queue == expected_failed_queue, f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
+        
+        print(f"      ✅ 队列名称: {queue.queue_name}")
+        print(f"      ✅ 处理中队列名称: {queue.processing_queue}")
+        print(f"      ✅ 失败队列名称: {queue.failed_queue}")
+        
+        # 2. 测试AioRedisFilter的key命名
+        print("   2. 测试AioRedisFilter的key命名...")
+        mock_crawler = MockCrawler(project_name)
+        filter_instance = AioRedisFilter.create_instance(mock_crawler)
+        
+        expected_filter_key = f"crawlo:{project_name}:filter:fingerprint"
+        assert filter_instance.redis_key == expected_filter_key, f"过滤器key不匹配: {filter_instance.redis_key} != {expected_filter_key}"
+        
+        print(f"      ✅ 过滤器key: {filter_instance.redis_key}")
+        
+        # 3. 测试实际的Redis连接和基本操作
+        print("   3. 测试实际的Redis连接...")
+        await queue.connect()
+        
+        # 为AioRedisFilter创建Redis连接
+        redis_client = aioredis.from_url(
+            "redis://127.0.0.1:6379/15",
+            decode_responses=False,
+            max_connections=20,
+            encoding='utf-8'
+        )
+        filter_instance.redis = redis_client
+        
+        # 确保连接正常
+        await queue._redis.ping()
+        await filter_instance.redis.ping()
+        
+        # 清理可能存在的旧数据
+        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
+        await filter_instance.redis.delete(filter_instance.redis_key)
+        
+        # 验证Redis中key的命名格式
+        print("   4. 验证Redis中key的命名格式...")
+        # 检查key是否符合命名规范
+        assert queue.queue_name.startswith("crawlo:"), "队列名称未使用crawlo前缀"
+        assert ":queue:requests" in queue.queue_name, "队列名称未包含queue:requests"
+        
+        assert queue.processing_queue.startswith("crawlo:"), "处理中队列名称未使用crawlo前缀"
+        assert ":queue:processing" in queue.processing_queue, "处理中队列名称未包含queue:processing"
+        
+        assert queue.failed_queue.startswith("crawlo:"), "失败队列名称未使用crawlo前缀"
+        assert ":queue:failed" in queue.failed_queue, "失败队列名称未包含queue:failed"
+        
+        assert filter_instance.redis_key.startswith("crawlo:"), "过滤器key未使用crawlo前缀"
+        assert ":filter:fingerprint" in filter_instance.redis_key, "过滤器key未包含filter:fingerprint"
+        
+        print("      ✅ 所有key都符合命名规范")
+        
+        # 5. 清理测试数据
+        print("   5. 清理测试数据...")
+        await queue._redis.delete(queue.queue_name, queue.processing_queue, queue.failed_queue)
+        await filter_instance.redis.delete(filter_instance.redis_key)
+        await queue.close()
+        await filter_instance.redis.close()
+        
+        print("✅ Redis key命名规范测试通过！")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Redis key命名规范测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("🚀 开始Redis key命名规范测试...")
+    print("=" * 50)
+    
+    try:
+        success = await test_redis_key_naming()
+        
+        print("=" * 50)
+        if success:
+            print("🎉 所有测试通过！新的Redis key命名规范工作正常")
+        else:
+            print("❌ 测试失败，请检查实现")
+            return 1
+            
+    except Exception as e:
+        print("=" * 50)
+        print(f"❌ 测试过程中发生异常: {e}")
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
     sys.exit(exit_code)

tests/test_redis_key_validator.py

@@ -1,124 +1,124 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Redis Key 验证工具测试脚本
-用于验证Redis Key验证工具的功能
-"""
-import sys
-import os
-import unittest
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.redis_key_validator import (
-    RedisKeyValidator, 
-    validate_redis_key_naming, 
-    validate_multiple_redis_keys,
-    get_redis_key_info,
-    print_validation_report
-)
-
-
-class TestRedisKeyValidator(unittest.TestCase):
-    """Redis Key 验证器测试类"""
-    
-    def setUp(self):
-        """测试前准备"""
-        self.validator = RedisKeyValidator()
-    
-    def test_valid_filter_key(self):
-        """测试有效的过滤器Key"""
-        key = "crawlo:test_project:filter:fingerprint"
-        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
-    
-    def test_valid_queue_keys(self):
-        """测试有效的队列Key"""
-        keys = [
-            "crawlo:test_project:queue:requests",
-            "crawlo:test_project:queue:processing",
-            "crawlo:test_project:queue:failed"
-        ]
-        
-        for key in keys:
-            self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
-    
-    def test_valid_item_key(self):
-        """测试有效的数据项Key"""
-        key = "crawlo:test_project:item:fingerprint"
-        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
-    
-    def test_invalid_key_format(self):
-        """测试无效的Key格式"""
-        invalid_keys = [
-            "invalid_format",  # 不以crawlo开头
-            "crawlo:test_project",  # 部分缺失
-            "crawlo:test_project:invalid_component:fingerprint",  # 无效组件
-            "crawlo:test_project:queue:invalid_subcomponent",  # 无效子组件
-            "",  # 空字符串
-            None  # None值
-        ]
-        
-        for key in invalid_keys:
-            self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
-    
-    def test_project_name_mismatch(self):
-        """测试项目名称不匹配"""
-        key = "crawlo:wrong_project:filter:fingerprint"
-        self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
-    
-    def test_convenience_functions(self):
-        """测试便利函数"""
-        # 测试单个Key验证
-        key = "crawlo:test_project:filter:fingerprint"
-        self.assertTrue(validate_redis_key_naming(key, "test_project"))
-        
-        # 测试多个Key验证
-        keys = [
-            "crawlo:test_project:filter:fingerprint",
-            "crawlo:test_project:queue:requests"
-        ]
-        is_valid, invalid_keys = validate_multiple_redis_keys(keys, "test_project")
-        self.assertTrue(is_valid)
-        self.assertEqual(len(invalid_keys), 0)
-        
-        # 测试Key信息获取
-        key = "crawlo:test_project:queue:requests"
-        info = get_redis_key_info(key)
-        self.assertTrue(info['valid'])
-        self.assertEqual(info['framework'], 'crawlo')
-        self.assertEqual(info['project'], 'test_project')
-        self.assertEqual(info['component'], 'queue')
-        self.assertEqual(info['sub_component'], 'requests')
-    
-    def test_multiple_key_validation(self):
-        """测试多个Key验证"""
-        keys = [
-            "crawlo:test_project:filter:fingerprint",  # 有效
-            "crawlo:test_project:queue:requests",      # 有效
-            "crawlo:test_project:queue:processing",    # 有效
-            "crawlo:test_project:queue:failed",        # 有效
-            "crawlo:test_project:item:fingerprint",    # 有效
-            "invalid_key_format"                       # 无效
-        ]
-        
-        is_valid, invalid_keys = self.validator.validate_multiple_keys(keys, "test_project")
-        self.assertFalse(is_valid)
-        self.assertEqual(len(invalid_keys), 1)
-        self.assertEqual(invalid_keys[0], "invalid_key_format")
-
-
-def main():
-    """主测试函数"""
-    print("🚀 开始Redis Key验证工具测试...")
-    print("=" * 50)
-    
-    # 运行测试
-    unittest.main(argv=['first-arg-is-ignored'], exit=False, verbosity=2)
-    
-    print("=" * 50)
-    print("✅ Redis Key验证工具测试完成")
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 验证工具测试脚本
+用于验证Redis Key验证工具的功能
+"""
+import sys
+import os
+import unittest
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.redis_key_validator import (
+    RedisKeyValidator, 
+    validate_redis_key_naming, 
+    validate_multiple_redis_keys,
+    get_redis_key_info,
+    print_validation_report
+)
+
+
+class TestRedisKeyValidator(unittest.TestCase):
+    """Redis Key 验证器测试类"""
+    
+    def setUp(self):
+        """测试前准备"""
+        self.validator = RedisKeyValidator()
+    
+    def test_valid_filter_key(self):
+        """测试有效的过滤器Key"""
+        key = "crawlo:test_project:filter:fingerprint"
+        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_valid_queue_keys(self):
+        """测试有效的队列Key"""
+        keys = [
+            "crawlo:test_project:queue:requests",
+            "crawlo:test_project:queue:processing",
+            "crawlo:test_project:queue:failed"
+        ]
+        
+        for key in keys:
+            self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_valid_item_key(self):
+        """测试有效的数据项Key"""
+        key = "crawlo:test_project:item:fingerprint"
+        self.assertTrue(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_invalid_key_format(self):
+        """测试无效的Key格式"""
+        invalid_keys = [
+            "invalid_format",  # 不以crawlo开头
+            "crawlo:test_project",  # 部分缺失
+            "crawlo:test_project:invalid_component:fingerprint",  # 无效组件
+            "crawlo:test_project:queue:invalid_subcomponent",  # 无效子组件
+            "",  # 空字符串
+            None  # None值
+        ]
+        
+        for key in invalid_keys:
+            self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_project_name_mismatch(self):
+        """测试项目名称不匹配"""
+        key = "crawlo:wrong_project:filter:fingerprint"
+        self.assertFalse(self.validator.validate_key_naming(key, "test_project"))
+    
+    def test_convenience_functions(self):
+        """测试便利函数"""
+        # 测试单个Key验证
+        key = "crawlo:test_project:filter:fingerprint"
+        self.assertTrue(validate_redis_key_naming(key, "test_project"))
+        
+        # 测试多个Key验证
+        keys = [
+            "crawlo:test_project:filter:fingerprint",
+            "crawlo:test_project:queue:requests"
+        ]
+        is_valid, invalid_keys = validate_multiple_redis_keys(keys, "test_project")
+        self.assertTrue(is_valid)
+        self.assertEqual(len(invalid_keys), 0)
+        
+        # 测试Key信息获取
+        key = "crawlo:test_project:queue:requests"
+        info = get_redis_key_info(key)
+        self.assertTrue(info['valid'])
+        self.assertEqual(info['framework'], 'crawlo')
+        self.assertEqual(info['project'], 'test_project')
+        self.assertEqual(info['component'], 'queue')
+        self.assertEqual(info['sub_component'], 'requests')
+    
+    def test_multiple_key_validation(self):
+        """测试多个Key验证"""
+        keys = [
+            "crawlo:test_project:filter:fingerprint",  # 有效
+            "crawlo:test_project:queue:requests",      # 有效
+            "crawlo:test_project:queue:processing",    # 有效
+            "crawlo:test_project:queue:failed",        # 有效
+            "crawlo:test_project:item:fingerprint",    # 有效
+            "invalid_key_format"                       # 无效
+        ]
+        
+        is_valid, invalid_keys = self.validator.validate_multiple_keys(keys, "test_project")
+        self.assertFalse(is_valid)
+        self.assertEqual(len(invalid_keys), 1)
+        self.assertEqual(invalid_keys[0], "invalid_key_format")
+
+
+def main():
+    """主测试函数"""
+    print("🚀 开始Redis Key验证工具测试...")
+    print("=" * 50)
+    
+    # 运行测试
+    unittest.main(argv=['first-arg-is-ignored'], exit=False, verbosity=2)
+    
+    print("=" * 50)
+    print("✅ Redis Key验证工具测试完成")
+
+
+if __name__ == "__main__":
     main()