crawlo 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

tests/test_queue_manager_redis_key.py

@@ -1,177 +1,180 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-QueueManager Redis Key 测试脚本
-用于验证QueueManager在创建RedisPriorityQueue时是否正确传递module_name参数
-"""
-import asyncio
-import sys
-import os
-import traceback
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
-
-
-class MockSettings:
-    """模拟设置类"""
-    def __init__(self, project_name="test_project"):
-        self.project_name = project_name
-        self.REDIS_URL = "redis://127.0.0.1:6379/15"  # 使用测试数据库
-        self.REDIS_TTL = 0
-        self.CLEANUP_FP = 0
-        self.FILTER_DEBUG = True
-        self.LOG_LEVEL = "INFO"
-        self.DECODE_RESPONSES = True
-    
-    def get(self, key, default=None):
-        if key == 'PROJECT_NAME':
-            return self.project_name
-        elif key == 'REDIS_URL':
-            return self.REDIS_URL
-        elif key == 'FILTER_DEBUG':
-            return self.FILTER_DEBUG
-        elif key == 'LOG_LEVEL':
-            return self.LOG_LEVEL
-        elif key == 'DECODE_RESPONSES':
-            return self.DECODE_RESPONSES
-        return default
-    
-    def get_bool(self, key, default=False):
-        if key == 'FILTER_DEBUG':
-            return self.FILTER_DEBUG
-        elif key == 'DECODE_RESPONSES':
-            return self.DECODE_RESPONSES
-        elif key == 'CLEANUP_FP':
-            return self.CLEANUP_FP
-        return default
-    
-    def get_int(self, key, default=0):
-        if key == 'REDIS_TTL':
-            return self.REDIS_TTL
-        elif key == 'REDIS_PORT':
-            return 6379
-        elif key == 'REDIS_DB':
-            return 0
-        elif key == 'SCHEDULER_MAX_QUEUE_SIZE':
-            return 1000
-        elif key == 'QUEUE_MAX_RETRIES':
-            return 3
-        elif key == 'QUEUE_TIMEOUT':
-            return 300
-        return default
-
-
-async def test_queue_manager_redis_key():
-    """测试QueueManager创建Redis队列时的key命名"""
-    print("测试QueueManager创建Redis队列时的key命名...")
-    
-    try:
-        # 测试不同的队列名称配置
-        test_cases = [
-            {
-                "queue_name": "crawlo:books_distributed:queue:requests",
-                "expected_module_name": "books_distributed",
-                "description": "标准项目名称"
-            },
-            {
-                "queue_name": "crawlo:api_data_collection:queue:requests",
-                "expected_module_name": "api_data_collection",
-                "description": "API数据采集项目"
-            },
-            {
-                "queue_name": "crawlo:test_project:queue:requests",
-                "expected_module_name": "test_project",
-                "description": "测试项目"
-            },
-            {
-                "queue_name": "simple_queue_name",
-                "expected_module_name": "simple_queue_name",
-                "description": "简单队列名称"
-            },
-            {
-                "queue_name": "",
-                "expected_module_name": "default",
-                "description": "空队列名称"
-            }
-        ]
-        
-        for i, test_case in enumerate(test_cases, 1):
-            print(f"   {i}. 测试 {test_case['description']}...")
-            
-            # 创建QueueConfig
-            config = QueueConfig(
-                queue_type=QueueType.REDIS,
-                redis_url="redis://127.0.0.1:6379/15",
-                queue_name=test_case["queue_name"],
-                max_queue_size=1000,
-                max_retries=3,
-                timeout=300
-            )
-            
-            # 创建QueueManager
-            queue_manager = QueueManager(config)
-            
-            # 创建队列实例
-            queue = await queue_manager._create_queue(QueueType.REDIS)
-            
-            # 验证module_name是否正确设置
-            assert hasattr(queue, 'module_name'), "RedisPriorityQueue缺少module_name属性"
-            assert queue.module_name == test_case["expected_module_name"], \
-                f"module_name不匹配: {queue.module_name} != {test_case['expected_module_name']}"
-            
-            # 验证队列名称是否符合规范
-            expected_queue_name = f"crawlo:{queue.module_name}:queue:requests"
-            expected_processing_queue = f"crawlo:{queue.module_name}:queue:processing"
-            expected_failed_queue = f"crawlo:{queue.module_name}:queue:failed"
-            
-            assert queue.queue_name == expected_queue_name, \
-                f"队列名称不匹配: {queue.queue_name} != {expected_queue_name}"
-            assert queue.processing_queue == expected_processing_queue, \
-                f"处理中队列名称不匹配: {queue.processing_queue} != {expected_processing_queue}"
-            assert queue.failed_queue == expected_failed_queue, \
-                f"失败队列名称不匹配: {queue.failed_queue} != {expected_failed_queue}"
-            
-            print(f"      module_name: {queue.module_name}")
-            print(f"      队列名称: {queue.queue_name}")
-            print(f"      处理中队列名称: {queue.processing_queue}")
-            print(f"      失败队列名称: {queue.failed_queue}")
-        
-        print("QueueManager Redis key命名测试通过！")
-        return True
-        
-    except Exception as e:
-        print(f"QueueManager Redis key命名测试失败: {e}")
-        traceback.print_exc()
-        return False
-
-
-async def main():
-    """主测试函数"""
-    print("开始QueueManager Redis key命名测试...")
-    print("=" * 50)
-    
-    try:
-        success = await test_queue_manager_redis_key()
-        
-        print("=" * 50)
-        if success:
-            print("所有测试通过！QueueManager正确传递module_name参数")
-        else:
-            print("测试失败，请检查实现")
-            return 1
-            
-    except Exception as e:
-        print("=" * 50)
-        print(f"测试过程中发生异常: {e}")
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == "__main__":
-    exit_code = asyncio.run(main())
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+QueueManager Redis Key测试脚本
+用于验证QueueManager创建Redis队列时是否正确传递module_name参数
+"""
+import asyncio
+import sys
+import os
+import traceback
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+# 导入相关模块
+from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
+
+
+class MockSettings:
+    """模拟设置类"""
+    def __init__(self):
+        self.REDIS_HOST = '127.0.0.1'
+        self.REDIS_PORT = 6379
+        self.REDIS_PASSWORD = ''
+        self.REDIS_DB = 0
+        self.REDIS_URL = 'redis://127.0.0.1:6379/0'
+        self.REDIS_TTL = 0
+        self.SCHEDULER_MAX_QUEUE_SIZE = 1000
+        self.QUEUE_MAX_RETRIES = 3
+        self.QUEUE_TIMEOUT = 300
+    
+    def get(self, key, default=None):
+        if key == 'REDIS_HOST':
+            return self.REDIS_HOST
+        elif key == 'REDIS_PASSWORD':
+            return self.REDIS_PASSWORD
+        elif key == 'REDIS_URL':
+            return self.REDIS_URL
+        elif key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        return default
+    
+    def get_int(self, key, default=0):
+        if key == 'REDIS_TTL':
+            return self.REDIS_TTL
+        elif key == 'REDIS_PORT':
+            return 6379
+        elif key == 'REDIS_DB':
+            return 0
+        elif key == 'SCHEDULER_MAX_QUEUE_SIZE':
+            return 1000
+        elif key == 'QUEUE_MAX_RETRIES':
+            return 3
+        elif key == 'QUEUE_TIMEOUT':
+            return 300
+        return default
+
+
+async def test_queue_manager_redis_key():
+    """测试QueueManager创建Redis队列时的key命名"""
+    print("测试QueueManager创建Redis队列时的key命名...")
+    
+    try:
+        # 测试不同的队列名称配置
+        test_cases = [
+            {
+                "queue_name": "crawlo:books_distributed:queue:requests",
+                "expected_module_name": "books_distributed",
+                "expected_queue_name": "crawlo:books_distributed:queue:requests",
+                "expected_processing_queue": "crawlo:books_distributed:queue:processing",
+                "expected_failed_queue": "crawlo:books_distributed:queue:failed",
+                "description": "标准项目名称"
+            },
+            {
+                "queue_name": "crawlo:api_data_collection:queue:requests",
+                "expected_module_name": "api_data_collection",
+                "expected_queue_name": "crawlo:api_data_collection:queue:requests",
+                "expected_processing_queue": "crawlo:api_data_collection:queue:processing",
+                "expected_failed_queue": "crawlo:api_data_collection:queue:failed",
+                "description": "API数据采集项目"
+            },
+            {
+                "queue_name": "crawlo:test_project:queue:requests",
+                "expected_module_name": "test_project",
+                "expected_queue_name": "crawlo:test_project:queue:requests",
+                "expected_processing_queue": "crawlo:test_project:queue:processing",
+                "expected_failed_queue": "crawlo:test_project:queue:failed",
+                "description": "测试项目"
+            },
+            {
+                "queue_name": "simple_queue_name",
+                "expected_module_name": "simple_queue_name",
+                "expected_queue_name": "crawlo:simple_queue_name",  # RedisPriorityQueue会规范化名称
+                "expected_processing_queue": "crawlo:simple_queue_name:processing",
+                "expected_failed_queue": "crawlo:simple_queue_name:failed",
+                "description": "简单队列名称"
+            },
+            {
+                "queue_name": "",
+                "expected_module_name": "default",
+                "expected_queue_name": "crawlo:",  # 空字符串会规范化为"crawlo:"
+                "expected_processing_queue": "crawlo::processing",
+                "expected_failed_queue": "crawlo::failed",
+                "description": "空队列名称"
+            }
+        ]
+        
+        for i, test_case in enumerate(test_cases, 1):
+            print(f"   {i}. 测试 {test_case['description']}...")
+            
+            # 创建QueueConfig
+            config = QueueConfig(
+                queue_type=QueueType.REDIS,
+                redis_url="redis://127.0.0.1:6379/15",
+                queue_name=test_case["queue_name"],
+                max_queue_size=1000,
+                max_retries=3,
+                timeout=300
+            )
+            
+            # 创建QueueManager
+            queue_manager = QueueManager(config)
+            
+            # 创建队列实例
+            queue = await queue_manager._create_queue(QueueType.REDIS)
+            
+            # 验证module_name是否正确设置
+            assert hasattr(queue, 'module_name'), "RedisPriorityQueue缺少module_name属性"
+            assert queue.module_name == test_case["expected_module_name"], \
+                f"module_name不匹配: {queue.module_name} != {test_case['expected_module_name']}"
+            
+            # 验证队列名称是否符合规范
+            assert queue.queue_name == test_case["expected_queue_name"], \
+                f"队列名称不匹配: {queue.queue_name} != {test_case['expected_queue_name']}"
+            assert queue.processing_queue == test_case["expected_processing_queue"], \
+                f"处理中队列名称不匹配: {queue.processing_queue} != {test_case['expected_processing_queue']}"
+            assert queue.failed_queue == test_case["expected_failed_queue"], \
+                f"失败队列名称不匹配: {queue.failed_queue} != {test_case['expected_failed_queue']}"
+            
+            print(f"      module_name: {queue.module_name}")
+            print(f"      队列名称: {queue.queue_name}")
+            print(f"      处理中队列名称: {queue.processing_queue}")
+            print(f"      失败队列名称: {queue.failed_queue}")
+        
+        print("QueueManager Redis key命名测试通过！")
+        return True
+        
+    except Exception as e:
+        print(f"QueueManager Redis key命名测试失败: {e}")
+        traceback.print_exc()
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("开始QueueManager Redis key命名测试...")
+    print("=" * 50)
+    
+    try:
+        success = await test_queue_manager_redis_key()
+        
+        print("=" * 50)
+        if success:
+            print("所有测试通过！QueueManager正确传递module_name参数")
+        else:
+            print("测试失败，请检查实现")
+            return 1
+            
+    except Exception as e:
+        print("=" * 50)
+        print(f"测试过程中发生异常: {e}")
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == "__main__":
+    exit_code = asyncio.run(main())
     sys.exit(exit_code)

tests/test_queue_naming.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+队列命名测试脚本
+用于验证Redis队列命名修复
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+
+
+class MockRequest:
+    def __init__(self, url):
+        self.url = url
+        self.priority = 0
+        self.meta = {}
+
+
+def test_queue_naming():
+    """测试队列命名"""
+    print("开始测试Redis队列命名...")
+    print("=" * 50)
+    
+    # 测试用例
+    test_cases = [
+        {
+            "name": "正常命名",
+            "queue_name": "crawlo:test_project:queue:requests",
+            "expected_queue": "crawlo:test_project:queue:requests",
+            "expected_processing": "crawlo:test_project:queue:processing",
+            "expected_failed": "crawlo:test_project:queue:failed"
+        },
+        {
+            "name": "双重 crawlo 前缀",
+            "queue_name": "crawlo:crawlo:queue:requests",
+            "expected_queue": "crawlo:queue:requests",
+            "expected_processing": "crawlo:queue:processing",
+            "expected_failed": "crawlo:queue:failed"
+        }
+    ]
+    
+    all_passed = True
+    
+    for i, test_case in enumerate(test_cases, 1):
+        print(f"测试 {i}: {test_case['name']}")
+        print(f"  输入队列名称: {test_case['queue_name']}")
+        
+        # 创建RedisPriorityQueue实例
+        queue = RedisPriorityQueue(
+            redis_url="redis://127.0.0.1:6379/0",
+            queue_name=test_case['queue_name'],
+            module_name="test_project"
+        )
+        
+        print(f"  实际队列名称: {queue.queue_name}")
+        print(f"  实际处理队列: {queue.processing_queue}")
+        print(f"  实际失败队列: {queue.failed_queue}")
+        
+        print(f"  期望队列名称: {test_case['expected_queue']}")
+        print(f"  期望处理队列: {test_case['expected_processing']}")
+        print(f"  期望失败队列: {test_case['expected_failed']}")
+        
+        # 验证结果
+        queue_name_ok = queue.queue_name == test_case['expected_queue']
+        processing_queue_ok = queue.processing_queue == test_case['expected_processing']
+        failed_queue_ok = queue.failed_queue == test_case['expected_failed']
+        
+        if queue_name_ok and processing_queue_ok and failed_queue_ok:
+            print("  ✓ 测试通过")
+        else:
+            print("  ✗ 测试失败")
+            all_passed = False
+            
+        print()
+    
+    return all_passed
+
+
+async def test_queue_operations():
+    """测试队列操作"""
+    print("开始测试Redis队列操作...")
+    print("=" * 50)
+    
+    # 创建一个RedisPriorityQueue实例
+    queue = RedisPriorityQueue(
+        redis_url='redis://127.0.0.1:6379/0',
+        queue_name='crawlo:test_project:queue:requests',
+        module_name='test_project'
+    )
+    
+    # 连接Redis
+    await queue.connect()
+    
+    print('队列名称:', queue.queue_name)
+    print('处理队列:', queue.processing_queue)
+    print('失败队列:', queue.failed_queue)
+    
+    # 清理之前的测试数据
+    await queue._redis.delete(queue.queue_name)
+    await queue._redis.delete(f'{queue.queue_name}:data')
+    await queue._redis.delete(queue.processing_queue)
+    await queue._redis.delete(f'{queue.processing_queue}:data')
+    await queue._redis.delete(queue.failed_queue)
+    
+    # 添加一个测试任务
+    request = MockRequest('https://example.com')
+    
+    # 测试放入队列
+    result = await queue.put(request, priority=1)
+    print('放入队列结果:', result)
+    
+    # 检查队列大小
+    size = await queue.qsize()
+    print('队列大小:', size)
+    
+    # 测试获取任务
+    retrieved = await queue.get(timeout=1.0)
+    print('获取请求:', retrieved.url if retrieved else None)
+    
+    if retrieved:
+        # 测试确认任务完成
+        await queue.ack(retrieved)
+        print('任务确认完成')
+    
+    # 关闭连接
+    await queue.close()
+    print("队列操作测试完成")
+
+
+async def main():
+    """主测试函数"""
+    print("开始Redis队列命名和操作测试...")
+    print("=" * 50)
+    
+    # 测试队列命名
+    naming_test_passed = test_queue_naming()
+    
+    # 测试队列操作
+    await test_queue_operations()
+    
+    print("=" * 50)
+    if naming_test_passed:
+        print("所有测试通过！")
+    else:
+        print("部分测试失败！")
+    print("测试完成！")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())