crawlo 1.4.2__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_queue_naming.py

@@ -1,155 +1,155 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-队列命名测试脚本
-用于验证Redis队列命名修复
-"""
-import asyncio
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
-
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-
-
-class MockRequest:
-    def __init__(self, url):
-        self.url = url
-        self.priority = 0
-        self.meta = {}
-
-
-def test_queue_naming():
-    """测试队列命名"""
-    print("开始测试Redis队列命名...")
-    print("=" * 50)
-    
-    # 测试用例
-    test_cases = [
-        {
-            "name": "正常命名",
-            "queue_name": "crawlo:test_project:queue:requests",
-            "expected_queue": "crawlo:test_project:queue:requests",
-            "expected_processing": "crawlo:test_project:queue:processing",
-            "expected_failed": "crawlo:test_project:queue:failed"
-        },
-        {
-            "name": "双重 crawlo 前缀",
-            "queue_name": "crawlo:crawlo:queue:requests",
-            "expected_queue": "crawlo:queue:requests",
-            "expected_processing": "crawlo:queue:processing",
-            "expected_failed": "crawlo:queue:failed"
-        }
-    ]
-    
-    all_passed = True
-    
-    for i, test_case in enumerate(test_cases, 1):
-        print(f"测试 {i}: {test_case['name']}")
-        print(f"  输入队列名称: {test_case['queue_name']}")
-        
-        # 创建RedisPriorityQueue实例
-        queue = RedisPriorityQueue(
-            redis_url="redis://127.0.0.1:6379/0",
-            queue_name=test_case['queue_name'],
-            module_name="test_project"
-        )
-        
-        print(f"  实际队列名称: {queue.queue_name}")
-        print(f"  实际处理队列: {queue.processing_queue}")
-        print(f"  实际失败队列: {queue.failed_queue}")
-        
-        print(f"  期望队列名称: {test_case['expected_queue']}")
-        print(f"  期望处理队列: {test_case['expected_processing']}")
-        print(f"  期望失败队列: {test_case['expected_failed']}")
-        
-        # 验证结果
-        queue_name_ok = queue.queue_name == test_case['expected_queue']
-        processing_queue_ok = queue.processing_queue == test_case['expected_processing']
-        failed_queue_ok = queue.failed_queue == test_case['expected_failed']
-        
-        if queue_name_ok and processing_queue_ok and failed_queue_ok:
-            print("  ✓ 测试通过")
-        else:
-            print("  ✗ 测试失败")
-            all_passed = False
-            
-        print()
-    
-    return all_passed
-
-
-async def test_queue_operations():
-    """测试队列操作"""
-    print("开始测试Redis队列操作...")
-    print("=" * 50)
-    
-    # 创建一个RedisPriorityQueue实例
-    queue = RedisPriorityQueue(
-        redis_url='redis://127.0.0.1:6379/0',
-        queue_name='crawlo:test_project:queue:requests',
-        module_name='test_project'
-    )
-    
-    # 连接Redis
-    await queue.connect()
-    
-    print('队列名称:', queue.queue_name)
-    print('处理队列:', queue.processing_queue)
-    print('失败队列:', queue.failed_queue)
-    
-    # 清理之前的测试数据
-    await queue._redis.delete(queue.queue_name)
-    await queue._redis.delete(f'{queue.queue_name}:data')
-    await queue._redis.delete(queue.processing_queue)
-    await queue._redis.delete(f'{queue.processing_queue}:data')
-    await queue._redis.delete(queue.failed_queue)
-    
-    # 添加一个测试任务
-    request = MockRequest('https://example.com')
-    
-    # 测试放入队列
-    result = await queue.put(request, priority=1)
-    print('放入队列结果:', result)
-    
-    # 检查队列大小
-    size = await queue.qsize()
-    print('队列大小:', size)
-    
-    # 测试获取任务
-    retrieved = await queue.get(timeout=1.0)
-    print('获取请求:', retrieved.url if retrieved else None)
-    
-    if retrieved:
-        # 测试确认任务完成
-        await queue.ack(retrieved)
-        print('任务确认完成')
-    
-    # 关闭连接
-    await queue.close()
-    print("队列操作测试完成")
-
-
-async def main():
-    """主测试函数"""
-    print("开始Redis队列命名和操作测试...")
-    print("=" * 50)
-    
-    # 测试队列命名
-    naming_test_passed = test_queue_naming()
-    
-    # 测试队列操作
-    await test_queue_operations()
-    
-    print("=" * 50)
-    if naming_test_passed:
-        print("所有测试通过！")
-    else:
-        print("部分测试失败！")
-    print("测试完成！")
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+队列命名测试脚本
+用于验证Redis队列命名修复
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
+
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+
+
+class MockRequest:
+    def __init__(self, url):
+        self.url = url
+        self.priority = 0
+        self.meta = {}
+
+
+def test_queue_naming():
+    """测试队列命名"""
+    print("开始测试Redis队列命名...")
+    print("=" * 50)
+    
+    # 测试用例
+    test_cases = [
+        {
+            "name": "正常命名",
+            "queue_name": "crawlo:test_project:queue:requests",
+            "expected_queue": "crawlo:test_project:queue:requests",
+            "expected_processing": "crawlo:test_project:queue:processing",
+            "expected_failed": "crawlo:test_project:queue:failed"
+        },
+        {
+            "name": "双重 crawlo 前缀",
+            "queue_name": "crawlo:crawlo:queue:requests",
+            "expected_queue": "crawlo:queue:requests",
+            "expected_processing": "crawlo:queue:processing",
+            "expected_failed": "crawlo:queue:failed"
+        }
+    ]
+    
+    all_passed = True
+    
+    for i, test_case in enumerate(test_cases, 1):
+        print(f"测试 {i}: {test_case['name']}")
+        print(f"  输入队列名称: {test_case['queue_name']}")
+        
+        # 创建RedisPriorityQueue实例
+        queue = RedisPriorityQueue(
+            redis_url="redis://127.0.0.1:6379/0",
+            queue_name=test_case['queue_name'],
+            module_name="test_project"
+        )
+        
+        print(f"  实际队列名称: {queue.queue_name}")
+        print(f"  实际处理队列: {queue.processing_queue}")
+        print(f"  实际失败队列: {queue.failed_queue}")
+        
+        print(f"  期望队列名称: {test_case['expected_queue']}")
+        print(f"  期望处理队列: {test_case['expected_processing']}")
+        print(f"  期望失败队列: {test_case['expected_failed']}")
+        
+        # 验证结果
+        queue_name_ok = queue.queue_name == test_case['expected_queue']
+        processing_queue_ok = queue.processing_queue == test_case['expected_processing']
+        failed_queue_ok = queue.failed_queue == test_case['expected_failed']
+        
+        if queue_name_ok and processing_queue_ok and failed_queue_ok:
+            print("  ✓ 测试通过")
+        else:
+            print("  ✗ 测试失败")
+            all_passed = False
+            
+        print()
+    
+    return all_passed
+
+
+async def test_queue_operations():
+    """测试队列操作"""
+    print("开始测试Redis队列操作...")
+    print("=" * 50)
+    
+    # 创建一个RedisPriorityQueue实例
+    queue = RedisPriorityQueue(
+        redis_url='redis://127.0.0.1:6379/0',
+        queue_name='crawlo:test_project:queue:requests',
+        module_name='test_project'
+    )
+    
+    # 连接Redis
+    await queue.connect()
+    
+    print('队列名称:', queue.queue_name)
+    print('处理队列:', queue.processing_queue)
+    print('失败队列:', queue.failed_queue)
+    
+    # 清理之前的测试数据
+    await queue._redis.delete(queue.queue_name)
+    await queue._redis.delete(f'{queue.queue_name}:data')
+    await queue._redis.delete(queue.processing_queue)
+    await queue._redis.delete(f'{queue.processing_queue}:data')
+    await queue._redis.delete(queue.failed_queue)
+    
+    # 添加一个测试任务
+    request = MockRequest('https://example.com')
+    
+    # 测试放入队列
+    result = await queue.put(request, priority=1)
+    print('放入队列结果:', result)
+    
+    # 检查队列大小
+    size = await queue.qsize()
+    print('队列大小:', size)
+    
+    # 测试获取任务
+    retrieved = await queue.get(timeout=1.0)
+    print('获取请求:', retrieved.url if retrieved else None)
+    
+    if retrieved:
+        # 测试确认任务完成
+        await queue.ack(retrieved)
+        print('任务确认完成')
+    
+    # 关闭连接
+    await queue.close()
+    print("队列操作测试完成")
+
+
+async def main():
+    """主测试函数"""
+    print("开始Redis队列命名和操作测试...")
+    print("=" * 50)
+    
+    # 测试队列命名
+    naming_test_passed = test_queue_naming()
+    
+    # 测试队列操作
+    await test_queue_operations()
+    
+    print("=" * 50)
+    if naming_test_passed:
+        print("所有测试通过！")
+    else:
+        print("部分测试失败！")
+    print("测试完成！")
+
+
+if __name__ == "__main__":
     asyncio.run(main())

tests/test_queue_type.py

@@ -1,107 +1,107 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试 QUEUE_TYPE 配置获取
-"""
-
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.framework import CrawloFramework
-from crawlo.config import CrawloConfig
-
-
-def test_queue_type_standalone():
-    """测试单机模式下的 QUEUE_TYPE"""
-    print("=== 测试单机模式下的 QUEUE_TYPE ===")
-    
-    # 创建单机模式配置
-    config = CrawloConfig.standalone(concurrency=4)
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'memory', f"期望 'memory'，实际得到 '{queue_type}'"
-    assert run_mode == 'standalone', f"期望 'standalone'，实际得到 '{run_mode}'"
-    
-    print("✅ 单机模式测试通过")
-
-
-def test_queue_type_distributed():
-    """测试分布式模式下的 QUEUE_TYPE"""
-    print("\n=== 测试分布式模式下的 QUEUE_TYPE ===")
-    
-    # 创建分布式模式配置
-    config = CrawloConfig.distributed(
-        redis_host='127.0.0.1',
-        redis_port=6379,
-        project_name='test_project',
-        concurrency=4
-    )
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'redis', f"期望 'redis'，实际得到 '{queue_type}'"
-    assert run_mode == 'distributed', f"期望 'distributed'，实际得到 '{run_mode}'"
-    
-    print("✅ 分布式模式测试通过")
-
-
-def test_queue_type_auto():
-    """测试自动模式下的 QUEUE_TYPE"""
-    print("\n=== 测试自动模式下的 QUEUE_TYPE ===")
-    
-    # 创建自动模式配置
-    config = CrawloConfig.auto(concurrency=4)
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'auto', f"期望 'auto'，实际得到 '{queue_type}'"
-    assert run_mode == 'auto', f"期望 'auto'，实际得到 '{run_mode}'"
-    
-    print("✅ 自动模式测试通过")
-
-
-if __name__ == "__main__":
-    print("开始测试 QUEUE_TYPE 配置获取...")
-    
-    try:
-        test_queue_type_standalone()
-        test_queue_type_distributed()
-        test_queue_type_auto()
-        
-        print("\n🎉 所有测试通过！可以成功获取到 QUEUE_TYPE 配置。")
-        
-    except Exception as e:
-        print(f"\n❌ 测试失败: {e}")
-        import traceback
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 QUEUE_TYPE 配置获取
+"""
+
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.framework import CrawloFramework
+from crawlo.config import CrawloConfig
+
+
+def test_queue_type_standalone():
+    """测试单机模式下的 QUEUE_TYPE"""
+    print("=== 测试单机模式下的 QUEUE_TYPE ===")
+    
+    # 创建单机模式配置
+    config = CrawloConfig.standalone(concurrency=4)
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'memory', f"期望 'memory'，实际得到 '{queue_type}'"
+    assert run_mode == 'standalone', f"期望 'standalone'，实际得到 '{run_mode}'"
+    
+    print("✅ 单机模式测试通过")
+
+
+def test_queue_type_distributed():
+    """测试分布式模式下的 QUEUE_TYPE"""
+    print("\n=== 测试分布式模式下的 QUEUE_TYPE ===")
+    
+    # 创建分布式模式配置
+    config = CrawloConfig.distributed(
+        redis_host='127.0.0.1',
+        redis_port=6379,
+        project_name='test_project',
+        concurrency=4
+    )
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'redis', f"期望 'redis'，实际得到 '{queue_type}'"
+    assert run_mode == 'distributed', f"期望 'distributed'，实际得到 '{run_mode}'"
+    
+    print("✅ 分布式模式测试通过")
+
+
+def test_queue_type_auto():
+    """测试自动模式下的 QUEUE_TYPE"""
+    print("\n=== 测试自动模式下的 QUEUE_TYPE ===")
+    
+    # 创建自动模式配置
+    config = CrawloConfig.auto(concurrency=4)
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'auto', f"期望 'auto'，实际得到 '{queue_type}'"
+    assert run_mode == 'auto', f"期望 'auto'，实际得到 '{run_mode}'"
+    
+    print("✅ 自动模式测试通过")
+
+
+if __name__ == "__main__":
+    print("开始测试 QUEUE_TYPE 配置获取...")
+    
+    try:
+        test_queue_type_standalone()
+        test_queue_type_distributed()
+        test_queue_type_auto()
+        
+        print("\n🎉 所有测试通过！可以成功获取到 QUEUE_TYPE 配置。")
+        
+    except Exception as e:
+        print(f"\n❌ 测试失败: {e}")
+        import traceback
         traceback.print_exc()

tests/test_queue_type_redis_config_consistency.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 QUEUE_TYPE = 'redis' 时的配置一致性
+验证当 QUEUE_TYPE 明确设置为 'redis' 时，过滤器和管道配置是否正确更新
+"""
+
+import sys
+import os
+import asyncio
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.config import CrawloConfig
+from crawlo.crawler import CrawlerProcess
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.queue_manager import QueueType
+
+
+def test_redis_config_consistency():
+    """测试 QUEUE_TYPE = 'redis' 时的配置一致性"""
+    print("=== 测试 QUEUE_TYPE = 'redis' 时的配置一致性 ===")
+    
+    # 创建配置，QUEUE_TYPE 设置为 'redis'，但过滤器和管道使用内存版本
+    config = {
+        'PROJECT_NAME': 'test_redis_consistency',
+        'QUEUE_TYPE': 'redis',
+        'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+        'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
+        'REDIS_URL': 'redis://127.0.0.1:6379/2',
+        'CONCURRENCY': 1,
+        'DOWNLOAD_DELAY': 0.1,
+        'LOG_LEVEL': 'INFO'
+    }
+    
+    # 验证初始配置
+    initial_filter = config['FILTER_CLASS']
+    initial_pipeline = config['DEFAULT_DEDUP_PIPELINE']
+    print(f"初始过滤器配置: {initial_filter}")
+    print(f"初始管道配置: {initial_pipeline}")
+    
+    # 验证初始配置是内存版本
+    assert 'memory_filter' in initial_filter, f"期望初始过滤器为内存版本，实际得到 {initial_filter}"
+    assert 'memory_dedup_pipeline' in initial_pipeline, f"期望初始管道为内存版本，实际得到 {initial_pipeline}"
+    print("✅ 初始配置正确（内存版本）")
+    
+    print("✅ 配置一致性测试完成")
+
+
+async def test_scheduler_redis_config_update():
+    """测试调度器在 QUEUE_TYPE = 'redis' 时的配置更新"""
+    print("\n=== 测试调度器在 QUEUE_TYPE = 'redis' 时的配置更新 ===")
+    
+    # 创建配置，QUEUE_TYPE 设置为 'redis'，但过滤器和管道使用内存版本
+    from crawlo.settings.setting_manager import SettingManager
+    settings = SettingManager()
+    settings.set('PROJECT_NAME', 'test_scheduler_redis_update')
+    settings.set('QUEUE_TYPE', 'redis')
+    settings.set('FILTER_CLASS', 'crawlo.filters.memory_filter.MemoryFilter')
+    settings.set('DEFAULT_DEDUP_PIPELINE', 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline')
+    settings.set('REDIS_URL', 'redis://127.0.0.1:6379/2')
+    settings.set('CONCURRENCY', 1)
+    settings.set('DOWNLOAD_DELAY', 0.1)
+    settings.set('LOG_LEVEL', 'INFO')
+    
+    # 创建一个模拟的爬虫对象
+    class MockCrawler:
+        def __init__(self, settings):
+            self.settings = settings
+            self.stats = None
+    
+    crawler = MockCrawler(settings)
+    
+    # 创建调度器实例
+    scheduler = Scheduler.create_instance(crawler)
+    
+    # 验证初始配置
+    initial_filter = crawler.settings.get('FILTER_CLASS')
+    initial_pipeline = crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
+    print(f"调度器创建前的过滤器配置: {initial_filter}")
+    print(f"调度器创建前的管道配置: {initial_pipeline}")
+    
+    # 初始化调度器
+    print("正在初始化调度器...")
+    await scheduler.open()
+    
+    # 检查配置是否已更新
+    updated_filter = crawler.settings.get('FILTER_CLASS')
+    updated_pipeline = crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
+    print(f"调度器初始化后的过滤器配置: {updated_filter}")
+    print(f"调度器初始化后的管道配置: {updated_pipeline}")
+    
+    # 获取队列状态
+    queue_status = scheduler.queue_manager.get_status()
+    print(f"队列类型: {queue_status['type']}")
+    print(f"队列健康状态: {queue_status['health']}")
+    
+    # 验证配置已更新为 Redis 版本
+    assert 'aioredis_filter' in updated_filter or 'redis_filter' in updated_filter, \
+        f"期望更新后的过滤器为 Redis 版本，实际得到 {updated_filter}"
+    assert 'redis_dedup_pipeline' in updated_pipeline, \
+        f"期望更新后的管道为 Redis 版本，实际得到 {updated_pipeline}"
+    print("✅ 配置已正确更新为 Redis 版本")
+    
+    # 验证队列类型为 Redis
+    assert queue_status['type'] == 'redis', f"期望队列类型为 'redis'，实际得到 '{queue_status['type']}'"
+    print("✅ 队列类型正确")
+    
+    # 清理资源
+    await scheduler.close()
+    
+    print("✅ 调度器配置更新测试完成")
+
+
+if __name__ == "__main__":
+    print("开始测试 QUEUE_TYPE = 'redis' 时的配置一致性...")
+    
+    try:
+        # 运行配置一致性测试
+        test_redis_config_consistency()
+        
+        # 运行调度器配置更新测试
+        asyncio.run(test_scheduler_redis_config_update())
+        
+        print("\n🎉 所有测试通过！QUEUE_TYPE = 'redis' 时的配置一致性已正确实现。")
+        
+    except Exception as e:
+        print(f"\n❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()