crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/queue_blocking_test.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+队列阻塞行为测试脚本
+验证队列满时的阻塞行为
+"""
+
+import sys
+import os
+import asyncio
+import time
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.request import Request
+from crawlo.queue.queue_manager import QueueConfig, QueueManager
+
+
+async def test_queue_blocking_behavior():
+    """测试队列阻塞行为"""
+    print("开始测试队列阻塞行为...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建一个小型队列配置进行测试
+    settings.set('SCHEDULER_MAX_QUEUE_SIZE', 5)  # 设置非常小的队列大小
+    
+    # 创建队列配置
+    queue_config = QueueConfig.from_settings(settings)
+    queue_config.max_queue_size = 5  # 设置小队列大小进行测试
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    
+    # 初始化队列
+    await queue_manager.initialize()
+    
+    print(f"队列类型: {queue_manager._queue_type}")
+    print(f"队列最大大小: {queue_config.max_queue_size}")
+    
+    # 创建生产者任务
+    async def producer(queue_manager, name, count):
+        """生产者：向队列添加请求"""
+        print(f"生产者 {name} 开始工作，将添加 {count} 个请求")
+        start_time = time.time()
+        
+        for i in range(count):
+            request = Request(url=f'https://example.com/page{name}_{i}.html')
+            try:
+                # 这里应该会阻塞当队列满时
+                await queue_manager.put(request)
+                print(f"生产者 {name} 成功添加请求 {i}")
+            except Exception as e:
+                print(f"生产者 {name} 添加请求 {i} 失败: {e}")
+        
+        end_time = time.time()
+        print(f"生产者 {name} 完成工作，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 创建消费者任务
+    async def consumer(queue_manager, name, count):
+        """消费者：从队列获取请求"""
+        print(f"消费者 {name} 开始工作，将获取 {count} 个请求")
+        start_time = time.time()
+        
+        retrieved_count = 0
+        while retrieved_count < count:
+            try:
+                request = await queue_manager.get(timeout=2.0)
+                if request:
+                    print(f"消费者 {name} 成功获取请求: {request.url}")
+                    retrieved_count += 1
+                    # 模拟处理时间
+                    await asyncio.sleep(0.1)
+                else:
+                    print(f"消费者 {name} 超时，没有获取到请求")
+                    break
+            except Exception as e:
+                print(f"消费者 {name} 获取请求失败: {e}")
+                break
+        
+        end_time = time.time()
+        print(f"消费者 {name} 完成工作，获取了 {retrieved_count} 个请求，耗时 {end_time - start_time:.2f} 秒")
+    
+    # 同时运行生产者和消费者
+    print("开始并发测试...")
+    
+    # 生产者尝试添加10个请求到大小为5的队列
+    # 消费者会逐渐消费，生产者应该会被阻塞直到有空间
+    tasks = [
+        asyncio.create_task(producer(queue_manager, "P1", 10)),
+        asyncio.create_task(consumer(queue_manager, "C1", 10))
+    ]
+    
+    # 等待所有任务完成
+    await asyncio.gather(*tasks, return_exceptions=True)
+    
+    final_queue_size = await queue_manager.size()
+    print(f"队列最终大小: {final_queue_size}")
+    
+    # 关闭队列
+    await queue_manager.close()
+    
+    print("队列阻塞行为测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始队列阻塞行为测试...")
+    asyncio.run(test_queue_blocking_behavior())
+
+
+if __name__ == "__main__":
+    main()

tests/queue_test.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+队列系统测试脚本
+"""
+
+import sys
+import os
+import asyncio
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.network.request import Request
+from crawlo.queue.queue_manager import QueueConfig, QueueManager
+
+
+async def test_queue_system():
+    """测试队列系统"""
+    print("开始测试队列系统...")
+    
+    # 初始化框架
+    from crawlo.initialization import initialize_framework
+    settings = initialize_framework()
+    
+    # 创建一个小型队列配置进行测试
+    settings.set('SCHEDULER_MAX_QUEUE_SIZE', 10)
+    
+    # 创建队列配置
+    queue_config = QueueConfig.from_settings(settings)
+    queue_config.max_queue_size = 10  # 设置小队列大小进行测试
+    
+    # 创建队列管理器
+    queue_manager = QueueManager(queue_config)
+    
+    # 初始化队列
+    await queue_manager.initialize()
+    
+    print(f"队列类型: {queue_manager._queue_type}")
+    print(f"队列最大大小: {queue_config.max_queue_size}")
+    
+    # 测试添加请求到队列
+    print("测试添加请求到队列...")
+    tasks = []
+    for i in range(15):  # 尝试添加15个请求，超过队列大小
+        request = Request(url=f'https://example.com/page{i}.html')
+        task = asyncio.create_task(queue_manager.put(request))
+        tasks.append(task)
+    
+    # 等待所有任务完成
+    results = await asyncio.gather(*tasks, return_exceptions=True)
+    
+    success_count = sum(1 for result in results if result is True)
+    print(f"成功添加 {success_count} 个请求到队列")
+    
+    queue_size = await queue_manager.size()
+    print(f"队列当前大小: {queue_size}")
+    
+    # 测试从队列获取请求
+    print("测试从队列获取请求...")
+    retrieved_count = 0
+    while retrieved_count < 15:
+        try:
+            request = await queue_manager.get(timeout=1.0)
+            if request:
+                print(f"成功获取请求: {request.url}")
+                retrieved_count += 1
+            else:
+                break
+        except Exception as e:
+            print(f"获取请求失败: {e}")
+            break
+    
+    print(f"总共获取了 {retrieved_count} 个请求")
+    
+    final_queue_size = await queue_manager.size()
+    print(f"队列最终大小: {final_queue_size}")
+    
+    # 关闭队列
+    await queue_manager.close()
+    
+    print("队列系统测试完成！")
+
+
+def main():
+    """主函数"""
+    print("开始队列系统测试...")
+    asyncio.run(test_queue_system())
+
+
+if __name__ == "__main__":
+    main()

tests/redis_key_validation_demo.py

@@ -1,131 +1,131 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Redis Key 验证演示脚本
-演示如何使用Redis Key验证工具
-"""
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.redis_key_validator import (
-    RedisKeyValidator, 
-    validate_redis_key_naming, 
-    validate_multiple_redis_keys,
-    get_redis_key_info,
-    print_validation_report
-)
-
-
-def demonstrate_redis_key_validation():
-    """演示Redis Key验证功能"""
-    print("=== Redis Key 验证功能演示 ===\n")
-    
-    # 1. 验证单个有效的Redis Key
-    print("1. 验证单个有效的Redis Key:")
-    valid_keys = [
-        "crawlo:books_distributed:filter:fingerprint",
-        "crawlo:books_distributed:queue:requests",
-        "crawlo:books_distributed:queue:processing",
-        "crawlo:books_distributed:queue:failed",
-        "crawlo:books_distributed:item:fingerprint"
-    ]
-    
-    for key in valid_keys:
-        is_valid = validate_redis_key_naming(key, "books_distributed")
-        print(f"  {key} - {'通过' if is_valid else '失败'}")
-    
-    print()
-    
-    # 2. 验证无效的Redis Key
-    print("2. 验证无效的Redis Key:")
-    invalid_keys = [
-        "invalid_format",  # 不以crawlo开头
-        "crawlo:books_distributed",  # 部分缺失
-        "crawlo:books_distributed:invalid_component:fingerprint",  # 无效组件
-        "crawlo:books_distributed:queue:invalid_subcomponent",  # 无效子组件
-        "crawlo:wrong_project:filter:fingerprint"  # 项目名称不匹配
-    ]
-    
-    for key in invalid_keys:
-        is_valid = validate_redis_key_naming(key, "books_distributed")
-        print(f"  {key} - {'通过' if is_valid else '失败'}")
-    
-    print()
-    
-    # 3. 批量验证Redis Key
-    print("3. 批量验证Redis Key:")
-    all_keys = valid_keys + invalid_keys
-    is_valid, invalid_keys_list = validate_multiple_redis_keys(all_keys, "books_distributed")
-    
-    print(f"  批量验证结果: {'全部通过' if is_valid else '存在无效Key'}")
-    if not is_valid:
-        print("  无效的Key:")
-        for key in invalid_keys_list:
-            print(f"    - {key}")
-    
-    print()
-    
-    # 4. 获取Redis Key信息
-    print("4. 获取Redis Key信息:")
-    sample_keys = [
-        "crawlo:api_data_collection:filter:fingerprint",
-        "crawlo:api_data_collection:queue:requests",
-        "crawlo:api_data_collection:item:fingerprint"
-    ]
-    
-    for key in sample_keys:
-        info = get_redis_key_info(key)
-        print(f"  Key: {key}")
-        if info['valid']:
-            print(f"    框架: {info['framework']}")
-            print(f"    项目: {info['project']}")
-            print(f"    组件: {info['component']}")
-            if 'sub_component' in info:
-                print(f"    子组件: {info['sub_component']}")
-        else:
-            print(f"    错误: {info.get('error', '无效')}")
-        print()
-    
-    # 5. 打印验证报告
-    print("5. 打印验证报告:")
-    print_validation_report(all_keys, "books_distributed")
-
-
-def demonstrate_advanced_validation():
-    """演示高级验证功能"""
-    print("\n=== 高级验证功能演示 ===\n")
-    
-    validator = RedisKeyValidator()
-    
-    # 1. 不指定项目名称的验证
-    print("1. 不指定项目名称的验证:")
-    key = "crawlo:any_project:filter:fingerprint"
-    is_valid = validator.validate_key_naming(key)  # 不指定项目名称
-    print(f"  {key} - {'通过' if is_valid else '失败'}")
-    
-    # 2. 验证不同项目的Key
-    print("\n2. 验证不同项目的Key:")
-    project_keys = {
-        "books_distributed": [
-            "crawlo:books_distributed:filter:fingerprint",
-            "crawlo:books_distributed:queue:requests"
-        ],
-        "api_data_collection": [
-            "crawlo:api_data_collection:filter:fingerprint",
-            "crawlo:api_data_collection:queue:requests"
-        ]
-    }
-    
-    for project_name, keys in project_keys.items():
-        print(f"  项目: {project_name}")
-        for key in keys:
-            is_valid = validator.validate_key_naming(key, project_name)
-            print(f"    {key} - {'通过' if is_valid else '失败'}")
-
-
-if __name__ == "__main__":
-    demonstrate_redis_key_validation()
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Redis Key 验证演示脚本
+演示如何使用Redis Key验证工具
+"""
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.redis_key_validator import (
+    RedisKeyValidator, 
+    validate_redis_key_naming, 
+    validate_multiple_redis_keys,
+    get_redis_key_info,
+    print_validation_report
+)
+
+
+def demonstrate_redis_key_validation():
+    """演示Redis Key验证功能"""
+    print("=== Redis Key 验证功能演示 ===\n")
+    
+    # 1. 验证单个有效的Redis Key
+    print("1. 验证单个有效的Redis Key:")
+    valid_keys = [
+        "crawlo:books_distributed:filter:fingerprint",
+        "crawlo:books_distributed:queue:requests",
+        "crawlo:books_distributed:queue:processing",
+        "crawlo:books_distributed:queue:failed",
+        "crawlo:books_distributed:item:fingerprint"
+    ]
+    
+    for key in valid_keys:
+        is_valid = validate_redis_key_naming(key, "books_distributed")
+        print(f"  {key} - {'通过' if is_valid else '失败'}")
+    
+    print()
+    
+    # 2. 验证无效的Redis Key
+    print("2. 验证无效的Redis Key:")
+    invalid_keys = [
+        "invalid_format",  # 不以crawlo开头
+        "crawlo:books_distributed",  # 部分缺失
+        "crawlo:books_distributed:invalid_component:fingerprint",  # 无效组件
+        "crawlo:books_distributed:queue:invalid_subcomponent",  # 无效子组件
+        "crawlo:wrong_project:filter:fingerprint"  # 项目名称不匹配
+    ]
+    
+    for key in invalid_keys:
+        is_valid = validate_redis_key_naming(key, "books_distributed")
+        print(f"  {key} - {'通过' if is_valid else '失败'}")
+    
+    print()
+    
+    # 3. 批量验证Redis Key
+    print("3. 批量验证Redis Key:")
+    all_keys = valid_keys + invalid_keys
+    is_valid, invalid_keys_list = validate_multiple_redis_keys(all_keys, "books_distributed")
+    
+    print(f"  批量验证结果: {'全部通过' if is_valid else '存在无效Key'}")
+    if not is_valid:
+        print("  无效的Key:")
+        for key in invalid_keys_list:
+            print(f"    - {key}")
+    
+    print()
+    
+    # 4. 获取Redis Key信息
+    print("4. 获取Redis Key信息:")
+    sample_keys = [
+        "crawlo:api_data_collection:filter:fingerprint",
+        "crawlo:api_data_collection:queue:requests",
+        "crawlo:api_data_collection:item:fingerprint"
+    ]
+    
+    for key in sample_keys:
+        info = get_redis_key_info(key)
+        print(f"  Key: {key}")
+        if info['valid']:
+            print(f"    框架: {info['framework']}")
+            print(f"    项目: {info['project']}")
+            print(f"    组件: {info['component']}")
+            if 'sub_component' in info:
+                print(f"    子组件: {info['sub_component']}")
+        else:
+            print(f"    错误: {info.get('error', '无效')}")
+        print()
+    
+    # 5. 打印验证报告
+    print("5. 打印验证报告:")
+    print_validation_report(all_keys, "books_distributed")
+
+
+def demonstrate_advanced_validation():
+    """演示高级验证功能"""
+    print("\n=== 高级验证功能演示 ===\n")
+    
+    validator = RedisKeyValidator()
+    
+    # 1. 不指定项目名称的验证
+    print("1. 不指定项目名称的验证:")
+    key = "crawlo:any_project:filter:fingerprint"
+    is_valid = validator.validate_key_naming(key)  # 不指定项目名称
+    print(f"  {key} - {'通过' if is_valid else '失败'}")
+    
+    # 2. 验证不同项目的Key
+    print("\n2. 验证不同项目的Key:")
+    project_keys = {
+        "books_distributed": [
+            "crawlo:books_distributed:filter:fingerprint",
+            "crawlo:books_distributed:queue:requests"
+        ],
+        "api_data_collection": [
+            "crawlo:api_data_collection:filter:fingerprint",
+            "crawlo:api_data_collection:queue:requests"
+        ]
+    }
+    
+    for project_name, keys in project_keys.items():
+        print(f"  项目: {project_name}")
+        for key in keys:
+            is_valid = validator.validate_key_naming(key, project_name)
+            print(f"    {key} - {'通过' if is_valid else '失败'}")
+
+
+if __name__ == "__main__":
+    demonstrate_redis_key_validation()
     demonstrate_advanced_validation()