crawlo 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

tests/test_queue_type.py

@@ -1,107 +1,107 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试 QUEUE_TYPE 配置获取
-"""
-
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.framework import CrawloFramework
-from crawlo.config import CrawloConfig
-
-
-def test_queue_type_standalone():
-    """测试单机模式下的 QUEUE_TYPE"""
-    print("=== 测试单机模式下的 QUEUE_TYPE ===")
-    
-    # 创建单机模式配置
-    config = CrawloConfig.standalone(concurrency=4)
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'memory', f"期望 'memory'，实际得到 '{queue_type}'"
-    assert run_mode == 'standalone', f"期望 'standalone'，实际得到 '{run_mode}'"
-    
-    print("✅ 单机模式测试通过")
-
-
-def test_queue_type_distributed():
-    """测试分布式模式下的 QUEUE_TYPE"""
-    print("\n=== 测试分布式模式下的 QUEUE_TYPE ===")
-    
-    # 创建分布式模式配置
-    config = CrawloConfig.distributed(
-        redis_host='127.0.0.1',
-        redis_port=6379,
-        project_name='test_project',
-        concurrency=4
-    )
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'redis', f"期望 'redis'，实际得到 '{queue_type}'"
-    assert run_mode == 'distributed', f"期望 'distributed'，实际得到 '{run_mode}'"
-    
-    print("✅ 分布式模式测试通过")
-
-
-def test_queue_type_auto():
-    """测试自动模式下的 QUEUE_TYPE"""
-    print("\n=== 测试自动模式下的 QUEUE_TYPE ===")
-    
-    # 创建自动模式配置
-    config = CrawloConfig.auto(concurrency=4)
-    
-    # 创建框架实例
-    framework = CrawloFramework(config.to_dict())
-    
-    # 获取 QUEUE_TYPE
-    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
-    run_mode = framework.settings.get('RUN_MODE', 'not found')
-    
-    print(f"RunMode: {run_mode}")
-    print(f"QueueType: {queue_type}")
-    
-    # 验证是否正确
-    assert queue_type == 'auto', f"期望 'auto'，实际得到 '{queue_type}'"
-    assert run_mode == 'auto', f"期望 'auto'，实际得到 '{run_mode}'"
-    
-    print("✅ 自动模式测试通过")
-
-
-if __name__ == "__main__":
-    print("开始测试 QUEUE_TYPE 配置获取...")
-    
-    try:
-        test_queue_type_standalone()
-        test_queue_type_distributed()
-        test_queue_type_auto()
-        
-        print("\n🎉 所有测试通过！可以成功获取到 QUEUE_TYPE 配置。")
-        
-    except Exception as e:
-        print(f"\n❌ 测试失败: {e}")
-        import traceback
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 QUEUE_TYPE 配置获取
+"""
+
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.framework import CrawloFramework
+from crawlo.config import CrawloConfig
+
+
+def test_queue_type_standalone():
+    """测试单机模式下的 QUEUE_TYPE"""
+    print("=== 测试单机模式下的 QUEUE_TYPE ===")
+    
+    # 创建单机模式配置
+    config = CrawloConfig.standalone(concurrency=4)
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'memory', f"期望 'memory'，实际得到 '{queue_type}'"
+    assert run_mode == 'standalone', f"期望 'standalone'，实际得到 '{run_mode}'"
+    
+    print("✅ 单机模式测试通过")
+
+
+def test_queue_type_distributed():
+    """测试分布式模式下的 QUEUE_TYPE"""
+    print("\n=== 测试分布式模式下的 QUEUE_TYPE ===")
+    
+    # 创建分布式模式配置
+    config = CrawloConfig.distributed(
+        redis_host='127.0.0.1',
+        redis_port=6379,
+        project_name='test_project',
+        concurrency=4
+    )
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'redis', f"期望 'redis'，实际得到 '{queue_type}'"
+    assert run_mode == 'distributed', f"期望 'distributed'，实际得到 '{run_mode}'"
+    
+    print("✅ 分布式模式测试通过")
+
+
+def test_queue_type_auto():
+    """测试自动模式下的 QUEUE_TYPE"""
+    print("\n=== 测试自动模式下的 QUEUE_TYPE ===")
+    
+    # 创建自动模式配置
+    config = CrawloConfig.auto(concurrency=4)
+    
+    # 创建框架实例
+    framework = CrawloFramework(config.to_dict())
+    
+    # 获取 QUEUE_TYPE
+    queue_type = framework.settings.get('QUEUE_TYPE', 'not found')
+    run_mode = framework.settings.get('RUN_MODE', 'not found')
+    
+    print(f"RunMode: {run_mode}")
+    print(f"QueueType: {queue_type}")
+    
+    # 验证是否正确
+    assert queue_type == 'auto', f"期望 'auto'，实际得到 '{queue_type}'"
+    assert run_mode == 'auto', f"期望 'auto'，实际得到 '{run_mode}'"
+    
+    print("✅ 自动模式测试通过")
+
+
+if __name__ == "__main__":
+    print("开始测试 QUEUE_TYPE 配置获取...")
+    
+    try:
+        test_queue_type_standalone()
+        test_queue_type_distributed()
+        test_queue_type_auto()
+        
+        print("\n🎉 所有测试通过！可以成功获取到 QUEUE_TYPE 配置。")
+        
+    except Exception as e:
+        print(f"\n❌ 测试失败: {e}")
+        import traceback
         traceback.print_exc()

tests/test_random_user_agent.py

@@ -1,73 +1,73 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-测试随机User-Agent功能
-"""
-
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.data.user_agents import get_random_user_agent
-from crawlo.middleware.default_header import DefaultHeaderMiddleware
-from crawlo.settings.setting_manager import SettingManager
-
-
-def test_random_user_agent_function():
-    """测试随机User-Agent函数"""
-    print("=== 随机User-Agent函数测试 ===")
-    
-    # 测试各种设备类型的随机User-Agent
-    device_types = ["desktop", "mobile", "all", "chrome", "firefox", "safari", "edge", "opera"]
-    
-    for device_type in device_types:
-        print(f"\n{device_type}类型随机User-Agent:")
-        for i in range(3):
-            ua = get_random_user_agent(device_type)
-            print(f"  {i+1}. {ua}")
-    
-    print()
-
-
-def test_default_header_middleware_with_random_ua():
-    """测试DefaultHeaderMiddleware与随机User-Agent"""
-    print("=== DefaultHeaderMiddleware与随机User-Agent测试 ===")
-    
-    # 创建设置管理器
-    settings = SettingManager()
-    settings.set("LOG_LEVEL", "INFO")
-    settings.set("RANDOM_USER_AGENT_ENABLED", True)
-    settings.set("USER_AGENT_DEVICE_TYPE", "all")
-    # 添加一个默认请求头以启用中间件
-    settings.set("DEFAULT_REQUEST_HEADERS", {"Accept": "text/html"})
-    
-    # 创建中间件实例
-    middleware = DefaultHeaderMiddleware(settings, "INFO")
-    
-    print(f"随机User-Agent功能已启用: {middleware.random_user_agent_enabled}")
-    print(f"User-Agent设备类型: {middleware.user_agent_device_type}")
-    print(f"内置User-Agent列表数量: {len(middleware.user_agents)}")
-    
-    # 测试获取随机User-Agent
-    print("\n随机User-Agent测试:")
-    for i in range(5):
-        random_ua = middleware._get_random_user_agent()
-        print(f"  {i+1}. {random_ua}")
-    
-    print()
-
-
-def main():
-    """主测试函数"""
-    print("开始测试随机User-Agent功能...\n")
-    
-    test_random_user_agent_function()
-    test_default_header_middleware_with_random_ua()
-    
-    print("所有测试完成!")
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试随机User-Agent功能
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.data.user_agents import get_random_user_agent
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.settings.setting_manager import SettingManager
+
+
+def test_random_user_agent_function():
+    """测试随机User-Agent函数"""
+    print("=== 随机User-Agent函数测试 ===")
+    
+    # 测试各种设备类型的随机User-Agent
+    device_types = ["desktop", "mobile", "all", "chrome", "firefox", "safari", "edge", "opera"]
+    
+    for device_type in device_types:
+        print(f"\n{device_type}类型随机User-Agent:")
+        for i in range(3):
+            ua = get_random_user_agent(device_type)
+            print(f"  {i+1}. {ua}")
+    
+    print()
+
+
+def test_default_header_middleware_with_random_ua():
+    """测试DefaultHeaderMiddleware与随机User-Agent"""
+    print("=== DefaultHeaderMiddleware与随机User-Agent测试 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set("LOG_LEVEL", "INFO")
+    settings.set("RANDOM_USER_AGENT_ENABLED", True)
+    settings.set("USER_AGENT_DEVICE_TYPE", "all")
+    # 添加一个默认请求头以启用中间件
+    settings.set("DEFAULT_REQUEST_HEADERS", {"Accept": "text/html"})
+    
+    # 创建中间件实例
+    middleware = DefaultHeaderMiddleware(settings, "INFO")
+    
+    print(f"随机User-Agent功能已启用: {middleware.random_user_agent_enabled}")
+    print(f"User-Agent设备类型: {middleware.user_agent_device_type}")
+    print(f"内置User-Agent列表数量: {len(middleware.user_agents)}")
+    
+    # 测试获取随机User-Agent
+    print("\n随机User-Agent测试:")
+    for i in range(5):
+        random_ua = middleware._get_random_user_agent()
+        print(f"  {i+1}. {random_ua}")
+    
+    print()
+
+
+def main():
+    """主测试函数"""
+    print("开始测试随机User-Agent功能...\n")
+    
+    test_random_user_agent_function()
+    test_default_header_middleware_with_random_ua()
+    
+    print("所有测试完成!")
+
+
+if __name__ == "__main__":
     main()