crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_user_agent_randomness.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+专门测试User-Agent随机性功能
+"""
+
+import sys
+import os
+import random
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.data.user_agents import get_random_user_agent
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+    def isEnabledFor(self, level):
+        return True
+
+
+def test_user_agent_randomness():
+    """测试User-Agent的随机性"""
+    print("=== 测试User-Agent的随机性 ===")
+    
+    # 收集20次不同中间件实例生成的User-Agent
+    ua_values = []
+    
+    for i in range(20):
+        # 每次都创建新的设置和中间件实例
+        settings = SettingManager()
+        settings.set('DEFAULT_REQUEST_HEADERS', {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        })
+        settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        settings.set('LOG_LEVEL', 'DEBUG')
+        settings.set('RANDOMNESS', True)
+        
+        crawler = Mock()
+        crawler.settings = settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            try:
+                middleware = DefaultHeaderMiddleware.create_instance(crawler)
+                
+                # 处理请求
+                request = Mock()
+                request.headers = {}
+                request.url = f'https://example.com/test{i}'
+                
+                middleware.process_request(request, Mock())
+                
+                if 'User-Agent' in request.headers:
+                    ua_values.append(request.headers['User-Agent'])
+                    
+            except Exception as e:
+                print(f"  测试失败: {e}")
+    
+    # 分析随机性
+    unique_uas = set(ua_values)
+    print(f"  生成了 {len(ua_values)} 个User-Agent")
+    print(f"  其中有 {len(unique_uas)} 个不同的User-Agent")
+    print(f"  示例: {list(unique_uas)[:5]}")
+    
+    if len(unique_uas) > 1:
+        print("  ✅ User-Agent具有良好的随机性")
+        return True
+    else:
+        print("  ❌ User-Agent缺乏随机性")
+        return False
+
+
+def test_direct_function_randomness():
+    """测试直接使用函数的随机性"""
+    print("\n=== 测试直接使用函数的随机性 ===")
+    
+    # 收集20次调用的结果
+    ua_values = []
+    
+    for i in range(20):
+        ua = get_random_user_agent()
+        ua_values.append(ua)
+    
+    # 分析随机性
+    unique_uas = set(ua_values)
+    print(f"  生成了 {len(ua_values)} 个User-Agent")
+    print(f"  其中有 {len(unique_uas)} 个不同的User-Agent")
+    print(f"  示例: {list(unique_uas)[:5]}")
+    
+    if len(unique_uas) > 1:
+        print("  ✅ 直接调用函数具有良好的随机性")
+        return True
+    else:
+        print("  ❌ 直接调用函数缺乏随机性")
+        return False
+
+
+def compare_approaches():
+    """比较不同方法的优缺点"""
+    print("\n=== 比较不同方法的优缺点 ===")
+    
+    print("方法1: 使用RANDOM_USER_AGENT_ENABLED")
+    print("  优点:")
+    print("    ✓ 内置大量真实User-Agent")
+    print("    ✓ 支持设备类型分类")
+    print("    ✓ 配置简单")
+    print("    ✓ 专门优化")
+    print("  缺点:")
+    print("    ✗ 仅限User-Agent")
+    
+    print("\n方法2: 使用RANDOM_HEADERS")
+    print("  优点:")
+    print("    ✓ 可以为任意头部添加随机值")
+    print("    ✓ 更加灵活")
+    print("    ✓ 适用于多种场景")
+    print("  缺点:")
+    print("    ✗ 需要用户提供值列表")
+    print("    ✗ 配置相对复杂")
+    
+    print("\n方法3: 直接使用get_random_user_agent()")
+    print("  优点:")
+    print("    ✓ 最直接")
+    print("    ✓ 可编程控制")
+    print("    ✓ 无需中间件")
+    print("  缺点:")
+    print("    ✗ 需要手动实现")
+    print("    ✗ 不如中间件方便")
+
+
+def main():
+    print("开始测试User-Agent随机性功能...")
+    
+    try:
+        # 运行所有测试
+        test1_result = test_user_agent_randomness()
+        test2_result = test_direct_function_randomness()
+        compare_approaches()
+        
+        if test1_result and test2_result:
+            print("\n🎉 User-Agent随机性测试通过！")
+            print("\n结论:")
+            print("  1. 现有的User-Agent功能具有良好的随机性")
+            print("  2. 可以满足大多数反爬虫需求")
+            print("  3. RANDOM_HEADERS参数提供了额外的灵活性，但不是必需的")
+        else:
+            print("\n❌ User-Agent随机性测试失败")
+        
+    except Exception as e:
+        print(f"\n❌ 测试过程中发生错误: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

tests/test_user_agents.py

@@ -1,97 +1,97 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
-测试User-Agent列表的功能
-"""
-
-import sys
-import os
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.data.user_agents import (
-    DESKTOP_USER_AGENTS,
-    MOBILE_USER_AGENTS,
-    BOT_USER_AGENTS,
-    CHROME_USER_AGENTS,
-    FIREFOX_USER_AGENTS,
-    SAFARI_USER_AGENTS,
-    EDGE_USER_AGENTS,
-    OPERA_USER_AGENTS,
-    ALL_USER_AGENTS,
-    USER_AGENTS_BY_TYPE,
-    get_user_agents,
-    get_random_user_agent
-)
-
-
-def test_user_agent_counts():
-    """测试User-Agent数量"""
-    print("=== User-Agent数量测试 ===")
-    print(f"桌面浏览器User-Agent数量: {len(DESKTOP_USER_AGENTS)}")
-    print(f"移动设备User-Agent数量: {len(MOBILE_USER_AGENTS)}")
-    print(f"爬虫User-Agent数量: {len(BOT_USER_AGENTS)}")
-    print(f"Chrome User-Agent数量: {len(CHROME_USER_AGENTS)}")
-    print(f"Firefox User-Agent数量: {len(FIREFOX_USER_AGENTS)}")
-    print(f"Safari User-Agent数量: {len(SAFARI_USER_AGENTS)}")
-    print(f"Edge User-Agent数量: {len(EDGE_USER_AGENTS)}")
-    print(f"Opera User-Agent数量: {len(OPERA_USER_AGENTS)}")
-    print(f"所有User-Agent数量: {len(ALL_USER_AGENTS)}")
-    print()
-
-
-def test_get_user_agents():
-    """测试get_user_agents函数"""
-    print("=== get_user_agents函数测试 ===")
-    for device_type in ["desktop", "mobile", "bot", "all", "chrome", "firefox", "safari", "edge", "opera"]:
-        user_agents = get_user_agents(device_type)
-        print(f"{device_type}类型User-Agent数量: {len(user_agents)}")
-    print()
-
-
-def test_get_random_user_agent():
-    """测试get_random_user_agent函数"""
-    print("=== get_random_user_agent函数测试 ===")
-    for device_type in ["desktop", "mobile", "all", "chrome", "firefox"]:
-        ua = get_random_user_agent(device_type)
-        print(f"{device_type}类型随机User-Agent: {ua[:100]}...")
-    print()
-
-
-def test_user_agents_content():
-    """测试User-Agent内容"""
-    print("=== User-Agent内容测试 ===")
-    
-    # 检查是否包含最新的浏览器版本
-    chrome_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Chrome/136" in ua)
-    firefox_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Firefox/136" in ua)
-    safari_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Version/18" in ua and "Safari" in ua)
-    
-    print(f"包含Chrome 136的User-Agent数量: {chrome_ua_count}")
-    print(f"包含Firefox 136的User-Agent数量: {firefox_ua_count}")
-    print(f"包含Safari 18的User-Agent数量: {safari_ua_count}")
-    
-    # 检查是否包含移动设备User-Agent
-    ios_ua_count = sum(1 for ua in ALL_USER_AGENTS if "iPhone" in ua or "iPad" in ua)
-    android_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Android" in ua)
-    
-    print(f"包含iOS设备的User-Agent数量: {ios_ua_count}")
-    print(f"包含Android设备的User-Agent数量: {android_ua_count}")
-    print()
-
-
-def main():
-    """主测试函数"""
-    print("开始测试User-Agent列表...\n")
-    
-    test_user_agent_counts()
-    test_get_user_agents()
-    test_get_random_user_agent()
-    test_user_agents_content()
-    
-    print("所有测试完成!")
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试User-Agent列表的功能
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.data.user_agents import (
+    DESKTOP_USER_AGENTS,
+    MOBILE_USER_AGENTS,
+    BOT_USER_AGENTS,
+    CHROME_USER_AGENTS,
+    FIREFOX_USER_AGENTS,
+    SAFARI_USER_AGENTS,
+    EDGE_USER_AGENTS,
+    OPERA_USER_AGENTS,
+    ALL_USER_AGENTS,
+    USER_AGENTS_BY_TYPE,
+    get_user_agents,
+    get_random_user_agent
+)
+
+
+def test_user_agent_counts():
+    """测试User-Agent数量"""
+    print("=== User-Agent数量测试 ===")
+    print(f"桌面浏览器User-Agent数量: {len(DESKTOP_USER_AGENTS)}")
+    print(f"移动设备User-Agent数量: {len(MOBILE_USER_AGENTS)}")
+    print(f"爬虫User-Agent数量: {len(BOT_USER_AGENTS)}")
+    print(f"Chrome User-Agent数量: {len(CHROME_USER_AGENTS)}")
+    print(f"Firefox User-Agent数量: {len(FIREFOX_USER_AGENTS)}")
+    print(f"Safari User-Agent数量: {len(SAFARI_USER_AGENTS)}")
+    print(f"Edge User-Agent数量: {len(EDGE_USER_AGENTS)}")
+    print(f"Opera User-Agent数量: {len(OPERA_USER_AGENTS)}")
+    print(f"所有User-Agent数量: {len(ALL_USER_AGENTS)}")
+    print()
+
+
+def test_get_user_agents():
+    """测试get_user_agents函数"""
+    print("=== get_user_agents函数测试 ===")
+    for device_type in ["desktop", "mobile", "bot", "all", "chrome", "firefox", "safari", "edge", "opera"]:
+        user_agents = get_user_agents(device_type)
+        print(f"{device_type}类型User-Agent数量: {len(user_agents)}")
+    print()
+
+
+def test_get_random_user_agent():
+    """测试get_random_user_agent函数"""
+    print("=== get_random_user_agent函数测试 ===")
+    for device_type in ["desktop", "mobile", "all", "chrome", "firefox"]:
+        ua = get_random_user_agent(device_type)
+        print(f"{device_type}类型随机User-Agent: {ua[:100]}...")
+    print()
+
+
+def test_user_agents_content():
+    """测试User-Agent内容"""
+    print("=== User-Agent内容测试 ===")
+    
+    # 检查是否包含最新的浏览器版本
+    chrome_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Chrome/136" in ua)
+    firefox_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Firefox/136" in ua)
+    safari_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Version/18" in ua and "Safari" in ua)
+    
+    print(f"包含Chrome 136的User-Agent数量: {chrome_ua_count}")
+    print(f"包含Firefox 136的User-Agent数量: {firefox_ua_count}")
+    print(f"包含Safari 18的User-Agent数量: {safari_ua_count}")
+    
+    # 检查是否包含移动设备User-Agent
+    ios_ua_count = sum(1 for ua in ALL_USER_AGENTS if "iPhone" in ua or "iPad" in ua)
+    android_ua_count = sum(1 for ua in ALL_USER_AGENTS if "Android" in ua)
+    
+    print(f"包含iOS设备的User-Agent数量: {ios_ua_count}")
+    print(f"包含Android设备的User-Agent数量: {android_ua_count}")
+    print()
+
+
+def main():
+    """主测试函数"""
+    print("开始测试User-Agent列表...\n")
+    
+    test_user_agent_counts()
+    test_get_user_agents()
+    test_get_random_user_agent()
+    test_user_agents_content()
+    
+    print("所有测试完成!")
+
+
+if __name__ == "__main__":
     main()