crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/test_random_headers_default.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试默认headers中间件的随机headers功能
+确认默认是否要启动随机headers
+"""
+
+import sys
+import os
+import random
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.exceptions import NotConfiguredError
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+    def isEnabledFor(self, level):
+        return True
+
+
+def test_default_configuration():
+    """测试默认配置下中间件的行为"""
+    print("=== 测试默认配置下中间件的行为 ===")
+    
+    # 创建设置管理器（使用默认配置）
+    settings = SettingManager()
+    # 不设置任何RANDOM相关的配置，使用默认值
+    # 但需要移除默认的DEFAULT_REQUEST_HEADERS和USER_AGENT来测试禁用情况
+    settings.set('DEFAULT_REQUEST_HEADERS', {})
+    settings.set('USER_AGENT', None)
+    
+    # 创建一个模拟的crawler对象
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            # 尝试创建中间件实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ❌ 中间件创建成功，但应该在默认配置下被禁用")
+            return False
+        except NotConfiguredError as e:
+            print("  ✅ 中间件正确地在默认配置下被禁用")
+            print(f"     错误信息: {e}")
+            return True
+        except Exception as e:
+            print(f"  ❌ 发生意外错误: {e}")
+            return False
+
+
+def test_default_headers_only():
+    """测试仅配置默认请求头时的行为"""
+    print("\n=== 测试仅配置默认请求头时的行为 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+    })
+    settings.set('LOG_LEVEL', 'DEBUG')
+    # 确保随机功能禁用
+    settings.set('RANDOMNESS', False)
+    
+    # 创建一个模拟的crawler对象
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ✅ 仅配置默认请求头时中间件创建成功")
+            
+            # 检查配置
+            print(f"     默认请求头数量: {len(middleware.headers)}")
+            print(f"     User-Agent: {middleware.user_agent}")
+            print(f"     随机User-Agent启用: {middleware.random_user_agent_enabled}")
+            print(f"     随机请求头数量: {len(middleware.random_headers)}")
+            print(f"     随机功能启用: {middleware.randomness}")
+            
+            # 测试处理请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            middleware.process_request(request, spider)
+            
+            # 检查默认请求头是否添加
+            if 'Accept' in request.headers and 'Accept-Language' in request.headers:
+                print("  ✅ 默认请求头正确添加到请求中")
+            else:
+                print("  ❌ 默认请求头未正确添加")
+                return False
+            
+            return True
+            
+        except Exception as e:
+            print(f"  ❌ 测试失败: {e}")
+            return False
+
+
+def test_random_user_agent_default():
+    """测试随机User-Agent的默认行为"""
+    print("\n=== 测试随机User-Agent的默认行为 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings.set('RANDOM_USER_AGENT_ENABLED', True)  # 启用随机User-Agent
+    settings.set('LOG_LEVEL', 'DEBUG')
+    # 确保随机功能启用
+    settings.set('RANDOMNESS', True)
+    
+    # 创建一个模拟的crawler对象
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ✅ 启用随机User-Agent时中间件创建成功")
+            
+            # 检查配置
+            print(f"     随机User-Agent启用: {middleware.random_user_agent_enabled}")
+            print(f"     User-Agent列表数量: {len(middleware.user_agents)}")
+            print(f"     User-Agent设备类型: {middleware.user_agent_device_type}")
+            
+            # 测试获取随机User-Agent
+            print("     随机User-Agent测试:")
+            for i in range(5):
+                random_ua = middleware._get_random_user_agent()
+                print(f"       {i+1}. {random_ua[:50]}...")
+            
+            # 测试处理请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            middleware.process_request(request, spider)
+            
+            # 检查User-Agent是否添加
+            if 'User-Agent' in request.headers:
+                print("  ✅ 随机User-Agent正确添加到请求中")
+                print(f"     User-Agent: {request.headers['User-Agent'][:50]}...")
+            else:
+                print("  ❌ 随机User-Agent未添加")
+                return False
+            
+            return True
+            
+        except Exception as e:
+            print(f"  ❌ 测试失败: {e}")
+            return False
+
+
+def test_random_headers_default():
+    """测试随机请求头的默认行为"""
+    print("\n=== 测试随机请求头的默认行为 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings.set('RANDOM_HEADERS', {
+        'X-Custom-Header': ['Value1', 'Value2', 'Value3'],
+        'X-Another-Header': 'FixedValue'
+    })
+    settings.set('RANDOMNESS', True)  # 启用随机功能
+    settings.set('LOG_LEVEL', 'DEBUG')
+    
+    # 创建一个模拟的crawler对象
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ✅ 启用随机请求头时中间件创建成功")
+            
+            # 检查配置
+            print(f"     随机功能启用: {middleware.randomness}")
+            print(f"     随机请求头数量: {len(middleware.random_headers)}")
+            
+            # 测试处理请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            middleware.process_request(request, spider)
+            
+            # 检查随机请求头是否添加
+            if 'X-Custom-Header' in request.headers or 'X-Another-Header' in request.headers:
+                print("  ✅ 随机请求头已添加到请求中")
+                print(f"     X-Custom-Header: {request.headers.get('X-Custom-Header', '未设置')}")
+                print(f"     X-Another-Header: {request.headers.get('X-Another-Header', '未设置')}")
+            else:
+                print("  ❌ 随机请求头未添加")
+                return False
+            
+            # 测试多次请求的随机性
+            print("     随机性测试:")
+            custom_header_values = []
+            for i in range(10):
+                test_request = Mock()
+                test_request.headers = {}
+                test_request.url = f'https://example.com/test{i}'
+                
+                middleware.process_request(test_request, spider)
+                if 'X-Custom-Header' in test_request.headers:
+                    custom_header_values.append(test_request.headers['X-Custom-Header'])
+            
+            # 检查是否有不同的值（应该有随机性）
+            unique_values = set(custom_header_values)
+            print(f"       10次请求中X-Custom-Header的不同值: {list(unique_values)}")
+            if len(unique_values) > 1:
+                print("  ✅ 随机请求头具有随机性")
+            else:
+                print("  ⚠️  随机请求头可能缺乏随机性")
+            
+            return True
+            
+        except Exception as e:
+            print(f"  ❌ 测试失败: {e}")
+            return False
+
+
+def test_recommendation():
+    """测试推荐配置"""
+    print("\n=== 推荐配置测试 ===")
+    
+    print("默认配置分析:")
+    print("  1. DEFAULT_REQUEST_HEADERS: 已配置（默认请求头）")
+    print("  2. USER_AGENT: 已配置（默认User-Agent）")
+    print("  3. RANDOM_USER_AGENT_ENABLED: False（默认禁用）")
+    print("  4. RANDOMNESS: True（默认启用，用于随机延迟）")
+    print("  5. RANDOM_HEADERS: {}（默认空字典）")
+    
+    print("\n推荐配置:")
+    print("  对于大多数爬虫场景，建议:")
+    print("    - 保持默认请求头（提供基本的浏览器兼容性）")
+    print("    - 保持默认User-Agent（模拟现代浏览器）")
+    print("    - 根据需要启用随机User-Agent（提高反爬虫能力）")
+    print("    - 根据需要启用随机请求头（进一步提高反爬虫能力）")
+    
+    print("\n是否启用随机headers的建议:")
+    print("  默认情况下不启用随机headers，原因:")
+    print("    1. 保持请求的一致性，便于调试和问题排查")
+    print("    2. 避免不必要的随机性导致的不可预测行为")
+    print("    3. 用户可以根据具体需求选择是否启用")
+    print("    4. 降低系统开销（随机选择需要额外计算）")
+    
+    print("\n注意:")
+    print("  RANDOMNESS默认为True，主要用于下载延迟的随机化")
+    print("  随机headers功能需要显式配置RANDOM_HEADERS和启用RANDOMNESS")
+    
+    return True
+
+
+def main():
+    print("开始测试默认headers中间件的随机headers功能...")
+    
+    try:
+        # 运行所有测试
+        test1_result = test_default_configuration()
+        test2_result = test_default_headers_only()
+        test3_result = test_random_user_agent_default()
+        test4_result = test_random_headers_default()
+        test5_result = test_recommendation()
+        
+        if test1_result and test2_result and test3_result and test4_result:
+            print("\n🎉 所有测试通过！")
+            print("\n结论:")
+            print("  1. 默认情况下，随机headers功能是禁用的")
+            print("  2. 只有在显式配置启用时，随机headers功能才会启动")
+            print("  3. 这种设计是合理的，符合用户偏好")
+            print("  4. RANDOMNESS默认为True，主要用于下载延迟随机化")
+        else:
+            print("\n❌ 部分测试失败，请检查实现")
+        
+    except Exception as e:
+        print(f"\n❌ 测试过程中发生错误: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()

tests/test_random_headers_necessity.py

@@ -0,0 +1,309 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试RANDOM_HEADERS参数的必要性
+验证是否可以仅使用现有的User-Agent功能满足需求
+"""
+
+import sys
+import os
+import random
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.exceptions import NotConfiguredError
+from crawlo.data.user_agents import get_random_user_agent
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+    def isEnabledFor(self, level):
+        return True
+
+
+def test_current_user_agent_functionality():
+    """测试当前User-Agent功能是否足够"""
+    print("=== 测试当前User-Agent功能是否足够 ===")
+    
+    # 创建设置管理器
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings.set('RANDOM_USER_AGENT_ENABLED', True)  # 启用随机User-Agent
+    settings.set('LOG_LEVEL', 'DEBUG')
+    settings.set('RANDOMNESS', True)  # 启用随机功能
+    
+    # 创建一个模拟的crawler对象
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ✅ 启用随机User-Agent时中间件创建成功")
+            
+            # 检查配置
+            print(f"     随机User-Agent启用: {middleware.random_user_agent_enabled}")
+            print(f"     User-Agent列表数量: {len(middleware.user_agents)}")
+            print(f"     User-Agent设备类型: {middleware.user_agent_device_type}")
+            
+            # 测试处理请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            middleware.process_request(request, spider)
+            
+            # 检查User-Agent是否添加
+            if 'User-Agent' in request.headers:
+                print("  ✅ 随机User-Agent正确添加到请求中")
+                print(f"     User-Agent: {request.headers['User-Agent'][:50]}...")
+                return True
+            else:
+                print("  ❌ 随机User-Agent未添加")
+                return False
+            
+        except Exception as e:
+            print(f"  ❌ 测试失败: {e}")
+            return False
+
+
+def test_random_headers_vs_user_agent():
+    """比较RANDOM_HEADERS和User-Agent功能的差异"""
+    print("\n=== 比较RANDOM_HEADERS和User-Agent功能的差异 ===")
+    
+    # 测试RANDOM_HEADERS功能
+    print("  RANDOM_HEADERS功能:")
+    settings1 = SettingManager()
+    settings1.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings1.set('RANDOM_HEADERS', {
+        'X-Custom-Header': ['Value1', 'Value2', 'Value3'],
+        'X-Another-Header': 'FixedValue',
+        'X-Random-Header': ['A', 'B', 'C', 'D']
+    })
+    settings1.set('RANDOMNESS', True)
+    settings1.set('LOG_LEVEL', 'DEBUG')
+    
+    # 测试User-Agent功能
+    print("  User-Agent功能:")
+    settings2 = SettingManager()
+    settings2.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings2.set('RANDOM_USER_AGENT_ENABLED', True)
+    settings2.set('LOG_LEVEL', 'DEBUG')
+    settings2.set('RANDOMNESS', True)
+    
+    # 创建crawler对象
+    crawler1 = Mock()
+    crawler1.settings = settings1
+    crawler2 = Mock()
+    crawler2.settings = settings2
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    
+    # 测试RANDOM_HEADERS
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            middleware1 = DefaultHeaderMiddleware.create_instance(crawler1)
+            
+            # 测试多次请求的随机性
+            print("    RANDOM_HEADERS随机性测试:")
+            header_values = {}
+            for i in range(20):
+                test_request = Mock()
+                test_request.headers = {}
+                test_request.url = f'https://example.com/test{i}'
+                
+                middleware1.process_request(test_request, Mock())
+                
+                # 收集各种随机头部的值
+                for header in ['X-Custom-Header', 'X-Another-Header', 'X-Random-Header']:
+                    if header in test_request.headers:
+                        if header not in header_values:
+                            header_values[header] = []
+                        header_values[header].append(test_request.headers[header])
+            
+            # 分析随机性
+            for header, values in header_values.items():
+                unique_values = set(values)
+                print(f"      {header}: {len(unique_values)} 个不同值 ({list(unique_values)[:3]}...)")
+            
+        except Exception as e:
+            print(f"    RANDOM_HEADERS测试失败: {e}")
+    
+    # 测试User-Agent
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            middleware2 = DefaultHeaderMiddleware.create_instance(crawler2)
+            
+            # 测试多次请求的随机性
+            print("    User-Agent随机性测试:")
+            ua_values = []
+            for i in range(20):
+                test_request = Mock()
+                test_request.headers = {}
+                test_request.url = f'https://example.com/test{i}'
+                
+                middleware2.process_request(test_request, Mock())
+                
+                if 'User-Agent' in test_request.headers:
+                    ua_values.append(test_request.headers['User-Agent'])
+            
+            # 分析随机性
+            unique_uas = set(ua_values)
+            print(f"      User-Agent: {len(unique_uas)} 个不同值")
+            print(f"      示例: {list(unique_uas)[:3]}")
+            
+        except Exception as e:
+            print(f"    User-Agent测试失败: {e}")
+
+
+def test_direct_user_agent_usage():
+    """测试直接使用user_agents模块的功能"""
+    print("\n=== 测试直接使用user_agents模块的功能 ===")
+    
+    # 测试get_random_user_agent函数
+    print("  直接使用get_random_user_agent函数:")
+    for i in range(5):
+        ua = get_random_user_agent()
+        print(f"    {i+1}. {ua[:50]}...")
+    
+    # 测试不同设备类型的User-Agent
+    print("  不同设备类型的User-Agent:")
+    device_types = ["desktop", "mobile", "chrome", "firefox", "safari"]
+    for device_type in device_types:
+        ua = get_random_user_agent(device_type)
+        print(f"    {device_type}: {ua[:50]}...")
+    
+    print("  ✅ 可以直接使用user_agents模块满足User-Agent随机化需求")
+
+
+def test_alternative_approach():
+    """测试替代方案：仅使用User-Agent功能"""
+    print("\n=== 测试替代方案：仅使用User-Agent功能 ===")
+    
+    print("  推荐的配置方式:")
+    print("    1. 启用RANDOM_USER_AGENT_ENABLED = True")
+    print("    2. 设置USER_AGENT_DEVICE_TYPE = 'desktop' 或 'mobile' 等")
+    print("    3. 无需配置RANDOM_HEADERS")
+    
+    # 模拟推荐配置
+    settings = SettingManager()
+    settings.set('DEFAULT_REQUEST_HEADERS', {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    })
+    settings.set('RANDOM_USER_AGENT_ENABLED', True)
+    settings.set('USER_AGENT_DEVICE_TYPE', 'desktop')
+    settings.set('LOG_LEVEL', 'DEBUG')
+    
+    crawler = Mock()
+    crawler.settings = settings
+    
+    logger = MockLogger('DefaultHeaderMiddleware')
+    with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+        try:
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            print("  ✅ 推荐配置可行")
+            
+            # 测试处理请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            middleware.process_request(request, spider)
+            
+            if 'User-Agent' in request.headers:
+                print(f"     User-Agent: {request.headers['User-Agent'][:50]}...")
+            
+            return True
+        except Exception as e:
+            print(f"  ❌ 推荐配置失败: {e}")
+            return False
+
+
+def analyze_necessity():
+    """分析RANDOM_HEADERS参数的必要性"""
+    print("\n=== 分析RANDOM_HEADERS参数的必要性 ===")
+    
+    print("功能对比:")
+    print("  User-Agent功能:")
+    print("    ✓ 专门用于User-Agent随机化")
+    print("    ✓ 内置大量真实User-Agent")
+    print("    ✓ 支持按设备类型分类")
+    print("    ✓ 易于使用和配置")
+    
+    print("  RANDOM_HEADERS功能:")
+    print("    ✓ 可以为任意头部添加随机值")
+    print("    ✓ 更加灵活，支持自定义头部")
+    print("    ✓ 适用于需要随机化其他头部的场景")
+    print("    ✗ 需要用户自己提供头部值列表")
+    
+    print("\n使用建议:")
+    print("  1. 对于User-Agent随机化：使用RANDOM_USER_AGENT_ENABLED")
+    print("  2. 对于其他头部随机化：使用RANDOM_HEADERS")
+    print("  3. 大多数场景下，User-Agent功能已足够")
+    print("  4. RANDOM_HEADERS适用于特殊需求场景")
+    
+    print("\n结论:")
+    print("  RANDOM_HEADERS参数不是必需的，但对于需要随机化其他头部的场景很有用")
+    print("  现有的User-Agent功能已经可以满足大多数反爬虫需求")
+
+
+def main():
+    print("开始测试RANDOM_HEADERS参数的必要性...")
+    
+    try:
+        # 运行所有测试
+        test1_result = test_current_user_agent_functionality()
+        test_random_headers_vs_user_agent()
+        test_direct_user_agent_usage()
+        test2_result = test_alternative_approach()
+        analyze_necessity()
+        
+        if test1_result and test2_result:
+            print("\n🎉 测试完成！")
+            print("\n总结:")
+            print("  1. 现有的User-Agent功能已能满足大多数随机化需求")
+            print("  2. RANDOM_HEADERS参数提供了额外的灵活性")
+            print("  3. 对于简单场景，仅使用User-Agent功能即可")
+            print("  4. 对于复杂场景，RANDOM_HEADERS参数仍然有价值")
+        else:
+            print("\n❌ 部分测试失败")
+        
+    except Exception as e:
+        print(f"\n❌ 测试过程中发生错误: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    main()