crawlo 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

tests/test_default_header_middleware.py

@@ -2,15 +2,15 @@
 # -*- coding:UTF-8 -*-
 """
 DefaultHeaderMiddleware 测试文件
-用于测试默认请求头中间件的功能
+用于测试默认请求头中间件的功能，包括随机更换header功能
 """
 
 import unittest
 from unittest.mock import Mock, patch
 
 from crawlo.middleware.default_header import DefaultHeaderMiddleware
-from crawlo.exceptions import NotConfiguredError
 from crawlo.settings.setting_manager import SettingManager
+from crawlo.exceptions import NotConfiguredError
 
 
 class MockLogger:
@@ -32,6 +32,9 @@ class MockLogger:
     def error(self, msg):
         self.logs.append(('error', msg))
 
+    def isEnabledFor(self, level):
+        return True
+
 
 class TestDefaultHeaderMiddleware(unittest.TestCase):
     """DefaultHeaderMiddleware 测试类"""
@@ -40,119 +43,271 @@ class TestDefaultHeaderMiddleware(unittest.TestCase):
         """测试前准备"""
         # 创建设置管理器
         self.settings = SettingManager()
+
+    def test_middleware_initialization_without_config(self):
+        """测试没有配置时中间件初始化"""
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 创建爬虫模拟对象
-        self.crawler = Mock()
-        self.crawler.settings = self.settings
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_without_config(self, mock_get_logger):
-        """测试没有配置时中间件初始化（清除默认配置）"""
-        # 清除默认的请求头配置
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
-        self.settings.set('USER_AGENT', None)
-        self.settings.set('USER_AGENTS', [])
-        self.settings.set('RANDOM_HEADERS', {})
-        self.settings.set('LOG_LEVEL', 'INFO')
-        
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
-        
-        # 应该抛出NotConfiguredError异常
-        with self.assertRaises(NotConfiguredError):
-            DefaultHeaderMiddleware.create_instance(self.crawler)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_with_default_headers(self, mock_get_logger):
-        """测试配置默认请求头时中间件初始化"""
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            # 应该抛出NotConfiguredError异常
+            with self.assertRaises(NotConfiguredError) as context:
+                DefaultHeaderMiddleware.create_instance(crawler)
+            
+            self.assertIn("未配置DEFAULT_REQUEST_HEADERS、USER_AGENT或随机头部配置，DefaultHeaderMiddleware已禁用", str(context.exception))
+
+    def test_middleware_initialization_with_default_headers(self):
+        """测试使用默认请求头配置时中间件初始化"""
         # 设置默认请求头
         self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
+            'Accept-Encoding': 'gzip, deflate',
         })
-        self.settings.set('LOG_LEVEL', 'INFO')
+        self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 应该正常创建实例
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
-        self.assertIn('User-Agent', middleware.headers)
-        self.assertIn('Accept', middleware.headers)
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+            self.assertEqual(len(middleware.headers), 3)
+            self.assertIn('Accept', middleware.headers)
+            self.assertIn('Accept-Language', middleware.headers)
+            self.assertIn('Accept-Encoding', middleware.headers)
 
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_with_user_agent(self, mock_get_logger):
-        """测试配置User-Agent时中间件初始化"""
-        # 清除默认的请求头配置
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
+    def test_middleware_initialization_with_user_agent(self):
+        """测试使用User-Agent配置时中间件初始化"""
         # 设置User-Agent
-        self.settings.set('USER_AGENT', 'Custom-Agent')
-        self.settings.set('LOG_LEVEL', 'INFO')
+        self.settings.set('USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
+        self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 应该正常创建实例
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
-        self.assertIn('User-Agent', middleware.headers)
-        self.assertEqual(middleware.headers['User-Agent'], 'Custom-Agent')
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            # 应该正常创建实例
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+            self.assertIn('User-Agent', middleware.headers)
+            self.assertEqual(middleware.headers['User-Agent'], 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
 
-    @patch('crawlo.utils.log.get_logger')
-    def test_process_request_with_default_headers(self, mock_get_logger):
+    def test_middleware_initialization_with_random_user_agent_enabled(self):
+        """测试启用随机User-Agent时中间件初始化"""
+        # 启用随机User-Agent并提供一个User-Agent
+        self.settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        self.settings.set('USER_AGENTS', ['Test-Agent/1.0'])  # 提供一个User-Agent以通过初始化检查
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            # 应该正常创建实例，使用内置User-Agent列表
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+            self.assertTrue(middleware.random_user_agent_enabled)
+            # 注意：这里user_agents会被get_user_agents覆盖，所以长度可能不为1
+
+    def test_middleware_initialization_with_custom_user_agents(self):
+        """测试使用自定义User-Agent列表时中间件初始化"""
+        # 设置自定义User-Agent列表
+        custom_user_agents = [
+            'Custom-Agent/1.0',
+            'Custom-Agent/2.0',
+            'Custom-Agent/3.0'
+        ]
+        self.settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        self.settings.set('USER_AGENTS', custom_user_agents)
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            # 应该正常创建实例，使用自定义User-Agent列表
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+            self.assertTrue(middleware.random_user_agent_enabled)
+            self.assertEqual(middleware.user_agents, custom_user_agents)
+
+    def test_process_request_with_default_headers(self):
         """测试处理请求时添加默认请求头"""
         # 设置默认请求头
         self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
         })
         self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        mock_logger = MockLogger('DefaultHeaderMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        
-        # 创建请求对象
-        request = Mock()
-        request.headers = {}
-        request.url = 'http://example.com'
-        
-        # 处理请求
-        middleware.process_request(request, Mock())
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 验证请求头被添加
-        self.assertIn('User-Agent', request.headers)
-        self.assertEqual(request.headers['User-Agent'], 'Test-Agent')
-        self.assertIn('Accept', request.headers)
-        self.assertEqual(request.headers['Accept'], 'text/html')
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 创建请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            
+            # 处理请求
+            middleware.process_request(request, spider)
+            
+            # 检查默认请求头是否添加
+            self.assertIn('Accept', request.headers)
+            self.assertIn('Accept-Language', request.headers)
+            self.assertEqual(request.headers['Accept'], 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
+            self.assertEqual(request.headers['Accept-Language'], 'en-US,en;q=0.5')
 
-    @patch('crawlo.utils.log.get_logger')
-    def test_process_request_without_overwriting_existing_headers(self, mock_get_logger):
-        """测试处理请求时不覆盖已存在的请求头"""
+    def test_process_request_with_existing_headers(self):
+        """测试处理已有请求头的请求"""
         # 设置默认请求头
         self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'Accept-Language': 'en-US,en;q=0.5',
         })
         self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        mock_logger = MockLogger('DefaultHeaderMiddleware')
-        mock_get_logger.return_value = mock_logger
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 创建已有请求头的请求
+            request = Mock()
+            request.headers = {
+                'Accept': 'application/json',  # 已存在的请求头
+            }
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            
+            # 处理请求
+            middleware.process_request(request, spider)
+            
+            # 检查已存在的请求头不被覆盖，新请求头被添加
+            self.assertEqual(request.headers['Accept'], 'application/json')  # 保持原值
+            self.assertIn('Accept-Language', request.headers)  # 新添加的请求头
+
+    def test_process_request_with_random_user_agent(self):
+        """测试处理请求时添加随机User-Agent"""
+        # 启用随机User-Agent并设置自定义列表
+        custom_user_agents = [
+            'Custom-Agent/1.0',
+            'Custom-Agent/2.0',
+            'Custom-Agent/3.0'
+        ]
+        self.settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        self.settings.set('USER_AGENTS', custom_user_agents)
+        self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 创建请求对象，已包含User-Agent
-        request = Mock()
-        request.headers = {'User-Agent': 'Existing-Agent'}
-        request.url = 'http://example.com'
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 创建没有User-Agent的请求
+            request = Mock()
+            request.headers = {}
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            
+            # 处理请求
+            middleware.process_request(request, spider)
+            
+            # 检查随机User-Agent是否添加
+            self.assertIn('User-Agent', request.headers)
+            self.assertIn(request.headers['User-Agent'], custom_user_agents)
+
+    def test_process_request_with_existing_user_agent(self):
+        """测试处理已有User-Agent的请求"""
+        # 启用随机User-Agent并设置自定义列表
+        custom_user_agents = [
+            'Custom-Agent/1.0',
+            'Custom-Agent/2.0',
+            'Custom-Agent/3.0'
+        ]
+        self.settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        self.settings.set('USER_AGENTS', custom_user_agents)
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
+        
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 创建已有User-Agent的请求
+            existing_ua = 'Existing-Agent/1.0'
+            request = Mock()
+            request.headers = {
+                'User-Agent': existing_ua,
+            }
+            request.url = 'https://example.com'
+            
+            spider = Mock()
+            
+            # 处理请求
+            middleware.process_request(request, spider)
+            
+            # 检查已存在的User-Agent不被覆盖
+            self.assertEqual(request.headers['User-Agent'], existing_ua)
+
+    def test_get_random_user_agent(self):
+        """测试获取随机User-Agent功能"""
+        # 设置自定义User-Agent列表
+        custom_user_agents = [
+            'Custom-Agent/1.0',
+            'Custom-Agent/2.0',
+            'Custom-Agent/3.0'
+        ]
+        self.settings.set('RANDOM_USER_AGENT_ENABLED', True)
+        self.settings.set('USER_AGENTS', custom_user_agents)
+        self.settings.set('LOG_LEVEL', 'DEBUG')
         
-        # 处理请求
-        middleware.process_request(request, Mock())
+        # 创建一个模拟的crawler对象
+        crawler = Mock()
+        crawler.settings = self.settings
         
-        # 验证已存在的请求头未被覆盖
-        self.assertEqual(request.headers['User-Agent'], 'Existing-Agent')
-        # 验证其他请求头被添加
-        self.assertIn('Accept', request.headers)
-        self.assertEqual(request.headers['Accept'], 'text/html')
+        logger = MockLogger('DefaultHeaderMiddleware')
+        with patch('crawlo.middleware.default_header.get_logger', return_value=logger):
+            middleware = DefaultHeaderMiddleware.create_instance(crawler)
+            
+            # 获取随机User-Agent
+            random_ua = middleware._get_random_user_agent()
+            
+            # 检查返回的User-Agent在列表中
+            self.assertIn(random_ua, custom_user_agents)
 
 
 if __name__ == '__main__':

tests/test_fingerprint_consistency.py

@@ -0,0 +1,136 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+指纹一致性测试
+==============
+验证框架中各组件对相同数据生成一致的指纹
+"""
+
+import unittest
+from unittest.mock import Mock
+
+from crawlo import Item
+from crawlo.pipelines.memory_dedup_pipeline import MemoryDedupPipeline
+from crawlo.pipelines.redis_dedup_pipeline import RedisDedupPipeline
+from crawlo.pipelines.bloom_dedup_pipeline import BloomDedupPipeline
+from crawlo.pipelines.database_dedup_pipeline import DatabaseDedupPipeline
+from crawlo.tools.distributed_coordinator import DeduplicationTool
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+class TestItem(Item):
+    """测试用数据项类"""
+    
+    def __init__(self, **kwargs):
+        super().__init__()
+        for key, value in kwargs.items():
+            setattr(self, key, value)
+    
+    def to_dict(self):
+        """转换为字典"""
+        return {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
+
+
+class FingerprintConsistencyTest(unittest.TestCase):
+    """指纹一致性测试"""
+    
+    def setUp(self):
+        """测试初始化"""
+        # 创建测试数据项
+        self.test_item = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        # 创建各去重管道实例
+        self.memory_pipeline = MemoryDedupPipeline()
+        self.redis_pipeline = RedisDedupPipeline(
+            redis_host='localhost',
+            redis_port=6379,
+            redis_db=0,
+            redis_key='test:fingerprints'
+        )
+        self.bloom_pipeline = BloomDedupPipeline()
+        self.database_pipeline = DatabaseDedupPipeline()
+        
+        # 创建去重工具实例
+        self.dedup_tool = DeduplicationTool()
+    
+    def test_item_fingerprint_consistency(self):
+        """测试数据项指纹一致性"""
+        # 使用各管道生成指纹
+        memory_fingerprint = self.memory_pipeline._generate_item_fingerprint(self.test_item)
+        redis_fingerprint = self.redis_pipeline._generate_item_fingerprint(self.test_item)
+        bloom_fingerprint = self.bloom_pipeline._generate_item_fingerprint(self.test_item)
+        database_fingerprint = self.database_pipeline._generate_item_fingerprint(self.test_item)
+        
+        # 使用指纹生成器直接生成指纹
+        direct_fingerprint = FingerprintGenerator.item_fingerprint(self.test_item)
+        
+        # 验证所有指纹一致
+        self.assertEqual(memory_fingerprint, redis_fingerprint)
+        self.assertEqual(memory_fingerprint, bloom_fingerprint)
+        self.assertEqual(memory_fingerprint, database_fingerprint)
+        self.assertEqual(memory_fingerprint, direct_fingerprint)
+        
+        print(f"Memory Pipeline Fingerprint: {memory_fingerprint}")
+        print(f"Redis Pipeline Fingerprint: {redis_fingerprint}")
+        print(f"Bloom Pipeline Fingerprint: {bloom_fingerprint}")
+        print(f"Database Pipeline Fingerprint: {database_fingerprint}")
+        print(f"Direct Fingerprint: {direct_fingerprint}")
+    
+    def test_data_fingerprint_consistency(self):
+        """测试通用数据指纹一致性"""
+        # 测试字典数据
+        test_data = {
+            "name": "test",
+            "value": 123,
+            "nested": {
+                "inner": "value"
+            }
+        }
+        
+        # 使用去重工具生成指纹
+        tool_fingerprint = self.dedup_tool.generate_fingerprint(test_data)
+        
+        # 使用指纹生成器生成指纹
+        generator_fingerprint = FingerprintGenerator.data_fingerprint(test_data)
+        
+        # 验证指纹一致
+        self.assertEqual(tool_fingerprint, generator_fingerprint)
+        
+        print(f"DeduplicationTool Fingerprint: {tool_fingerprint}")
+        print(f"FingerprintGenerator Fingerprint: {generator_fingerprint}")
+    
+    def test_fingerprint_stability(self):
+        """测试指纹稳定性"""
+        # 创建相同的测试数据项多次
+        item1 = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        item2 = TestItem(
+            title="Test Title",
+            url="https://example.com",
+            content="Test content",
+            price=99.99
+        )
+        
+        # 生成指纹
+        fingerprint1 = FingerprintGenerator.item_fingerprint(item1)
+        fingerprint2 = FingerprintGenerator.item_fingerprint(item2)
+        
+        # 验证相同数据生成相同指纹
+        self.assertEqual(fingerprint1, fingerprint2)
+        
+        print(f"First fingerprint: {fingerprint1}")
+        print(f"Second fingerprint: {fingerprint2}")
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_fingerprint_simple.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+简化版指纹一致性测试
+==============
+验证框架中各组件对相同数据生成一致的指纹
+"""
+
+import sys
+import os
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+def test_fingerprint_consistency():
+    """测试指纹一致性"""
+    # 测试数据
+    test_data = {
+        "title": "Test Title",
+        "url": "https://example.com",
+        "content": "Test content",
+        "price": 99.99
+    }
+    
+    # 使用指纹生成器生成指纹
+    fingerprint1 = FingerprintGenerator.data_fingerprint(test_data)
+    fingerprint2 = FingerprintGenerator.data_fingerprint(test_data)
+    
+    # 验证相同数据生成相同指纹
+    print(f"First fingerprint: {fingerprint1}")
+    print(f"Second fingerprint: {fingerprint2}")
+    print(f"指纹一致: {fingerprint1 == fingerprint2}")
+    
+    # 测试请求指纹
+    method = "GET"
+    url = "https://example.com"
+    body = b""
+    headers = {"User-Agent": "test-agent"}
+    
+    request_fingerprint1 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
+    request_fingerprint2 = FingerprintGenerator.request_fingerprint(method, url, body, headers)
+    
+    print(f"\nRequest fingerprint 1: {request_fingerprint1}")
+    print(f"Request fingerprint 2: {request_fingerprint2}")
+    print(f"请求指纹一致: {request_fingerprint1 == request_fingerprint2}")
+
+
+if __name__ == '__main__':
+    test_fingerprint_consistency()

tests/test_framework_logger.py

@@ -7,7 +7,7 @@ import sys
 import os
 sys.path.insert(0, '/')
 
-from crawlo.core.framework_initializer import initialize_framework, get_framework_initializer
+from crawlo.initialization import initialize_framework, get_framework_initializer
 from crawlo.utils.log import get_logger, LoggerManager
 
 def test_framework_logger():

tests/test_framework_startup.py

@@ -24,7 +24,7 @@ def test_framework_startup():
     }
     
     # 初始化框架
-    from crawlo.core.framework_initializer import initialize_framework
+    from crawlo.initialization import initialize_framework
     settings = initialize_framework(test_settings)
     
     print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")