crawlo 1.2.8__py3-none-any.whl → 1.2.9__py3-none-any.whl

tests/test_default_header_middleware.py

@@ -1,159 +1,159 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-DefaultHeaderMiddleware 测试文件
-用于测试默认请求头中间件的功能
-"""
-
-import unittest
-from unittest.mock import Mock, patch
-
-from crawlo.middleware.default_header import DefaultHeaderMiddleware
-from crawlo.exceptions import NotConfiguredError
-from crawlo.settings.setting_manager import SettingManager
-
-
-class MockLogger:
-    """Mock Logger 类，用于测试日志输出"""
-    def __init__(self, name, level=None):
-        self.name = name
-        self.level = level
-        self.logs = []
-
-    def debug(self, msg):
-        self.logs.append(('debug', msg))
-
-    def info(self, msg):
-        self.logs.append(('info', msg))
-
-    def warning(self, msg):
-        self.logs.append(('warning', msg))
-
-    def error(self, msg):
-        self.logs.append(('error', msg))
-
-
-class TestDefaultHeaderMiddleware(unittest.TestCase):
-    """DefaultHeaderMiddleware 测试类"""
-
-    def setUp(self):
-        """测试前准备"""
-        # 创建设置管理器
-        self.settings = SettingManager()
-        
-        # 创建爬虫模拟对象
-        self.crawler = Mock()
-        self.crawler.settings = self.settings
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_without_config(self, mock_get_logger):
-        """测试没有配置时中间件初始化（清除默认配置）"""
-        # 清除默认的请求头配置
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
-        self.settings.set('USER_AGENT', None)
-        self.settings.set('USER_AGENTS', [])
-        self.settings.set('RANDOM_HEADERS', {})
-        self.settings.set('LOG_LEVEL', 'INFO')
-        
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
-        
-        # 应该抛出NotConfiguredError异常
-        with self.assertRaises(NotConfiguredError):
-            DefaultHeaderMiddleware.create_instance(self.crawler)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_with_default_headers(self, mock_get_logger):
-        """测试配置默认请求头时中间件初始化"""
-        # 设置默认请求头
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
-        })
-        self.settings.set('LOG_LEVEL', 'INFO')
-        
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
-        
-        # 应该正常创建实例
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
-        self.assertIn('User-Agent', middleware.headers)
-        self.assertIn('Accept', middleware.headers)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization_with_user_agent(self, mock_get_logger):
-        """测试配置User-Agent时中间件初始化"""
-        # 清除默认的请求头配置
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
-        # 设置User-Agent
-        self.settings.set('USER_AGENT', 'Custom-Agent')
-        self.settings.set('LOG_LEVEL', 'INFO')
-        
-        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
-        
-        # 应该正常创建实例
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
-        self.assertIn('User-Agent', middleware.headers)
-        self.assertEqual(middleware.headers['User-Agent'], 'Custom-Agent')
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_process_request_with_default_headers(self, mock_get_logger):
-        """测试处理请求时添加默认请求头"""
-        # 设置默认请求头
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
-        })
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        
-        mock_logger = MockLogger('DefaultHeaderMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        
-        # 创建请求对象
-        request = Mock()
-        request.headers = {}
-        request.url = 'http://example.com'
-        
-        # 处理请求
-        middleware.process_request(request, Mock())
-        
-        # 验证请求头被添加
-        self.assertIn('User-Agent', request.headers)
-        self.assertEqual(request.headers['User-Agent'], 'Test-Agent')
-        self.assertIn('Accept', request.headers)
-        self.assertEqual(request.headers['Accept'], 'text/html')
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_process_request_without_overwriting_existing_headers(self, mock_get_logger):
-        """测试处理请求时不覆盖已存在的请求头"""
-        # 设置默认请求头
-        self.settings.set('DEFAULT_REQUEST_HEADERS', {
-            'User-Agent': 'Test-Agent',
-            'Accept': 'text/html'
-        })
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        
-        mock_logger = MockLogger('DefaultHeaderMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
-        
-        # 创建请求对象，已包含User-Agent
-        request = Mock()
-        request.headers = {'User-Agent': 'Existing-Agent'}
-        request.url = 'http://example.com'
-        
-        # 处理请求
-        middleware.process_request(request, Mock())
-        
-        # 验证已存在的请求头未被覆盖
-        self.assertEqual(request.headers['User-Agent'], 'Existing-Agent')
-        # 验证其他请求头被添加
-        self.assertIn('Accept', request.headers)
-        self.assertEqual(request.headers['Accept'], 'text/html')
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+DefaultHeaderMiddleware 测试文件
+用于测试默认请求头中间件的功能
+"""
+
+import unittest
+from unittest.mock import Mock, patch
+
+from crawlo.middleware.default_header import DefaultHeaderMiddleware
+from crawlo.exceptions import NotConfiguredError
+from crawlo.settings.setting_manager import SettingManager
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+
+class TestDefaultHeaderMiddleware(unittest.TestCase):
+    """DefaultHeaderMiddleware 测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建设置管理器
+        self.settings = SettingManager()
+        
+        # 创建爬虫模拟对象
+        self.crawler = Mock()
+        self.crawler.settings = self.settings
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_without_config(self, mock_get_logger):
+        """测试没有配置时中间件初始化（清除默认配置）"""
+        # 清除默认的请求头配置
+        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
+        self.settings.set('USER_AGENT', None)
+        self.settings.set('USER_AGENTS', [])
+        self.settings.set('RANDOM_HEADERS', {})
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
+        
+        # 应该抛出NotConfiguredError异常
+        with self.assertRaises(NotConfiguredError):
+            DefaultHeaderMiddleware.create_instance(self.crawler)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_with_default_headers(self, mock_get_logger):
+        """测试配置默认请求头时中间件初始化"""
+        # 设置默认请求头
+        self.settings.set('DEFAULT_REQUEST_HEADERS', {
+            'User-Agent': 'Test-Agent',
+            'Accept': 'text/html'
+        })
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
+        
+        # 应该正常创建实例
+        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+        self.assertIn('User-Agent', middleware.headers)
+        self.assertIn('Accept', middleware.headers)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization_with_user_agent(self, mock_get_logger):
+        """测试配置User-Agent时中间件初始化"""
+        # 清除默认的请求头配置
+        self.settings.set('DEFAULT_REQUEST_HEADERS', {})
+        # 设置User-Agent
+        self.settings.set('USER_AGENT', 'Custom-Agent')
+        self.settings.set('LOG_LEVEL', 'INFO')
+        
+        mock_get_logger.return_value = MockLogger('DefaultHeaderMiddleware')
+        
+        # 应该正常创建实例
+        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, DefaultHeaderMiddleware)
+        self.assertIn('User-Agent', middleware.headers)
+        self.assertEqual(middleware.headers['User-Agent'], 'Custom-Agent')
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_process_request_with_default_headers(self, mock_get_logger):
+        """测试处理请求时添加默认请求头"""
+        # 设置默认请求头
+        self.settings.set('DEFAULT_REQUEST_HEADERS', {
+            'User-Agent': 'Test-Agent',
+            'Accept': 'text/html'
+        })
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        
+        mock_logger = MockLogger('DefaultHeaderMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
+        
+        # 创建请求对象
+        request = Mock()
+        request.headers = {}
+        request.url = 'http://example.com'
+        
+        # 处理请求
+        middleware.process_request(request, Mock())
+        
+        # 验证请求头被添加
+        self.assertIn('User-Agent', request.headers)
+        self.assertEqual(request.headers['User-Agent'], 'Test-Agent')
+        self.assertIn('Accept', request.headers)
+        self.assertEqual(request.headers['Accept'], 'text/html')
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_process_request_without_overwriting_existing_headers(self, mock_get_logger):
+        """测试处理请求时不覆盖已存在的请求头"""
+        # 设置默认请求头
+        self.settings.set('DEFAULT_REQUEST_HEADERS', {
+            'User-Agent': 'Test-Agent',
+            'Accept': 'text/html'
+        })
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        
+        mock_logger = MockLogger('DefaultHeaderMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        middleware = DefaultHeaderMiddleware.create_instance(self.crawler)
+        
+        # 创建请求对象，已包含User-Agent
+        request = Mock()
+        request.headers = {'User-Agent': 'Existing-Agent'}
+        request.url = 'http://example.com'
+        
+        # 处理请求
+        middleware.process_request(request, Mock())
+        
+        # 验证已存在的请求头未被覆盖
+        self.assertEqual(request.headers['User-Agent'], 'Existing-Agent')
+        # 验证其他请求头被添加
+        self.assertIn('Accept', request.headers)
+        self.assertEqual(request.headers['Accept'], 'text/html')
+
+
+if __name__ == '__main__':
     unittest.main()

tests/test_distributed.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+分布式采集测试脚本
+用于验证分布式采集功能是否正常工作
+"""
+
+import asyncio
+import sys
+import os
+import time
+
+# 添加项目根目录到 Python 路径
+project_root = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, project_root)
+
+# 切换到项目根目录
+os.chdir(project_root)
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.log import get_logger
+
+logger = get_logger(__name__)
+
+
+async def test_distributed_crawling():
+    """测试分布式采集功能"""
+    logger.info("开始测试分布式采集功能...")
+
+    # 创建爬虫进程并应用配置
+    try:
+        # 确保 spider 模块被正确导入
+        spider_modules = ['ofweek_distributed.spiders']
+        process = CrawlerProcess(spider_modules=spider_modules)
+        logger.info("爬虫进程初始化成功")
+
+        # 运行指定的爬虫，使用正确的爬虫名称
+        result = await process.crawl('of_week_distributed')
+        logger.info(f"爬虫运行完成，结果: {result}")
+
+    except Exception as e:
+        logger.error(f"运行失败: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+def main():
+    """主函数"""
+    start_time = time.time()
+    logger.info("开始分布式采集测试")
+
+    try:
+        asyncio.run(test_distributed_crawling())
+        end_time = time.time()
+        logger.info(f"分布式采集测试完成，耗时: {end_time - start_time:.2f} 秒")
+    except KeyboardInterrupt:
+        logger.info("用户中断测试")
+    except Exception as e:
+        logger.error(f"测试过程中发生错误: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()