crawlo 1.2.7__py3-none-any.whl → 1.2.9__py3-none-any.whl

tests/test_request_ignore_middleware.py

@@ -1,183 +1,183 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-RequestIgnoreMiddleware 测试文件
-用于测试请求忽略中间件的功能
-"""
-
-import asyncio
-import unittest
-from unittest.mock import Mock, patch
-
-from crawlo.middleware.request_ignore import RequestIgnoreMiddleware
-from crawlo.exceptions import IgnoreRequestError
-from crawlo.settings.setting_manager import SettingManager
-
-
-class MockLogger:
-    """Mock Logger 类，用于测试日志输出"""
-    def __init__(self, name, level=None):
-        self.name = name
-        self.level = level
-        self.logs = []
-
-    def debug(self, msg):
-        self.logs.append(('debug', msg))
-
-    def info(self, msg):
-        self.logs.append(('info', msg))
-
-    def warning(self, msg):
-        self.logs.append(('warning', msg))
-
-    def error(self, msg):
-        self.logs.append(('error', msg))
-
-
-class MockStats:
-    """Mock Stats 类，用于测试统计信息"""
-    def __init__(self):
-        self.stats = {}
-
-    def inc_value(self, key, value=1):
-        if key in self.stats:
-            self.stats[key] += value
-        else:
-            self.stats[key] = value
-
-
-class TestRequestIgnoreMiddleware(unittest.TestCase):
-    """RequestIgnoreMiddleware 测试类"""
-
-    def setUp(self):
-        """测试前准备"""
-        # 创建设置管理器
-        self.settings = SettingManager()
-        
-        # 创建爬虫模拟对象
-        self.crawler = Mock()
-        self.crawler.settings = self.settings
-        self.crawler.stats = MockStats()
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization(self, mock_get_logger):
-        """测试中间件初始化"""
-        self.settings.set('LOG_LEVEL', 'INFO')
-        mock_get_logger.return_value = MockLogger('RequestIgnoreMiddleware')
-        
-        # 应该正常创建实例
-        middleware = RequestIgnoreMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, RequestIgnoreMiddleware)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling(self, mock_get_logger):
-        """测试请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        request.url = 'http://example.com/page'
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证统计信息
-        self.assertIn('request_ignore_count', mock_stats.stats)
-        self.assertEqual(mock_stats.stats['request_ignore_count'], 1)
-        self.assertIn('request_ignore_count/reason/test reason', mock_stats.stats)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling_with_domain(self, mock_get_logger):
-        """测试带域名的请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        request.url = 'http://example.com/page'
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证域名统计信息
-        self.assertIn('request_ignore_count/domain/example.com', mock_stats.stats)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling_with_invalid_url(self, mock_get_logger):
-        """测试带无效URL的请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象（没有url属性，会触发异常）
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        # 不设置request.url属性，这样在访问时会抛出AttributeError
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证无效URL统计信息
-        self.assertIn('request_ignore_count/domain/invalid_url', mock_stats.stats)
-
-    def test_process_exception_with_ignore_request_error(self):
-        """测试处理IgnoreRequestError异常"""
-        # 创建中间件实例
-        middleware = RequestIgnoreMiddleware(
-            stats=MockStats(),
-            log_level='INFO'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        
-        # 应该返回True表示异常已被处理
-        result = middleware.process_exception(request, exc, Mock())
-        self.assertTrue(result)
-
-    def test_process_exception_with_other_exception(self):
-        """测试处理其他异常"""
-        # 创建中间件实例
-        middleware = RequestIgnoreMiddleware(
-            stats=MockStats(),
-            log_level='INFO'
-        )
-        
-        # 创建异常和请求对象
-        exc = ValueError("test error")
-        request = Mock()
-        
-        # 应该返回None表示异常未被处理
-        result = middleware.process_exception(request, exc, Mock())
-        self.assertIsNone(result)
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+RequestIgnoreMiddleware 测试文件
+用于测试请求忽略中间件的功能
+"""
+
+import asyncio
+import unittest
+from unittest.mock import Mock, patch
+
+from crawlo.middleware.request_ignore import RequestIgnoreMiddleware
+from crawlo.exceptions import IgnoreRequestError
+from crawlo.settings.setting_manager import SettingManager
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+
+class MockStats:
+    """Mock Stats 类，用于测试统计信息"""
+    def __init__(self):
+        self.stats = {}
+
+    def inc_value(self, key, value=1):
+        if key in self.stats:
+            self.stats[key] += value
+        else:
+            self.stats[key] = value
+
+
+class TestRequestIgnoreMiddleware(unittest.TestCase):
+    """RequestIgnoreMiddleware 测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建设置管理器
+        self.settings = SettingManager()
+        
+        # 创建爬虫模拟对象
+        self.crawler = Mock()
+        self.crawler.settings = self.settings
+        self.crawler.stats = MockStats()
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization(self, mock_get_logger):
+        """测试中间件初始化"""
+        self.settings.set('LOG_LEVEL', 'INFO')
+        mock_get_logger.return_value = MockLogger('RequestIgnoreMiddleware')
+        
+        # 应该正常创建实例
+        middleware = RequestIgnoreMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, RequestIgnoreMiddleware)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling(self, mock_get_logger):
+        """测试请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        request.url = 'http://example.com/page'
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证统计信息
+        self.assertIn('request_ignore_count', mock_stats.stats)
+        self.assertEqual(mock_stats.stats['request_ignore_count'], 1)
+        self.assertIn('request_ignore_count/reason/test reason', mock_stats.stats)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling_with_domain(self, mock_get_logger):
+        """测试带域名的请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        request.url = 'http://example.com/page'
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证域名统计信息
+        self.assertIn('request_ignore_count/domain/example.com', mock_stats.stats)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling_with_invalid_url(self, mock_get_logger):
+        """测试带无效URL的请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象（没有url属性，会触发异常）
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        # 不设置request.url属性，这样在访问时会抛出AttributeError
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证无效URL统计信息
+        self.assertIn('request_ignore_count/domain/invalid_url', mock_stats.stats)
+
+    def test_process_exception_with_ignore_request_error(self):
+        """测试处理IgnoreRequestError异常"""
+        # 创建中间件实例
+        middleware = RequestIgnoreMiddleware(
+            stats=MockStats(),
+            log_level='INFO'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        
+        # 应该返回True表示异常已被处理
+        result = middleware.process_exception(request, exc, Mock())
+        self.assertTrue(result)
+
+    def test_process_exception_with_other_exception(self):
+        """测试处理其他异常"""
+        # 创建中间件实例
+        middleware = RequestIgnoreMiddleware(
+            stats=MockStats(),
+            log_level='INFO'
+        )
+        
+        # 创建异常和请求对象
+        exc = ValueError("test error")
+        request = Mock()
+        
+        # 应该返回None表示异常未被处理
+        result = middleware.process_exception(request, exc, Mock())
+        self.assertIsNone(result)
+
+
+if __name__ == '__main__':
     unittest.main()

tests/test_request_params.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Request参数处理测试
+"""
+import unittest
+from crawlo.network.request import Request
+
+
+class TestRequestParams(unittest.TestCase):
+    """Request参数处理测试类"""
+
+    def test_get_params_handling(self):
+        """测试GET参数处理"""
+        # 测试GET请求带参数
+        params = {"page": 1, "limit": 10, "search": "test"}
+        request = Request(
+            url="https://api.example.com/users",
+            method="GET",
+            get_params=params
+        )
+        
+        # 验证参数是否正确附加到URL上
+        self.assertIn("https://api.example.com/users?", request.url)
+        self.assertIn("page=1", request.url)
+        self.assertIn("limit=10", request.url)
+        self.assertIn("search=test", request.url)
+        self.assertEqual(request.method, "GET")
+
+    def test_form_data_with_get(self):
+        """测试GET请求使用form_data"""
+        # 测试GET请求使用form_data参数
+        form_data = {"category": "books", "sort": "name"}
+        request = Request(
+            url="https://api.example.com/products",
+            method="GET",
+            form_data=form_data
+        )
+        
+        # 验证form_data是否正确作为GET参数附加到URL上
+        self.assertIn("https://api.example.com/products?", request.url)
+        self.assertIn("category=books", request.url)
+        self.assertIn("sort=name", request.url)
+        self.assertEqual(request.method, "GET")
+
+    def test_form_data_with_post(self):
+        """测试POST请求使用form_data"""
+        # 测试POST请求使用form_data参数
+        form_data = {"username": "testuser", "password": "testpass"}
+        request = Request(
+            url="https://api.example.com/login",
+            method="POST",
+            form_data=form_data
+        )
+        
+        # 验证form_data是否作为请求体发送
+        self.assertEqual(request.method, "POST")
+        self.assertIn("username=testuser", request.body.decode('utf-8'))
+        self.assertIn("password=testpass", request.body.decode('utf-8'))
+        self.assertEqual(request.headers.get('Content-Type'), 'application/x-www-form-urlencoded')
+
+    def test_json_body_with_post(self):
+        """测试POST请求使用json_body"""
+        # 测试POST请求使用json_body参数
+        json_data = {"name": "John", "age": 30}
+        request = Request(
+            url="https://api.example.com/users",
+            method="POST",
+            json_body=json_data
+        )
+        
+        # 验证json_body是否正确序列化
+        self.assertEqual(request.method, "POST")
+        import json
+        self.assertEqual(json.loads(request.body.decode('utf-8')), json_data)
+        self.assertEqual(request.headers.get('Content-Type'), 'application/json')
+
+    def test_get_params_with_existing_query(self):
+        """测试向已有查询参数的URL添加GET参数"""
+        # 测试向已有查询参数的URL添加更多参数
+        params = {"page": 2, "limit": 20}
+        request = Request(
+            url="https://api.example.com/users?category=active",
+            method="GET",
+            get_params=params
+        )
+        
+        # 验证原有参数和新参数都存在
+        self.assertIn("https://api.example.com/users?", request.url)
+        self.assertIn("category=active", request.url)
+        self.assertIn("page=2", request.url)
+        self.assertIn("limit=20", request.url)
+
+    def test_form_data_conversion_to_get(self):
+        """测试form_data在GET请求中自动转换为URL参数"""
+        # 测试form_data在GET请求中自动转换为URL参数的行为
+        form_data = {"filter": "active", "sort": "date"}
+        request = Request(
+            url="https://api.example.com/posts",
+            method="GET",
+            form_data=form_data
+        )
+        
+        # 验证form_data被正确转换为GET参数
+        self.assertIn("https://api.example.com/posts?", request.url)
+        self.assertIn("filter=active", request.url)
+        self.assertIn("sort=date", request.url)
+        self.assertEqual(request.method, "GET")
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_request_serialization.py

@@ -1,71 +1,71 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试 Request 序列化问题修复
-"""
-import pickle
-import sys
-sys.path.insert(0, "..")
-
-from crawlo.network.request import Request
-from crawlo.core.scheduler import Scheduler
-from unittest.mock import Mock
-
-# 模拟一个带 logger 的 Request
-class TestRequest(Request):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # 添加一个 logger 属性模拟问题
-        from crawlo.utils.log import get_logger
-        self.logger = get_logger("test_request")
-        self.meta['spider_logger'] = get_logger("spider_logger")
-
-def test_request_serialization():
-    """测试 Request 序列化"""
-    print("🔍 测试 Request 序列化修复...")
-    
-    # 创建一个带 logger 的请求
-    request = TestRequest(
-        url="https://example.com",
-        meta={"test": "data"}  # 移除 Mock 对象
-    )
-    
-    print(f"   📦 原始请求: {request}")
-    print(f"   🔧 请求有 logger: {hasattr(request, 'logger')}")
-    print(f"   🔧 meta 有 logger: {'spider_logger' in request.meta}")
-    
-    # 创建一个 mock scheduler 来测试清理
-    class MockScheduler:
-        def _deep_clean_loggers(self, request):
-            return Scheduler._deep_clean_loggers(self, request)
-        def _remove_logger_from_dict(self, d):
-            return Scheduler._remove_logger_from_dict(self, d)
-    
-    scheduler = MockScheduler()
-    
-    # 执行清理
-    scheduler._deep_clean_loggers(request)
-    
-    print(f"   🧹 清理后有 logger: {hasattr(request, 'logger')}")
-    print(f"   🧹 清理后 meta 有 logger: {'spider_logger' in request.meta}")
-    
-    # 测试序列化
-    try:
-        serialized = pickle.dumps(request)
-        print(f"   ✅ 序列化成功，大小: {len(serialized)} bytes")
-        
-        # 测试反序列化
-        deserialized = pickle.loads(serialized)
-        print(f"   ✅ 反序列化成功: {deserialized}")
-        return True
-        
-    except Exception as e:
-        print(f"   ❌ 序列化失败: {e}")
-        return False
-
-if __name__ == "__main__":
-    success = test_request_serialization()
-    if success:
-        print("🎉 Request 序列化修复成功！")
-    else:
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试 Request 序列化问题修复
+"""
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.core.scheduler import Scheduler
+from unittest.mock import Mock
+
+# 模拟一个带 logger 的 Request
+class TestRequest(Request):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # 添加一个 logger 属性模拟问题
+        from crawlo.utils.log import get_logger
+        self.logger = get_logger("test_request")
+        self.meta['spider_logger'] = get_logger("spider_logger")
+
+def test_request_serialization():
+    """测试 Request 序列化"""
+    print("🔍 测试 Request 序列化修复...")
+    
+    # 创建一个带 logger 的请求
+    request = TestRequest(
+        url="https://example.com",
+        meta={"test": "data"}  # 移除 Mock 对象
+    )
+    
+    print(f"   📦 原始请求: {request}")
+    print(f"   请求有 logger: {hasattr(request, 'logger')}")
+    print(f"   meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 创建一个 mock scheduler 来测试清理
+    class MockScheduler:
+        def _deep_clean_loggers(self, request):
+            return Scheduler._deep_clean_loggers(self, request)
+        def _remove_logger_from_dict(self, d):
+            return Scheduler._remove_logger_from_dict(self, d)
+    
+    scheduler = MockScheduler()
+    
+    # 执行清理
+    scheduler._deep_clean_loggers(request)
+    
+    print(f"   🧹 清理后有 logger: {hasattr(request, 'logger')}")
+    print(f"   🧹 清理后 meta 有 logger: {'spider_logger' in request.meta}")
+    
+    # 测试序列化
+    try:
+        serialized = pickle.dumps(request)
+        print(f"   序列化成功，大小: {len(serialized)} bytes")
+        
+        # 测试反序列化
+        deserialized = pickle.loads(serialized)
+        print(f"   反序列化成功: {deserialized}")
+        return True
+        
+    except Exception as e:
+        print(f"   序列化失败: {e}")
+        return False
+
+if __name__ == "__main__":
+    success = test_request_serialization()
+    if success:
+        print("Request 序列化修复成功！")
+    else:
         print("❌ 序列化问题仍未解决")