crawlo 1.3.1__py3-none-any.whl → 1.3.3__py3-none-any.whl

tests/test_request_ignore_middleware.py

@@ -1,183 +1,183 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-RequestIgnoreMiddleware 测试文件
-用于测试请求忽略中间件的功能
-"""
-
-import asyncio
-import unittest
-from unittest.mock import Mock, patch
-
-from crawlo.middleware.request_ignore import RequestIgnoreMiddleware
-from crawlo.exceptions import IgnoreRequestError
-from crawlo.settings.setting_manager import SettingManager
-
-
-class MockLogger:
-    """Mock Logger 类，用于测试日志输出"""
-    def __init__(self, name, level=None):
-        self.name = name
-        self.level = level
-        self.logs = []
-
-    def debug(self, msg):
-        self.logs.append(('debug', msg))
-
-    def info(self, msg):
-        self.logs.append(('info', msg))
-
-    def warning(self, msg):
-        self.logs.append(('warning', msg))
-
-    def error(self, msg):
-        self.logs.append(('error', msg))
-
-
-class MockStats:
-    """Mock Stats 类，用于测试统计信息"""
-    def __init__(self):
-        self.stats = {}
-
-    def inc_value(self, key, value=1):
-        if key in self.stats:
-            self.stats[key] += value
-        else:
-            self.stats[key] = value
-
-
-class TestRequestIgnoreMiddleware(unittest.TestCase):
-    """RequestIgnoreMiddleware 测试类"""
-
-    def setUp(self):
-        """测试前准备"""
-        # 创建设置管理器
-        self.settings = SettingManager()
-        
-        # 创建爬虫模拟对象
-        self.crawler = Mock()
-        self.crawler.settings = self.settings
-        self.crawler.stats = MockStats()
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_middleware_initialization(self, mock_get_logger):
-        """测试中间件初始化"""
-        self.settings.set('LOG_LEVEL', 'INFO')
-        mock_get_logger.return_value = MockLogger('RequestIgnoreMiddleware')
-        
-        # 应该正常创建实例
-        middleware = RequestIgnoreMiddleware.create_instance(self.crawler)
-        self.assertIsInstance(middleware, RequestIgnoreMiddleware)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling(self, mock_get_logger):
-        """测试请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        request.url = 'http://example.com/page'
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证统计信息
-        self.assertIn('request_ignore_count', mock_stats.stats)
-        self.assertEqual(mock_stats.stats['request_ignore_count'], 1)
-        self.assertIn('request_ignore_count/reason/test reason', mock_stats.stats)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling_with_domain(self, mock_get_logger):
-        """测试带域名的请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        request.url = 'http://example.com/page'
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证域名统计信息
-        self.assertIn('request_ignore_count/domain/example.com', mock_stats.stats)
-
-    @patch('crawlo.utils.log.get_logger')
-    def test_request_ignore_event_handling_with_invalid_url(self, mock_get_logger):
-        """测试带无效URL的请求忽略事件处理"""
-        self.settings.set('LOG_LEVEL', 'DEBUG')
-        mock_logger = MockLogger('RequestIgnoreMiddleware')
-        mock_get_logger.return_value = mock_logger
-        
-        # 创建中间件实例
-        mock_stats = MockStats()
-        middleware = RequestIgnoreMiddleware(
-            stats=mock_stats,
-            log_level='DEBUG'
-        )
-        
-        # 创建异常和请求对象（没有url属性，会触发异常）
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        # 不设置request.url属性，这样在访问时会抛出AttributeError
-        
-        # 处理忽略事件
-        asyncio.run(middleware.request_ignore(exc, request, Mock()))
-        
-        # 验证无效URL统计信息
-        self.assertIn('request_ignore_count/domain/invalid_url', mock_stats.stats)
-
-    def test_process_exception_with_ignore_request_error(self):
-        """测试处理IgnoreRequestError异常"""
-        # 创建中间件实例
-        middleware = RequestIgnoreMiddleware(
-            stats=MockStats(),
-            log_level='INFO'
-        )
-        
-        # 创建异常和请求对象
-        exc = IgnoreRequestError("test reason")
-        request = Mock()
-        
-        # 应该返回True表示异常已被处理
-        result = middleware.process_exception(request, exc, Mock())
-        self.assertTrue(result)
-
-    def test_process_exception_with_other_exception(self):
-        """测试处理其他异常"""
-        # 创建中间件实例
-        middleware = RequestIgnoreMiddleware(
-            stats=MockStats(),
-            log_level='INFO'
-        )
-        
-        # 创建异常和请求对象
-        exc = ValueError("test error")
-        request = Mock()
-        
-        # 应该返回None表示异常未被处理
-        result = middleware.process_exception(request, exc, Mock())
-        self.assertIsNone(result)
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+RequestIgnoreMiddleware 测试文件
+用于测试请求忽略中间件的功能
+"""
+
+import asyncio
+import unittest
+from unittest.mock import Mock, patch
+
+from crawlo.middleware.request_ignore import RequestIgnoreMiddleware
+from crawlo.exceptions import IgnoreRequestError
+from crawlo.settings.setting_manager import SettingManager
+
+
+class MockLogger:
+    """Mock Logger 类，用于测试日志输出"""
+    def __init__(self, name, level=None):
+        self.name = name
+        self.level = level
+        self.logs = []
+
+    def debug(self, msg):
+        self.logs.append(('debug', msg))
+
+    def info(self, msg):
+        self.logs.append(('info', msg))
+
+    def warning(self, msg):
+        self.logs.append(('warning', msg))
+
+    def error(self, msg):
+        self.logs.append(('error', msg))
+
+
+class MockStats:
+    """Mock Stats 类，用于测试统计信息"""
+    def __init__(self):
+        self.stats = {}
+
+    def inc_value(self, key, value=1):
+        if key in self.stats:
+            self.stats[key] += value
+        else:
+            self.stats[key] = value
+
+
+class TestRequestIgnoreMiddleware(unittest.TestCase):
+    """RequestIgnoreMiddleware 测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建设置管理器
+        self.settings = SettingManager()
+        
+        # 创建爬虫模拟对象
+        self.crawler = Mock()
+        self.crawler.settings = self.settings
+        self.crawler.stats = MockStats()
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_middleware_initialization(self, mock_get_logger):
+        """测试中间件初始化"""
+        self.settings.set('LOG_LEVEL', 'INFO')
+        mock_get_logger.return_value = MockLogger('RequestIgnoreMiddleware')
+        
+        # 应该正常创建实例
+        middleware = RequestIgnoreMiddleware.create_instance(self.crawler)
+        self.assertIsInstance(middleware, RequestIgnoreMiddleware)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling(self, mock_get_logger):
+        """测试请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        request.url = 'http://example.com/page'
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证统计信息
+        self.assertIn('request_ignore_count', mock_stats.stats)
+        self.assertEqual(mock_stats.stats['request_ignore_count'], 1)
+        self.assertIn('request_ignore_count/reason/test reason', mock_stats.stats)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling_with_domain(self, mock_get_logger):
+        """测试带域名的请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        request.url = 'http://example.com/page'
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证域名统计信息
+        self.assertIn('request_ignore_count/domain/example.com', mock_stats.stats)
+
+    @patch('crawlo.utils.log.get_logger')
+    def test_request_ignore_event_handling_with_invalid_url(self, mock_get_logger):
+        """测试带无效URL的请求忽略事件处理"""
+        self.settings.set('LOG_LEVEL', 'DEBUG')
+        mock_logger = MockLogger('RequestIgnoreMiddleware')
+        mock_get_logger.return_value = mock_logger
+        
+        # 创建中间件实例
+        mock_stats = MockStats()
+        middleware = RequestIgnoreMiddleware(
+            stats=mock_stats,
+            log_level='DEBUG'
+        )
+        
+        # 创建异常和请求对象（没有url属性，会触发异常）
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        # 不设置request.url属性，这样在访问时会抛出AttributeError
+        
+        # 处理忽略事件
+        asyncio.run(middleware.request_ignore(exc, request, Mock()))
+        
+        # 验证无效URL统计信息
+        self.assertIn('request_ignore_count/domain/invalid_url', mock_stats.stats)
+
+    def test_process_exception_with_ignore_request_error(self):
+        """测试处理IgnoreRequestError异常"""
+        # 创建中间件实例
+        middleware = RequestIgnoreMiddleware(
+            stats=MockStats(),
+            log_level='INFO'
+        )
+        
+        # 创建异常和请求对象
+        exc = IgnoreRequestError("test reason")
+        request = Mock()
+        
+        # 应该返回True表示异常已被处理
+        result = middleware.process_exception(request, exc, Mock())
+        self.assertTrue(result)
+
+    def test_process_exception_with_other_exception(self):
+        """测试处理其他异常"""
+        # 创建中间件实例
+        middleware = RequestIgnoreMiddleware(
+            stats=MockStats(),
+            log_level='INFO'
+        )
+        
+        # 创建异常和请求对象
+        exc = ValueError("test error")
+        request = Mock()
+        
+        # 应该返回None表示异常未被处理
+        result = middleware.process_exception(request, exc, Mock())
+        self.assertIsNone(result)
+
+
+if __name__ == '__main__':
     unittest.main()

tests/test_request_params.py

@@ -1,112 +1,112 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Request参数处理测试
-"""
-import unittest
-from crawlo.network.request import Request
-
-
-class TestRequestParams(unittest.TestCase):
-    """Request参数处理测试类"""
-
-    def test_get_params_handling(self):
-        """测试GET参数处理"""
-        # 测试GET请求带参数
-        params = {"page": 1, "limit": 10, "search": "test"}
-        request = Request(
-            url="https://api.example.com/users",
-            method="GET",
-            get_params=params
-        )
-        
-        # 验证参数是否正确附加到URL上
-        self.assertIn("https://api.example.com/users?", request.url)
-        self.assertIn("page=1", request.url)
-        self.assertIn("limit=10", request.url)
-        self.assertIn("search=test", request.url)
-        self.assertEqual(request.method, "GET")
-
-    def test_form_data_with_get(self):
-        """测试GET请求使用form_data"""
-        # 测试GET请求使用form_data参数
-        form_data = {"category": "books", "sort": "name"}
-        request = Request(
-            url="https://api.example.com/products",
-            method="GET",
-            form_data=form_data
-        )
-        
-        # 验证form_data是否正确作为GET参数附加到URL上
-        self.assertIn("https://api.example.com/products?", request.url)
-        self.assertIn("category=books", request.url)
-        self.assertIn("sort=name", request.url)
-        self.assertEqual(request.method, "GET")
-
-    def test_form_data_with_post(self):
-        """测试POST请求使用form_data"""
-        # 测试POST请求使用form_data参数
-        form_data = {"username": "testuser", "password": "testpass"}
-        request = Request(
-            url="https://api.example.com/login",
-            method="POST",
-            form_data=form_data
-        )
-        
-        # 验证form_data是否作为请求体发送
-        self.assertEqual(request.method, "POST")
-        self.assertIn("username=testuser", request.body.decode('utf-8'))
-        self.assertIn("password=testpass", request.body.decode('utf-8'))
-        self.assertEqual(request.headers.get('Content-Type'), 'application/x-www-form-urlencoded')
-
-    def test_json_body_with_post(self):
-        """测试POST请求使用json_body"""
-        # 测试POST请求使用json_body参数
-        json_data = {"name": "John", "age": 30}
-        request = Request(
-            url="https://api.example.com/users",
-            method="POST",
-            json_body=json_data
-        )
-        
-        # 验证json_body是否正确序列化
-        self.assertEqual(request.method, "POST")
-        import json
-        self.assertEqual(json.loads(request.body.decode('utf-8')), json_data)
-        self.assertEqual(request.headers.get('Content-Type'), 'application/json')
-
-    def test_get_params_with_existing_query(self):
-        """测试向已有查询参数的URL添加GET参数"""
-        # 测试向已有查询参数的URL添加更多参数
-        params = {"page": 2, "limit": 20}
-        request = Request(
-            url="https://api.example.com/users?category=active",
-            method="GET",
-            get_params=params
-        )
-        
-        # 验证原有参数和新参数都存在
-        self.assertIn("https://api.example.com/users?", request.url)
-        self.assertIn("category=active", request.url)
-        self.assertIn("page=2", request.url)
-        self.assertIn("limit=20", request.url)
-
-    def test_form_data_conversion_to_get(self):
-        """测试form_data在GET请求中自动转换为URL参数"""
-        # 测试form_data在GET请求中自动转换为URL参数的行为
-        form_data = {"filter": "active", "sort": "date"}
-        request = Request(
-            url="https://api.example.com/posts",
-            method="GET",
-            form_data=form_data
-        )
-        
-        # 验证form_data被正确转换为GET参数
-        self.assertIn("https://api.example.com/posts?", request.url)
-        self.assertIn("filter=active", request.url)
-        self.assertIn("sort=date", request.url)
-        self.assertEqual(request.method, "GET")
-
-
-if __name__ == '__main__':
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Request参数处理测试
+"""
+import unittest
+from crawlo.network.request import Request
+
+
+class TestRequestParams(unittest.TestCase):
+    """Request参数处理测试类"""
+
+    def test_get_params_handling(self):
+        """测试GET参数处理"""
+        # 测试GET请求带参数
+        params = {"page": 1, "limit": 10, "search": "test"}
+        request = Request(
+            url="https://api.example.com/users",
+            method="GET",
+            get_params=params
+        )
+        
+        # 验证参数是否正确附加到URL上
+        self.assertIn("https://api.example.com/users?", request.url)
+        self.assertIn("page=1", request.url)
+        self.assertIn("limit=10", request.url)
+        self.assertIn("search=test", request.url)
+        self.assertEqual(request.method, "GET")
+
+    def test_form_data_with_get(self):
+        """测试GET请求使用form_data"""
+        # 测试GET请求使用form_data参数
+        form_data = {"category": "books", "sort": "name"}
+        request = Request(
+            url="https://api.example.com/products",
+            method="GET",
+            form_data=form_data
+        )
+        
+        # 验证form_data是否正确作为GET参数附加到URL上
+        self.assertIn("https://api.example.com/products?", request.url)
+        self.assertIn("category=books", request.url)
+        self.assertIn("sort=name", request.url)
+        self.assertEqual(request.method, "GET")
+
+    def test_form_data_with_post(self):
+        """测试POST请求使用form_data"""
+        # 测试POST请求使用form_data参数
+        form_data = {"username": "testuser", "password": "testpass"}
+        request = Request(
+            url="https://api.example.com/login",
+            method="POST",
+            form_data=form_data
+        )
+        
+        # 验证form_data是否作为请求体发送
+        self.assertEqual(request.method, "POST")
+        self.assertIn("username=testuser", request.body.decode('utf-8'))
+        self.assertIn("password=testpass", request.body.decode('utf-8'))
+        self.assertEqual(request.headers.get('Content-Type'), 'application/x-www-form-urlencoded')
+
+    def test_json_body_with_post(self):
+        """测试POST请求使用json_body"""
+        # 测试POST请求使用json_body参数
+        json_data = {"name": "John", "age": 30}
+        request = Request(
+            url="https://api.example.com/users",
+            method="POST",
+            json_body=json_data
+        )
+        
+        # 验证json_body是否正确序列化
+        self.assertEqual(request.method, "POST")
+        import json
+        self.assertEqual(json.loads(request.body.decode('utf-8')), json_data)
+        self.assertEqual(request.headers.get('Content-Type'), 'application/json')
+
+    def test_get_params_with_existing_query(self):
+        """测试向已有查询参数的URL添加GET参数"""
+        # 测试向已有查询参数的URL添加更多参数
+        params = {"page": 2, "limit": 20}
+        request = Request(
+            url="https://api.example.com/users?category=active",
+            method="GET",
+            get_params=params
+        )
+        
+        # 验证原有参数和新参数都存在
+        self.assertIn("https://api.example.com/users?", request.url)
+        self.assertIn("category=active", request.url)
+        self.assertIn("page=2", request.url)
+        self.assertIn("limit=20", request.url)
+
+    def test_form_data_conversion_to_get(self):
+        """测试form_data在GET请求中自动转换为URL参数"""
+        # 测试form_data在GET请求中自动转换为URL参数的行为
+        form_data = {"filter": "active", "sort": "date"}
+        request = Request(
+            url="https://api.example.com/posts",
+            method="GET",
+            form_data=form_data
+        )
+        
+        # 验证form_data被正确转换为GET参数
+        self.assertIn("https://api.example.com/posts?", request.url)
+        self.assertIn("filter=active", request.url)
+        self.assertIn("sort=date", request.url)
+        self.assertEqual(request.method, "GET")
+
+
+if __name__ == '__main__':
     unittest.main()