crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/test_batch_processor.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+批处理工具测试
+测试 BatchProcessor, RedisBatchProcessor, batch_process
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+import asyncio
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.batch_processor import BatchProcessor, RedisBatchProcessor, batch_process
+
+
+class TestBatchProcessor(unittest.TestCase):
+    """批处理工具测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.batch_processor = BatchProcessor(batch_size=3, max_concurrent_batches=2)
+        
+    def test_batch_processor_initialization(self):
+        """测试批处理器初始化"""
+        self.assertEqual(self.batch_processor.batch_size, 3)
+        self.assertEqual(self.batch_processor.max_concurrent_batches, 2)
+        
+    def sync_process_item(self, item):
+        """同步处理函数"""
+        return item * 2
+        
+    def test_batch_processor_process_batch_sync(self):
+        """测试批处理器同步处理批次"""
+        items = [1, 2, 3]
+        # 使用事件循环运行异步方法
+        results = asyncio.run(self.batch_processor.process_batch(items, self.sync_process_item))
+        self.assertEqual(results, [2, 4, 6])
+        
+    def test_batch_processor_process_in_batches_sync(self):
+        """测试批处理器同步分批处理大量数据"""
+        items = [1, 2, 3, 4, 5, 6, 7]
+        # 使用事件循环运行异步方法
+        results = asyncio.run(self.batch_processor.process_in_batches(items, self.sync_process_item))
+        expected = [2, 4, 6, 8, 10, 12, 14]
+        self.assertEqual(results, expected)
+        
+    def test_batch_processor_with_exception_handling(self):
+        """测试批处理器异常处理"""
+        def failing_processor(item):
+            if item == 2:
+                raise ValueError("处理失败")
+            return item * 2
+            
+        items = [1, 2, 3]
+        # 使用事件循环运行异步方法
+        results = asyncio.run(self.batch_processor.process_batch(items, failing_processor))
+        # 异常项应该被过滤掉
+        self.assertIn(2, results)
+        self.assertIn(6, results)
+        # 检查长度至少为2
+        self.assertGreaterEqual(len(results), 2)
+        
+    def test_batch_processor_decorator(self):
+        """测试批处理器装饰器"""
+        @self.batch_processor.batch_process_decorator(batch_size=2)
+        def process_func(items):
+            return [item * 3 for item in items]
+            
+        items = [1, 2, 3, 4]
+        results = process_func(items)
+        # 检查结果不为空
+        self.assertIsNotNone(results)
+        self.assertIsInstance(results, list)
+
+
+class TestRedisBatchProcessor(unittest.TestCase):
+    """Redis批处理器测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.mock_redis_client = Mock()
+        self.redis_batch_processor = RedisBatchProcessor(self.mock_redis_client, batch_size=3)
+        
+    def test_redis_batch_processor_initialization(self):
+        """测试Redis批处理器初始化"""
+        self.assertEqual(self.redis_batch_processor.batch_size, 3)
+        self.assertEqual(self.redis_batch_processor.redis_client, self.mock_redis_client)
+        
+    def test_redis_batch_processor_batch_set(self):
+        """测试Redis批处理器批量设置"""
+        items = [
+            {'key': 'key1', 'value': 'value1'},
+            {'key': 'key2', 'value': 'value2'},
+            {'key': 'key3', 'value': 'value3'}
+        ]
+        
+        # 模拟pipeline行为
+        mock_pipe = Mock()
+        self.mock_redis_client.pipeline.return_value = mock_pipe
+        mock_pipe.execute.return_value = None  # execute方法返回None
+        mock_pipe.set.return_value = mock_pipe  # set方法返回pipe自身以支持链式调用
+        
+        # 使用事件循环运行异步方法
+        count = asyncio.run(self.redis_batch_processor.batch_set(items))
+        self.assertEqual(count, 3)
+        
+    def test_redis_batch_processor_batch_set_empty(self):
+        """测试Redis批处理器批量设置空列表"""
+        items = []
+        # 使用事件循环运行异步方法
+        count = asyncio.run(self.redis_batch_processor.batch_set(items))
+        self.assertEqual(count, 0)
+        
+    def test_redis_batch_processor_batch_get(self):
+        """测试Redis批处理器批量获取"""
+        keys = ['key1', 'key2', 'key3']
+        
+        # 模拟pipeline行为
+        mock_pipe = Mock()
+        self.mock_redis_client.pipeline.return_value = mock_pipe
+        mock_pipe.get.return_value = mock_pipe  # get方法返回pipe自身以支持链式调用
+        mock_pipe.execute.return_value = ['value1', 'value2', 'value3']
+        
+        # 使用事件循环运行异步方法
+        result = asyncio.run(self.redis_batch_processor.batch_get(keys))
+        expected = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
+        self.assertEqual(result, expected)
+        
+    def test_redis_batch_processor_batch_get_with_none_values(self):
+        """测试Redis批处理器批量获取包含None值"""
+        keys = ['key1', 'key2', 'key3']
+        
+        # 模拟pipeline行为，其中key2返回None
+        mock_pipe = Mock()
+        self.mock_redis_client.pipeline.return_value = mock_pipe
+        mock_pipe.get.return_value = mock_pipe  # get方法返回pipe自身以支持链式调用
+        mock_pipe.execute.return_value = ['value1', None, 'value3']
+        
+        # 使用事件循环运行异步方法
+        result = asyncio.run(self.redis_batch_processor.batch_get(keys))
+        expected = {'key1': 'value1', 'key3': 'value3'}  # key2应该被过滤掉
+        self.assertEqual(result, expected)
+        
+    def test_redis_batch_processor_batch_delete(self):
+        """测试Redis批处理器批量删除"""
+        keys = ['key1', 'key2', 'key3']
+        
+        # 模拟pipeline行为
+        mock_pipe = Mock()
+        self.mock_redis_client.pipeline.return_value = mock_pipe
+        mock_pipe.delete.return_value = mock_pipe  # delete方法返回pipe自身以支持链式调用
+        mock_pipe.execute.return_value = None
+        
+        # 使用事件循环运行异步方法
+        count = asyncio.run(self.redis_batch_processor.batch_delete(keys))
+        self.assertEqual(count, 3)
+
+
+class TestBatchProcessFunction(unittest.TestCase):
+    """批处理便捷函数测试类"""
+
+    def sync_process_item(self, item):
+        """同步处理函数"""
+        return item * 2
+        
+    def test_batch_process_sync_function(self):
+        """测试批处理便捷函数处理同步函数"""
+        items = [1, 2, 3, 4, 5]
+        results = batch_process(items, self.sync_process_item, batch_size=2, max_concurrent_batches=2)
+        expected = [2, 4, 6, 8, 10]
+        self.assertEqual(results, expected)
+
+
+if __name__ == '__main__':
+    # 运行测试
+    unittest.main()

tests/test_component_factory.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+组件工厂测试
+"""
+import sys
+import os
+import unittest
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.factories import (
+    ComponentRegistry, 
+    ComponentFactory, 
+    ComponentSpec, 
+    DefaultComponentFactory,
+    CrawlerComponentFactory,
+    get_component_registry
+)
+from crawlo.factories.base import ComponentSpec
+
+
+class TestComponent:
+    """测试组件类"""
+    def __init__(self, name="test_component", value=42):
+        self.name = name
+        self.value = value
+
+
+class TestComponentFactory(unittest.TestCase):
+    """组件工厂测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.registry = ComponentRegistry()
+        
+    def test_component_spec_creation(self):
+        """测试组件规范创建"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="test_component",
+            component_type=TestComponent,
+            factory_func=factory_func,
+            dependencies=[],
+            singleton=False
+        )
+        
+        self.assertEqual(spec.name, "test_component")
+        self.assertEqual(spec.component_type, TestComponent)
+        self.assertEqual(spec.dependencies, [])
+        self.assertFalse(spec.singleton)
+        
+    def test_default_component_factory(self):
+        """测试默认组件工厂"""
+        factory = DefaultComponentFactory()
+        
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="test_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 测试创建组件
+        component = factory.create(spec, name="created_component", value=100)
+        self.assertIsInstance(component, TestComponent)
+        self.assertEqual(component.name, "created_component")
+        self.assertEqual(component.value, 100)
+        
+        # 测试单例模式
+        spec_singleton = ComponentSpec(
+            name="singleton_component",
+            component_type=TestComponent,
+            factory_func=factory_func,
+            singleton=True
+        )
+        
+        component1 = factory.create(spec_singleton, name="singleton_1", value=200)
+        component2 = factory.create(spec_singleton, name="singleton_2", value=300)
+        
+        # 单例应该返回相同的实例
+        self.assertIs(component1, component2)
+        self.assertEqual(component1.value, 200)  # 应该是第一次创建时的值
+        
+    def test_component_registry_registration(self):
+        """测试组件注册表注册功能"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="registered_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 注册组件规范
+        self.registry.register(spec)
+        
+        # 验证注册
+        retrieved_spec = self.registry.get_spec("registered_component")
+        self.assertEqual(retrieved_spec, spec)
+        
+        # 测试列出组件
+        components = self.registry.list_components()
+        self.assertIn("registered_component", components)
+        
+    def test_component_registry_creation(self):
+        """测试组件注册表创建功能"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="creatable_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 注册组件规范
+        self.registry.register(spec)
+        
+        # 创建组件
+        component = self.registry.create("creatable_component", name="created", value=500)
+        self.assertIsInstance(component, TestComponent)
+        self.assertEqual(component.name, "created")
+        self.assertEqual(component.value, 500)
+        
+    def test_global_component_registry(self):
+        """测试全局组件注册表"""
+        registry = get_component_registry()
+        self.assertIsInstance(registry, ComponentRegistry)
+        
+        # 测试注册表是否包含预注册的组件
+        components = registry.list_components()
+        # 应该至少包含crawler组件
+        self.assertGreater(len(components), 0)
+        
+    def test_crawler_component_factory(self):
+        """测试Crawler组件工厂"""
+        factory = CrawlerComponentFactory()
+        
+        # 测试支持检查
+        class MockEngine:
+            pass
+            
+        self.assertTrue(factory.supports(MockEngine))  # 默认支持所有类型
+        
+        # 测试创建功能（需要crawler依赖）
+        def mock_engine_factory(crawler=None, **kwargs):
+            if crawler is None:
+                raise ValueError("需要crawler实例")
+            return "mock_engine"
+            
+        spec = ComponentSpec(
+            name="mock_engine",
+            component_type=type('MockEngine', (), {}),
+            factory_func=mock_engine_factory,
+            dependencies=['crawler']
+        )
+        
+        # 测试缺少依赖时的错误处理
+        with self.assertRaises(ValueError):
+            factory.create(spec)
+            
+        # 测试带依赖的创建
+        result = factory.create(spec, crawler="mock_crawler")
+        self.assertEqual(result, "mock_engine")
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_controlled_spider_mixin.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+受控爬虫混入类测试
+测试 ControlledRequestMixin, AsyncControlledRequestMixin
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+import asyncio
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.controlled_spider_mixin import ControlledRequestMixin, AsyncControlledRequestMixin
+
+
+class TestControlledRequestMixin(unittest.TestCase):
+    """受控请求混入类测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.mixin = ControlledRequestMixin()
+        
+    def test_mixin_initialization(self):
+        """测试混入类初始化"""
+        self.assertEqual(self.mixin.max_pending_requests, 100)
+        self.assertEqual(self.mixin.batch_size, 50)
+        self.assertEqual(self.mixin.generation_interval, 0.1)
+        self.assertEqual(self.mixin.backpressure_threshold, 200)
+        
+    def test_mixin_configuration(self):
+        """测试混入类配置"""
+        # 修改配置
+        self.mixin.max_pending_requests = 200
+        self.mixin.batch_size = 100
+        self.mixin.generation_interval = 0.05
+        self.mixin.backpressure_threshold = 300
+        
+        self.assertEqual(self.mixin.max_pending_requests, 200)
+        self.assertEqual(self.mixin.batch_size, 100)
+        self.assertEqual(self.mixin.generation_interval, 0.05)
+        self.assertEqual(self.mixin.backpressure_threshold, 300)
+        
+    def test_get_generation_stats(self):
+        """测试获取生成统计信息"""
+        stats = self.mixin.get_generation_stats()
+        self.assertIsInstance(stats, dict)
+        self.assertIn('generated', stats)
+        self.assertIn('skipped', stats)
+        self.assertIn('backpressure_events', stats)
+        self.assertIn('total_generated', stats)
+        self.assertIn('last_generation_time', stats)
+
+
+class TestAsyncControlledRequestMixin(unittest.TestCase):
+    """异步受控请求混入类测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.mixin = AsyncControlledRequestMixin()
+        
+    def test_async_mixin_initialization(self):
+        """测试异步混入类初始化"""
+        self.assertEqual(self.mixin.max_concurrent_generations, 10)
+        self.assertEqual(self.mixin.queue_monitor_interval, 1.0)
+        
+    def test_async_mixin_configuration(self):
+        """测试异步混入类配置"""
+        # 修改配置
+        self.mixin.max_concurrent_generations = 20
+        self.mixin.queue_monitor_interval = 0.5
+        
+        self.assertEqual(self.mixin.max_concurrent_generations, 20)
+        self.assertEqual(self.mixin.queue_monitor_interval, 0.5)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_enhanced_error_handler_comprehensive.py

@@ -0,0 +1,246 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+增强错误处理工具综合测试
+测试 ErrorContext, DetailedException, EnhancedErrorHandler 的更多功能
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+import asyncio
+import traceback
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.enhanced_error_handler import (
+    EnhancedErrorHandler, 
+    ErrorContext, 
+    DetailedException, 
+    handle_exception
+)
+
+
+class TestErrorContext(unittest.TestCase):
+    """错误上下文测试"""
+
+    def test_error_context_initialization(self):
+        """测试错误上下文初始化"""
+        context = ErrorContext(
+            context="测试上下文",
+            module="test_module",
+            function="test_function"
+        )
+        
+        self.assertEqual(context.context, "测试上下文")
+        self.assertEqual(context.module, "test_module")
+        self.assertEqual(context.function, "test_function")
+        self.assertIsNotNone(context.timestamp)
+        
+    def test_error_context_string_representation(self):
+        """测试错误上下文字符串表示"""
+        context = ErrorContext(
+            context="测试上下文",
+            module="test_module",
+            function="test_function"
+        )
+        
+        context_str = str(context)
+        self.assertIn("Module: test_module", context_str)
+        self.assertIn("Function: test_function", context_str)
+        self.assertIn("Context: 测试上下文", context_str)
+        self.assertIn("Time:", context_str)
+
+
+class TestDetailedException(unittest.TestCase):
+    """详细异常测试"""
+
+    def test_detailed_exception_initialization(self):
+        """测试详细异常初始化"""
+        context = ErrorContext(context="测试上下文")
+        
+        exception = DetailedException(
+            message="测试异常消息",
+            context=context,
+            error_code="TEST_001",
+            detail1="详细信息1",
+            detail2="详细信息2"
+        )
+        
+        self.assertIn("测试异常消息", str(exception))
+        self.assertEqual(exception.context, context)
+        self.assertEqual(exception.error_code, "TEST_001")
+        self.assertEqual(exception.details["detail1"], "详细信息1")
+        self.assertEqual(exception.details["detail2"], "详细信息2")
+        self.assertIsNotNone(exception.timestamp)
+        
+    def test_detailed_exception_without_context(self):
+        """测试没有上下文的详细异常"""
+        exception = DetailedException(
+            message="测试异常消息",
+            error_code="TEST_002"
+        )
+        
+        self.assertEqual(str(exception), "测试异常消息")
+        self.assertIsNone(exception.context)
+        self.assertEqual(exception.error_code, "TEST_002")
+        self.assertIsNotNone(exception.timestamp)
+        
+    def test_detailed_exception_string_with_context(self):
+        """测试带上下文的详细异常字符串表示"""
+        context = ErrorContext(context="测试上下文")
+        exception = DetailedException(
+            message="测试异常消息",
+            context=context
+        )
+        
+        exception_str = str(exception)
+        self.assertIn("测试异常消息", exception_str)
+        self.assertIn("测试上下文", exception_str)
+        
+    def test_get_full_details(self):
+        """测试获取完整详情"""
+        context = ErrorContext(context="测试上下文")
+        exception = DetailedException(
+            message="测试异常消息",
+            context=context,
+            error_code="TEST_003",
+            detail1="详细信息1"
+        )
+        
+        details = exception.get_full_details()
+        self.assertIn("测试异常消息", details["message"])
+        self.assertEqual(details["error_code"], "TEST_003")
+        self.assertIn("测试上下文", details["context"])
+        self.assertEqual(details["details"]["detail1"], "详细信息1")
+        self.assertEqual(details["exception_type"], "DetailedException")
+
+
+class TestEnhancedErrorHandler(unittest.TestCase):
+    """增强错误处理器测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.handler = EnhancedErrorHandler("test_logger", "ERROR")
+        
+    def test_handler_initialization(self):
+        """测试处理器初始化"""
+        self.assertEqual(len(self.handler.error_history), 0)
+        self.assertEqual(self.handler.max_history_size, 100)
+        
+    def test_handle_error_without_raising(self):
+        """测试不抛出异常的错误处理"""
+        try:
+            raise ValueError("测试错误")
+        except Exception as e:
+            # 处理错误但不重新抛出
+            error_details = self.handler.handle_error(e, raise_error=False, log_error=False)
+            
+            # 验证返回的错误详情
+            self.assertIsInstance(error_details, dict)
+            self.assertEqual(error_details["exception_type"], "ValueError")
+            self.assertEqual(error_details["message"], "测试错误")
+            
+    def test_safe_call_success(self):
+        """测试安全调用成功"""
+        def normal_function(x, y):
+            return x + y
+            
+        result = self.handler.safe_call(normal_function, 1, 2, default_return=0)
+        self.assertEqual(result, 3)
+        
+    def test_safe_call_with_exception(self):
+        """测试安全调用异常"""
+        def failing_function():
+            raise RuntimeError("函数执行失败")
+            
+        result = self.handler.safe_call(failing_function, default_return="默认值")
+        self.assertEqual(result, "默认值")
+        
+    def test_get_and_clear_error_history(self):
+        """测试获取和清空错误历史"""
+        # 产生一些错误
+        try:
+            raise ValueError("错误1")
+        except Exception as e:
+            self.handler.handle_error(e, raise_error=False, log_error=False)
+            
+        try:
+            raise RuntimeError("错误2")
+        except Exception as e:
+            self.handler.handle_error(e, raise_error=False, log_error=False)
+            
+        # 检查历史记录
+        history = self.handler.get_error_history()
+        self.assertEqual(len(history), 2)
+        
+        # 清空历史记录
+        self.handler.clear_error_history()
+        history = self.handler.get_error_history()
+        self.assertEqual(len(history), 0)
+        
+    def test_error_history_size_limit(self):
+        """测试错误历史大小限制"""
+        # 产生超过100个错误
+        for i in range(110):
+            try:
+                raise ValueError(f"错误{i}")
+            except Exception as e:
+                self.handler.handle_error(e, raise_error=False, log_error=False)
+                
+        # 检查历史记录大小限制
+        history = self.handler.get_error_history()
+        self.assertEqual(len(history), 100)  # 应该限制在100个
+
+
+class TestHandleExceptionDecorator(unittest.TestCase):
+    """异常处理装饰器测试"""
+
+    def test_handle_exception_decorator_sync(self):
+        """测试同步函数的异常处理装饰器"""
+        @handle_exception(context="测试装饰器", module="test_module", raise_error=False)
+        def failing_function():
+            raise ValueError("装饰器测试错误")
+            
+        # 调用应该捕获异常但不抛出
+        try:
+            result = failing_function()
+            self.assertIsNone(result)  # 默认返回None
+        except Exception:
+            self.fail("异常应该被捕获")
+            
+    def test_handle_exception_decorator_async(self):
+        """测试异步函数的异常处理装饰器"""
+        @handle_exception(context="异步测试装饰器", module="test_module", raise_error=False)
+        async def async_failing_function():
+            raise RuntimeError("异步装饰器测试错误")
+            
+        # 异步调用
+        async def test_async():
+            try:
+                result = await async_failing_function()
+                self.assertIsNone(result)  # 默认返回None
+            except Exception:
+                self.fail("异常应该被捕获")
+                
+        # 使用事件循环运行异步函数
+        asyncio.run(test_async())
+        
+    def test_handle_exception_decorator_with_detailed_exception(self):
+        """测试装饰器处理详细异常"""
+        @handle_exception(context="详细异常测试", module="test_module", raise_error=False)
+        def function_with_detailed_exception():
+            context = ErrorContext(context="函数内部上下文")
+            raise DetailedException("详细异常消息", context=context, error_code="DETAIL_001")
+            
+        # 调用应该捕获异常
+        try:
+            result = function_with_detailed_exception()
+            self.assertIsNone(result)
+        except Exception:
+            self.fail("详细异常应该被捕获")
+
+
+if __name__ == '__main__':
+    unittest.main()