crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/test_factories.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+工厂模式测试
+测试 ComponentRegistry, ComponentFactory, DefaultComponentFactory, CrawlerComponentFactory
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.factories import (
+    ComponentRegistry, 
+    ComponentFactory, 
+    ComponentSpec, 
+    CrawlerComponentFactory,
+    get_component_registry
+)
+from crawlo.factories.base import DefaultComponentFactory
+
+
+class TestComponent:
+    """测试组件类"""
+    def __init__(self, name="test_component", value=42):
+        self.name = name
+        self.value = value
+
+
+class TestFactories(unittest.TestCase):
+    """工厂模式测试类"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.registry = ComponentRegistry()
+        
+    def test_component_spec_creation(self):
+        """测试组件规范创建"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="test_component",
+            component_type=TestComponent,
+            factory_func=factory_func,
+            dependencies=[],
+            singleton=False
+        )
+        
+        self.assertEqual(spec.name, "test_component")
+        self.assertEqual(spec.component_type, TestComponent)
+        self.assertEqual(spec.dependencies, [])
+        self.assertFalse(spec.singleton)
+        
+    def test_default_component_factory(self):
+        """测试默认组件工厂"""
+        factory = DefaultComponentFactory()
+        
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="test_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 测试创建组件
+        component = factory.create(spec, name="created_component", value=100)
+        self.assertIsInstance(component, TestComponent)
+        self.assertEqual(component.name, "created_component")
+        self.assertEqual(component.value, 100)
+        
+        # 测试单例模式
+        spec_singleton = ComponentSpec(
+            name="singleton_component",
+            component_type=TestComponent,
+            factory_func=factory_func,
+            singleton=True
+        )
+        
+        component1 = factory.create(spec_singleton, name="singleton_1", value=200)
+        component2 = factory.create(spec_singleton, name="singleton_2", value=300)
+        
+        # 单例应该返回相同的实例
+        self.assertIs(component1, component2)
+        self.assertEqual(component1.value, 200)  # 应该是第一次创建时的值
+        
+    def test_component_registry_registration(self):
+        """测试组件注册表注册功能"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="registered_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 注册组件规范
+        self.registry.register(spec)
+        
+        # 验证注册
+        retrieved_spec = self.registry.get_spec("registered_component")
+        self.assertEqual(retrieved_spec, spec)
+        
+        # 测试列出组件
+        components = self.registry.list_components()
+        self.assertIn("registered_component", components)
+        
+    def test_component_registry_creation(self):
+        """测试组件注册表创建功能"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="creatable_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 注册组件规范
+        self.registry.register(spec)
+        
+        # 创建组件时应该出现错误，因为没有传递name参数
+        with self.assertRaises(TypeError):
+            component = self.registry.create("creatable_component", name="created", value=500)
+        
+    def test_global_component_registry(self):
+        """测试全局组件注册表"""
+        registry = get_component_registry()
+        self.assertIsInstance(registry, ComponentRegistry)
+        
+        # 测试注册表是否包含预注册的组件
+        components = registry.list_components()
+        # 应该至少包含crawler组件
+        self.assertGreater(len(components), 0)
+        
+    def test_crawler_component_factory_supports(self):
+        """测试Crawler组件工厂支持检查"""
+        factory = CrawlerComponentFactory()
+        
+        # 测试支持检查（CrawlerComponentFactory只支持特定类型）
+        class Engine:
+            pass
+            
+        class MockEngine:
+            pass
+            
+        # Engine应该被支持
+        self.assertTrue(factory.supports(Engine))
+        
+        # MockEngine不应该被支持
+        self.assertFalse(factory.supports(MockEngine))
+        
+    def test_crawler_component_factory_create_without_crawler(self):
+        """测试Crawler组件工厂创建时缺少crawler依赖"""
+        factory = CrawlerComponentFactory()
+        
+        # 测试创建功能（需要crawler依赖）
+        def mock_engine_factory(crawler=None, **kwargs):
+            if crawler is None:
+                raise ValueError("需要crawler实例")
+            return "mock_engine"
+            
+        spec = ComponentSpec(
+            name="mock_engine",
+            component_type=type('Engine', (), {}),  # 使用Engine类型
+            factory_func=mock_engine_factory,
+            dependencies=['crawler']
+        )
+        
+        # 测试缺少依赖时的错误处理
+        with self.assertRaises(ValueError) as context:
+            factory.create(spec)
+        self.assertIn("Crawler instance required for component", str(context.exception))
+            
+    def test_crawler_component_factory_create_with_crawler(self):
+        """测试Crawler组件工厂带crawler依赖的创建"""
+        factory = CrawlerComponentFactory()
+        
+        def mock_engine_factory(crawler=None, **kwargs):
+            if crawler is None:
+                raise ValueError("需要crawler实例")
+            return "mock_engine"
+            
+        spec = ComponentSpec(
+            name="mock_engine",
+            component_type=type('MockEngine', (), {}),
+            factory_func=mock_engine_factory,
+            dependencies=['crawler']
+        )
+        
+        # 测试带依赖的创建
+        result = factory.create(spec, crawler="mock_crawler")
+        self.assertEqual(result, "mock_engine")
+        
+    def test_component_registry_clear(self):
+        """测试组件注册表清空功能"""
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec = ComponentSpec(
+            name="clearable_component",
+            component_type=TestComponent,
+            factory_func=factory_func
+        )
+        
+        # 注册组件
+        self.registry.register(spec)
+        self.assertIsNotNone(self.registry.get_spec("clearable_component"))
+        
+        # 清空注册表
+        self.registry.clear()
+        
+        # 验证已清空
+        self.assertIsNone(self.registry.get_spec("clearable_component"))
+        self.assertEqual(len(self.registry.list_components()), 0)
+        
+    def test_default_component_factory_clear_singletons(self):
+        """测试默认组件工厂清空单例功能"""
+        factory = DefaultComponentFactory()
+        
+        def factory_func(**kwargs):
+            return TestComponent(**kwargs)
+            
+        spec_singleton = ComponentSpec(
+            name="clear_singleton_component",
+            component_type=TestComponent,
+            factory_func=factory_func,
+            singleton=True
+        )
+        
+        # 创建单例组件
+        component1 = factory.create(spec_singleton, name="singleton_1", value=200)
+        component2 = factory.create(spec_singleton, name="singleton_2", value=300)
+        
+        # 验证是单例
+        self.assertIs(component1, component2)
+        
+        # 清空单例
+        factory.clear_singletons()
+        
+        # 再次创建应该得到新实例
+        component3 = factory.create(spec_singleton, name="singleton_3", value=400)
+        self.assertIsNot(component1, component3)
+        self.assertEqual(component3.value, 400)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_framework_logger.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试框架日志系统
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.core.framework_initializer import initialize_framework, get_framework_initializer
+from crawlo.utils.log import get_logger, LoggerManager
+
+def test_framework_logger():
+    print("=== 测试框架日志系统 ===")
+    
+    # 1. 初始化框架，传入基本配置
+    print("1. 初始化框架...")
+    custom_settings = {
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': 'test_framework.log',
+        'PROJECT_NAME': 'test_framework',
+        'RUN_MODE': 'standalone'
+    }
+    settings = initialize_framework(custom_settings)
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    
+    # 2. 获取框架初始化管理器
+    init_manager = get_framework_initializer()
+    print(f"   框架是否就绪: {init_manager.is_ready}")
+    print(f"   初始化阶段: {init_manager.phase}")
+    
+    # 3. 测试框架logger
+    framework_logger = init_manager.logger
+    if framework_logger:
+        print(f"   框架logger名称: {framework_logger.name}")
+        print(f"   框架logger级别: {framework_logger.level}")
+        print(f"   框架logger处理器数量: {len(framework_logger.handlers)}")
+        
+        for i, handler in enumerate(framework_logger.handlers):
+            handler_type = type(handler).__name__
+            print(f"     处理器{i}: {handler_type}, 级别: {handler.level}")
+    else:
+        print("   框架logger为None!")
+        framework_logger = get_logger('crawlo.framework')
+        print(f"   手动创建的框架logger: {framework_logger.name}")
+    
+    # 4. 测试日志输出
+    print("2. 测试日志输出...")
+    framework_logger.info("Crawlo框架初始化完成")
+    framework_logger.info("Crawlo Framework Started 1.3.3")
+    framework_logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+    framework_logger.info("Run Mode: standalone")
+    framework_logger.info("Starting running test_spider")
+    
+    # 5. 测试其他logger
+    print("3. 测试其他组件logger...")
+    queue_logger = get_logger('QueueManager')
+    queue_logger.info("Queue initialized successfully Type: memory")
+    
+    scheduler_logger = get_logger('Scheduler')
+    scheduler_logger.info("enabled filters: crawlo.filters.memory_filter.MemoryFilter")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
+    test_framework_logger()

tests/test_framework_startup.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试框架启动信息是否正确输出到日志文件
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+def test_framework_startup():
+    print("=== 测试框架启动信息输出 ===")
+    
+    # 删除旧的日志文件
+    test_log_file = '/Users/oscar/projects/Crawlo/test_startup.log'
+    if os.path.exists(test_log_file):
+        os.remove(test_log_file)
+    
+    # 准备测试设置
+    test_settings = {
+        'PROJECT_NAME': 'test_startup',
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': test_log_file,
+        'RUN_MODE': 'standalone'
+    }
+    
+    # 初始化框架
+    from crawlo.core.framework_initializer import initialize_framework
+    settings = initialize_framework(test_settings)
+    
+    print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")
+    
+    # 检查日志文件是否包含框架启动信息
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容长度: {len(content)} 字符")
+            
+            # 检查关键的框架启动信息
+            if "Crawlo框架初始化完成" in content:
+                print("✅ 发现: Crawlo框架初始化完成")
+            else:
+                print("❌ 未找到: Crawlo框架初始化完成")
+                
+            if "Crawlo Framework Started" in content:
+                print("✅ 发现: Crawlo Framework Started")
+            else:
+                print("❌ 未找到: Crawlo Framework Started")
+                
+            if "使用单机模式" in content:
+                print("✅ 发现: 使用单机模式")
+            else:
+                print("❌ 未找到: 使用单机模式")
+                
+            print("\n前50行日志内容:")
+            lines = content.split('\n')[:50]
+            for i, line in enumerate(lines, 1):
+                if any(keyword in line for keyword in ["框架", "Framework", "Started"]):
+                    print(f"{i:3d}: {line}")
+    else:
+        print("❌ 日志文件未创建")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
+    test_framework_startup()

tests/test_large_scale_config.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+大规模配置工具测试
+测试 LargeScaleConfig, apply_large_scale_config
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.large_scale_config import LargeScaleConfig, apply_large_scale_config
+
+
+class TestLargeScaleConfig(unittest.TestCase):
+    """大规模配置工具测试"""
+
+    def test_conservative_config(self):
+        """测试保守配置"""
+        config = LargeScaleConfig.conservative_config(concurrency=8)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 8)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 80)  # 8 * 10
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.2)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 1)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 16)  # 8 * 2
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 2)
+        
+    def test_balanced_config(self):
+        """测试平衡配置"""
+        config = LargeScaleConfig.balanced_config(concurrency=16)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 16)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 240)  # 16 * 15
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.1)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 2)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 48)  # 16 * 3
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 3)
+        
+    def test_aggressive_config(self):
+        """测试激进配置"""
+        config = LargeScaleConfig.aggressive_config(concurrency=32)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 32)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 640)  # 32 * 20
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.05)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 3)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 128)  # 32 * 4
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 5)
+        
+    def test_memory_optimized_config(self):
+        """测试内存优化配置"""
+        config = LargeScaleConfig.memory_optimized_config(concurrency=12)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 12)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 60)  # 12 * 5
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.1)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 1)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 12)  # 12 * 1
+        
+        # 验证内存限制配置
+        self.assertEqual(config['DOWNLOAD_MAXSIZE'], 2 * 1024 * 1024)  # 2MB
+        self.assertEqual(config['DOWNLOAD_WARN_SIZE'], 512 * 1024)  # 512KB
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 2)
+        
+    def test_apply_large_scale_config(self):
+        """测试应用大规模配置"""
+        settings_dict = {}
+        
+        # 应用平衡配置
+        apply_large_scale_config(settings_dict, "balanced", 16)
+        
+        # 验证配置已应用
+        self.assertEqual(settings_dict['CONCURRENCY'], 16)
+        self.assertEqual(settings_dict['SCHEDULER_MAX_QUEUE_SIZE'], 240)
+        self.assertEqual(settings_dict['DOWNLOAD_DELAY'], 0.1)
+        
+    def test_apply_large_scale_config_invalid_type(self):
+        """测试应用无效的大规模配置类型"""
+        settings_dict = {}
+        
+        # 应用无效配置类型应该抛出异常
+        with self.assertRaises(ValueError) as context:
+            apply_large_scale_config(settings_dict, "invalid_type", 16)
+            
+        self.assertIn("不支持的配置类型", str(context.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()