crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/test_large_scale_config.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+大规模配置工具测试
+测试 LargeScaleConfig, apply_large_scale_config
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.large_scale_config import LargeScaleConfig, apply_large_scale_config
+
+
+class TestLargeScaleConfig(unittest.TestCase):
+    """大规模配置工具测试"""
+
+    def test_conservative_config(self):
+        """测试保守配置"""
+        config = LargeScaleConfig.conservative_config(concurrency=8)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 8)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 80)  # 8 * 10
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.2)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 1)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 16)  # 8 * 2
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 2)
+        
+    def test_balanced_config(self):
+        """测试平衡配置"""
+        config = LargeScaleConfig.balanced_config(concurrency=16)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 16)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 240)  # 16 * 15
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.1)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 2)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 48)  # 16 * 3
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 3)
+        
+    def test_aggressive_config(self):
+        """测试激进配置"""
+        config = LargeScaleConfig.aggressive_config(concurrency=32)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 32)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 640)  # 32 * 20
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.05)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 3)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 128)  # 32 * 4
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 5)
+        
+    def test_memory_optimized_config(self):
+        """测试内存优化配置"""
+        config = LargeScaleConfig.memory_optimized_config(concurrency=12)
+        
+        # 验证基本配置
+        self.assertEqual(config['CONCURRENCY'], 12)
+        self.assertEqual(config['SCHEDULER_MAX_QUEUE_SIZE'], 60)  # 12 * 5
+        self.assertEqual(config['DOWNLOAD_DELAY'], 0.1)
+        self.assertEqual(config['MAX_RUNNING_SPIDERS'], 1)
+        
+        # 验证连接池配置
+        self.assertEqual(config['CONNECTION_POOL_LIMIT'], 12)  # 12 * 1
+        
+        # 验证内存限制配置
+        self.assertEqual(config['DOWNLOAD_MAXSIZE'], 2 * 1024 * 1024)  # 2MB
+        self.assertEqual(config['DOWNLOAD_WARN_SIZE'], 512 * 1024)  # 512KB
+        
+        # 验证重试配置
+        self.assertEqual(config['MAX_RETRY_TIMES'], 2)
+        
+    def test_apply_large_scale_config(self):
+        """测试应用大规模配置"""
+        settings_dict = {}
+        
+        # 应用平衡配置
+        apply_large_scale_config(settings_dict, "balanced", 16)
+        
+        # 验证配置已应用
+        self.assertEqual(settings_dict['CONCURRENCY'], 16)
+        self.assertEqual(settings_dict['SCHEDULER_MAX_QUEUE_SIZE'], 240)
+        self.assertEqual(settings_dict['DOWNLOAD_DELAY'], 0.1)
+        
+    def test_apply_large_scale_config_invalid_type(self):
+        """测试应用无效的大规模配置类型"""
+        settings_dict = {}
+        
+        # 应用无效配置类型应该抛出异常
+        with self.assertRaises(ValueError) as context:
+            apply_large_scale_config(settings_dict, "invalid_type", 16)
+            
+        self.assertIn("不支持的配置类型", str(context.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_large_scale_helper.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+大规模爬虫辅助工具测试
+测试 LargeScaleHelper, ProgressManager, MemoryOptimizer, DataSourceAdapter, LargeScaleSpiderMixin
+"""
+import sys
+import os
+import unittest
+from unittest.mock import Mock, patch, MagicMock
+import asyncio
+import json
+import tempfile
+
+# 添加项目根目录到 Python 路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.large_scale_helper import (
+    LargeScaleHelper, 
+    ProgressManager, 
+    MemoryOptimizer, 
+    DataSourceAdapter, 
+    LargeScaleSpiderMixin
+)
+
+
+class TestLargeScaleHelper(unittest.TestCase):
+    """大规模爬虫辅助类测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.helper = LargeScaleHelper(batch_size=100, checkpoint_interval=500)
+        
+    def test_helper_initialization(self):
+        """测试辅助类初始化"""
+        self.assertEqual(self.helper.batch_size, 100)
+        self.assertEqual(self.helper.checkpoint_interval, 500)
+        
+    def test_batch_iterator_with_list(self):
+        """测试批次迭代器与列表数据源"""
+        data = list(range(250))  # 250个元素
+        batches = list(self.helper.batch_iterator(data))
+        
+        # 验证批次数量
+        self.assertEqual(len(batches), 3)  # 250/100 = 3批次(向上取整)
+        
+        # 验证每个批次的大小
+        self.assertEqual(len(batches[0]), 100)
+        self.assertEqual(len(batches[1]), 100)
+        self.assertEqual(len(batches[2]), 50)  # 最后一个批次
+        
+        # 验证数据完整性
+        all_data = []
+        for batch in batches:
+            all_data.extend(batch)
+        self.assertEqual(all_data, data)
+        
+    def test_batch_iterator_with_offset(self):
+        """测试批次迭代器与偏移量"""
+        data = list(range(250))  # 250个元素
+        batches = list(self.helper.batch_iterator(data, start_offset=50))
+        
+        # 验证批次数量
+        self.assertEqual(len(batches), 2)  # 剩余200个元素，2个批次
+        
+        # 验证数据正确性
+        all_data = []
+        for batch in batches:
+            all_data.extend(batch)
+        self.assertEqual(all_data, list(range(50, 250)))
+        
+    def test_batch_iterator_invalid_source(self):
+        """测试批次迭代器与无效数据源"""
+        with self.assertRaises(ValueError) as context:
+            list(self.helper.batch_iterator(123))  # 整数不是有效的数据源
+        self.assertIn("不支持的数据源类型", str(context.exception))
+
+
+class TestProgressManager(unittest.TestCase):
+    """进度管理器测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        # 创建临时文件用于测试
+        self.temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.json')
+        self.temp_file.close()
+        self.progress_manager = ProgressManager(self.temp_file.name)
+        
+    def tearDown(self):
+        """测试后清理"""
+        try:
+            os.unlink(self.temp_file.name)
+        except:
+            pass
+            
+    def test_progress_manager_initialization(self):
+        """测试进度管理器初始化"""
+        self.assertEqual(self.progress_manager.progress_file, self.temp_file.name)
+        
+    def test_save_and_load_progress(self):
+        """测试保存和加载进度"""
+        # 保存进度
+        self.progress_manager.save_progress(
+            batch_num=10,
+            processed_count=1000,
+            skipped_count=50
+        )
+        
+        # 加载进度
+        progress = self.progress_manager.load_progress()
+        
+        # 验证进度数据
+        self.assertEqual(progress['batch_num'], 10)
+        self.assertEqual(progress['processed_count'], 1000)
+        self.assertEqual(progress['skipped_count'], 50)
+        self.assertIn('timestamp', progress)
+        self.assertIn('formatted_time', progress)
+        
+    def test_load_progress_file_not_found(self):
+        """测试加载不存在的进度文件"""
+        # 创建一个新的进度管理器与不存在的文件
+        non_existent_file = tempfile.gettempdir() + "/non_existent_progress.json"
+        pm = ProgressManager(non_existent_file)
+        
+        # 加载进度应该返回默认值
+        progress = pm.load_progress()
+        self.assertEqual(progress['batch_num'], 0)
+        self.assertEqual(progress['processed_count'], 0)
+        self.assertEqual(progress['skipped_count'], 0)
+
+
+class TestMemoryOptimizer(unittest.TestCase):
+    """内存优化器测试"""
+
+    def setUp(self):
+        """测试前准备"""
+        self.optimizer = MemoryOptimizer(max_memory_mb=100)
+        
+    def test_optimizer_initialization(self):
+        """测试内存优化器初始化"""
+        self.assertEqual(self.optimizer.max_memory_mb, 100)
+        
+    def test_should_pause_for_memory_without_psutil(self):
+        """测试在没有psutil时的内存检查"""
+        # 在没有psutil的情况下，should_pause_for_memory应该返回False
+        result = self.optimizer.should_pause_for_memory()
+        self.assertFalse(result)
+        
+    def test_force_garbage_collection(self):
+        """测试强制垃圾回收"""
+        # 这个方法应该能正常执行而不抛出异常
+        try:
+            self.optimizer.force_garbage_collection()
+            success = True
+        except:
+            success = False
+        self.assertTrue(success)
+
+
+class TestDataSourceAdapter(unittest.TestCase):
+    """数据源适配器测试"""
+
+    def test_from_file_adapter(self):
+        """测试文件数据源适配器"""
+        # 创建临时文件
+        with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
+            for i in range(10):
+                f.write(f"line {i}\n")
+            temp_file_name = f.name
+            
+        try:
+            # 创建文件数据源适配器
+            adapter = DataSourceAdapter.from_file(temp_file_name, batch_size=5)
+            
+            # 获取第一批数据
+            batch = adapter(0, 5)
+            self.assertEqual(len(batch), 5)
+            self.assertEqual(batch[0], "line 0")
+            self.assertEqual(batch[4], "line 4")
+            
+            # 获取第二批数据
+            batch = adapter(5, 5)
+            self.assertEqual(len(batch), 5)
+            self.assertEqual(batch[0], "line 5")
+            self.assertEqual(batch[4], "line 9")
+            
+        finally:
+            # 清理临时文件
+            os.unlink(temp_file_name)
+
+
+class TestLargeScaleSpiderMixin(unittest.TestCase):
+    """大规模爬虫混入类测试"""
+
+    def test_mixin_initialization(self):
+        """测试混入类初始化"""
+        # 创建一个模拟的爬虫类
+        class MockSpider:
+            def __init__(self):
+                self.name = "test_spider"
+                
+        class TestSpider(MockSpider, LargeScaleSpiderMixin):
+            def __init__(self):
+                MockSpider.__init__(self)
+                LargeScaleSpiderMixin.__init__(self)
+                
+        spider = TestSpider()
+        
+        # 验证初始化
+        self.assertEqual(spider.name, "test_spider")
+        self.assertIsNotNone(spider.large_scale_helper)
+        self.assertIsNotNone(spider.progress_manager)
+        self.assertIsNotNone(spider.memory_optimizer)
+        
+    def test_mixin_attributes(self):
+        """测试混入类属性"""
+        # 创建一个模拟的爬虫类
+        class MockSpider:
+            def __init__(self):
+                self.name = "test_spider"
+                
+        class TestSpider(MockSpider, LargeScaleSpiderMixin):
+            def __init__(self):
+                MockSpider.__init__(self)
+                LargeScaleSpiderMixin.__init__(self)
+                
+        spider = TestSpider()
+        
+        # 验证属性
+        self.assertIsInstance(spider.large_scale_helper, LargeScaleHelper)
+        self.assertIsInstance(spider.progress_manager, ProgressManager)
+        self.assertIsInstance(spider.memory_optimizer, MemoryOptimizer)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_mode_change.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试运行模式日志级别修改
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+def test_mode_log_level():
+    print("=== 测试运行模式日志级别修改 ===")
+    
+    # 删除旧的日志文件
+    test_log_file = '/Users/oscar/projects/Crawlo/test_mode_change.log'
+    if os.path.exists(test_log_file):
+        os.remove(test_log_file)
+    
+    # 准备测试设置
+    test_settings = {
+        'PROJECT_NAME': 'test_mode_change',
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': test_log_file,
+        'RUN_MODE': 'standalone'
+    }
+    
+    try:
+        # 初始化框架
+        from crawlo.core.framework_initializer import initialize_framework
+        settings = initialize_framework(test_settings)
+        
+        print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")
+        
+        # 检查日志文件是否包含运行模式信息
+        if os.path.exists(test_log_file):
+            with open(test_log_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+                print(f"日志文件内容长度: {len(content)} 字符")
+                
+                # 检查是否还有INFO级别的运行模式信息
+                info_lines = [line for line in content.split('\n') if 'INFO' in line and '使用单机模式' in line]
+                debug_lines = [line for line in content.split('\n') if 'DEBUG' in line and '使用单机模式' in line]
+                
+                if info_lines:
+                    print("❌ 仍然发现INFO级别的运行模式信息:")
+                    for line in info_lines:
+                        print(f"   {line}")
+                else:
+                    print("✅ 没有发现INFO级别的运行模式信息")
+                
+                if debug_lines:
+                    print("✅ 发现DEBUG级别的运行模式信息:")
+                    for line in debug_lines:
+                        print(f"   {line}")
+                else:
+                    print("❌ 没有发现DEBUG级别的运行模式信息")
+                    
+                print("\n所有日志内容:")
+                lines = content.split('\n')
+                for i, line in enumerate(lines, 1):
+                    if line.strip():
+                        print(f"{i:3d}: {line}")
+        else:
+            print("❌ 日志文件未创建")
+    
+    except Exception as e:
+        print(f"错误: {e}")
+        import traceback
+        traceback.print_exc()
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
+    test_mode_log_level()

tests/test_mode_consistency.py

@@ -1,52 +1,52 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-测试模式一致性提示
-"""
-import asyncio
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.spider import Spider
-from crawlo import Request
-
-
-class TestSpider(Spider):
-    name = "test_mode_spider"
-    
-    def start_requests(self):
-        yield Request("https://httpbin.org/get")
-    
-    def parse(self, response):
-        yield {"url": response.url, "status": response.status}
-
-
-async def test_mode_consistency():
-    """测试模式一致性提示"""
-    print("测试模式一致性提示...")
-    
-    try:
-        # 创建爬虫进程
-        process = CrawlerProcess()
-        
-        # 添加爬虫
-        await process.crawl(TestSpider)
-        
-        print("模式一致性测试完成")
-        
-    except Exception as e:
-        print(f"测试失败: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-if __name__ == "__main__":
-    # 设置日志级别
-    import logging
-    logging.basicConfig(level=logging.INFO)
-    
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+测试模式一致性提示
+"""
+import asyncio
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.spider import Spider
+from crawlo import Request
+
+
+class TestSpider(Spider):
+    name = "test_mode_spider"
+    
+    def start_requests(self):
+        yield Request("https://httpbin.org/get")
+    
+    def parse(self, response):
+        yield {"url": response.url, "status": response.status_code}  # 修复：使用status_code而不是status
+
+
+async def test_mode_consistency():
+    """测试模式一致性提示"""
+    print("测试模式一致性提示...")
+    
+    try:
+        # 创建爬虫进程
+        process = CrawlerProcess()
+        
+        # 添加爬虫
+        await process.crawl(TestSpider)
+        
+        print("模式一致性测试完成")
+        
+    except Exception as e:
+        print(f"测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+
+
+if __name__ == "__main__":
+    # 设置日志级别
+    import logging
+    logging.basicConfig(level=logging.INFO)
+    
     asyncio.run(test_mode_consistency())