crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

tests/test_final_validation.py

@@ -1,154 +1,154 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
-"""
-import asyncio
-import pickle
-import sys
-sys.path.insert(0, "..")
-
-from crawlo.network.request import Request
-from crawlo.spider import Spider
-from crawlo.core.scheduler import Scheduler
-from crawlo.queue.redis_priority_queue import RedisPriorityQueue
-from crawlo.utils.log import get_logger
-from unittest.mock import Mock
-
-
-class TestSpider(Spider):
-    """测试爬虫"""
-    name = "validation_spider"
-    
-    def __init__(self):
-        super().__init__()
-        # 故意添加多个 logger 来测试清理
-        self.custom_logger = get_logger("custom")
-        self.debug_logger = get_logger("debug")
-        self.nested_data = {
-            'logger': get_logger("nested"),
-            'sub': {
-                'logger_ref': get_logger("sub_logger")
-            }
-        }
-    
-    def parse(self, response):
-        # 验证主 logger 还在
-        self.logger.info(f"主 logger 工作正常: {response.url}")
-        return {"url": response.url, "status": "success"}
-
-
-def test_scheduler_cleaning():
-    """测试调度器的 logger 清理"""
-    print("测试调度器 logger 清理...")
-    
-    spider = TestSpider()
-    request = Request(
-        url="https://scheduler-test.com",
-        callback=spider.parse,
-        meta={"logger": get_logger("meta_logger")}
-    )
-    
-    # Mock crawler 和 scheduler
-    class MockCrawler:
-        def __init__(self):
-            self.spider = spider
-    
-    class MockScheduler(Scheduler):
-        def __init__(self):
-            self.crawler = MockCrawler()
-            self.logger = get_logger("MockScheduler")
-    
-    scheduler = MockScheduler()
-    
-    # 清理前检查
-    print(f"   清理前 - spider.logger: {spider.logger is not None}")
-    print(f"   清理前 - spider.custom_logger: {spider.custom_logger is not None}")
-    print(f"   清理前 - request.callback: {request.callback is not None}")
-    
-    # 执行清理
-    cleaned_request = scheduler._deep_clean_loggers(request)
-    
-    # 清理后检查
-    print(f"   清理后 - spider.logger: {spider.logger is not None}")
-    print(f"   清理后 - spider.custom_logger: {spider.custom_logger is None}")
-    print(f"   清理后 - request.callback: {cleaned_request.callback is None}")
-    
-    # 序列化测试
-    try:
-        serialized = pickle.dumps(cleaned_request)
-        print(f"   调度器清理后序列化成功，大小: {len(serialized)} bytes")
-        return True
-    except Exception as e:
-        print(f"   调度器清理后序列化失败: {e}")
-        return False
-
-
-async def test_redis_queue_cleaning():
-    """测试 Redis 队列的 logger 清理"""
-    print("\\n测试 Redis 队列 logger 清理...")
-    
-    spider = TestSpider()
-    request = Request(
-        url="https://redis-test.com",
-        callback=spider.parse,
-        meta={"logger": get_logger("meta_logger")}
-    )
-    
-    try:
-        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
-        await queue.connect()
-        
-        # 入队测试
-        success = await queue.put(request, priority=0)
-        print(f"   Redis 队列入队成功: {success}")
-        
-        if success:
-            # 出队测试
-            retrieved = await queue.get(timeout=2.0)
-            if retrieved:
-                print(f"   Redis 队列出队成功: {retrieved.url}")
-                print(f"   callback 信息保存: {'_callback_info' in retrieved.meta}")
-                await queue.close()
-                return True
-            else:
-                print("   出队失败")
-                await queue.close()
-                return False
-        else:
-            await queue.close()
-            return False
-            
-    except Exception as e:
-        print(f"   Redis 队列测试失败: {e}")
-        return False
-
-
-async def main():
-    """主测试函数"""
-    print("开始最终验证测试...")
-    print("=" * 60)
-    
-    # 测试 1: 调度器清理
-    scheduler_ok = test_scheduler_cleaning()
-    
-    # 测试 2: Redis 队列清理
-    redis_ok = await test_redis_queue_cleaning()
-    
-    print("\\n" + "=" * 60)
-    print("测试结果汇总:")
-    print(f"   调度器 logger 清理: {'通过' if scheduler_ok else '失败'}")
-    print(f"   Redis 队列清理: {'通过' if redis_ok else '失败'}")
-    
-    if scheduler_ok and redis_ok:
-        print("\\n所有测试通过！")
-        print("分布式队列的 logger 序列化问题已完全修复！")
-        print("Crawlo 现在可以正常使用 Redis 分布式队列了！")
-        return True
-    else:
-        print("\\n部分测试失败，需要进一步修复")
-        return False
-
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+最终验证测试：确认分布式队列的 logger 序列化问题已完全解决
+"""
+import asyncio
+import pickle
+import sys
+sys.path.insert(0, "..")
+
+from crawlo.network.request import Request
+from crawlo.spider import Spider
+from crawlo.core.scheduler import Scheduler
+from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+from crawlo.utils.log import get_logger
+from unittest.mock import Mock
+
+
+class TestSpider(Spider):
+    """测试爬虫"""
+    name = "validation_spider"
+    
+    def __init__(self):
+        super().__init__()
+        # 故意添加多个 logger 来测试清理
+        self.custom_logger = get_logger("custom")
+        self.debug_logger = get_logger("debug")
+        self.nested_data = {
+            'logger': get_logger("nested"),
+            'sub': {
+                'logger_ref': get_logger("sub_logger")
+            }
+        }
+    
+    def parse(self, response):
+        # 验证主 logger 还在
+        self.logger.info(f"主 logger 工作正常: {response.url}")
+        return {"url": response.url, "status": "success"}
+
+
+def test_scheduler_cleaning():
+    """测试调度器的 logger 清理"""
+    print("测试调度器 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://scheduler-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    # Mock crawler 和 scheduler
+    class MockCrawler:
+        def __init__(self):
+            self.spider = spider
+    
+    class MockScheduler(Scheduler):
+        def __init__(self):
+            self.crawler = MockCrawler()
+            self.logger = get_logger("MockScheduler")
+    
+    scheduler = MockScheduler()
+    
+    # 清理前检查
+    print(f"   清理前 - spider.logger: {spider.logger is not None}")
+    print(f"   清理前 - spider.custom_logger: {spider.custom_logger is not None}")
+    print(f"   清理前 - request.callback: {request.callback is not None}")
+    
+    # 执行清理
+    cleaned_request = scheduler._deep_clean_loggers(request)
+    
+    # 清理后检查
+    print(f"   清理后 - spider.logger: {spider.logger is not None}")
+    print(f"   清理后 - spider.custom_logger: {spider.custom_logger is None}")
+    print(f"   清理后 - request.callback: {cleaned_request.callback is None}")
+    
+    # 序列化测试
+    try:
+        serialized = pickle.dumps(cleaned_request)
+        print(f"   调度器清理后序列化成功，大小: {len(serialized)} bytes")
+        return True
+    except Exception as e:
+        print(f"   调度器清理后序列化失败: {e}")
+        return False
+
+
+async def test_redis_queue_cleaning():
+    """测试 Redis 队列的 logger 清理"""
+    print("\\n测试 Redis 队列 logger 清理...")
+    
+    spider = TestSpider()
+    request = Request(
+        url="https://redis-test.com",
+        callback=spider.parse,
+        meta={"logger": get_logger("meta_logger")}
+    )
+    
+    try:
+        queue = RedisPriorityQueue(redis_url="redis://127.0.0.1:6379/0")
+        await queue.connect()
+        
+        # 入队测试
+        success = await queue.put(request, priority=0)
+        print(f"   Redis 队列入队成功: {success}")
+        
+        if success:
+            # 出队测试
+            retrieved = await queue.get(timeout=2.0)
+            if retrieved:
+                print(f"   Redis 队列出队成功: {retrieved.url}")
+                print(f"   callback 信息保存: {'_callback_info' in retrieved.meta}")
+                await queue.close()
+                return True
+            else:
+                print("   出队失败")
+                await queue.close()
+                return False
+        else:
+            await queue.close()
+            return False
+            
+    except Exception as e:
+        print(f"   Redis 队列测试失败: {e}")
+        return False
+
+
+async def main():
+    """主测试函数"""
+    print("开始最终验证测试...")
+    print("=" * 60)
+    
+    # 测试 1: 调度器清理
+    scheduler_ok = test_scheduler_cleaning()
+    
+    # 测试 2: Redis 队列清理
+    redis_ok = await test_redis_queue_cleaning()
+    
+    print("\\n" + "=" * 60)
+    print("测试结果汇总:")
+    print(f"   调度器 logger 清理: {'通过' if scheduler_ok else '失败'}")
+    print(f"   Redis 队列清理: {'通过' if redis_ok else '失败'}")
+    
+    if scheduler_ok and redis_ok:
+        print("\\n所有测试通过！")
+        print("分布式队列的 logger 序列化问题已完全修复！")
+        print("Crawlo 现在可以正常使用 Redis 分布式队列了！")
+        return True
+    else:
+        print("\\n部分测试失败，需要进一步修复")
+        return False
+
+
+if __name__ == "__main__":
     asyncio.run(main())

tests/test_framework_env_usage.py

@@ -1,104 +1,104 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-框架环境变量使用测试
-验证整个框架中环境变量的正确使用
-"""
-import sys
-import os
-import unittest
-from unittest.mock import patch, MagicMock
-
-# 添加项目根目录到Python路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
-from crawlo.settings.setting_manager import SettingManager
-from crawlo.settings import default_settings
-from crawlo.mode_manager import from_env
-
-
-class TestFrameworkEnvUsage(unittest.TestCase):
-    """框架环境变量使用测试"""
-
-    def test_default_settings_env_usage(self):
-        """测试 default_settings.py 中的环境变量使用"""
-        # 验证 default_settings.py 不直接使用 os.getenv
-        import inspect
-        import crawlo.settings.default_settings as default_settings_module
-        
-        source_code = inspect.getsource(default_settings_module)
-        # 检查是否还有直接使用 os.getenv 的地方
-        self.assertNotIn('os.getenv', source_code, 
-                         "default_settings.py 不应该直接使用 os.getenv")
-        
-        # 但应该使用 env_config 工具
-        self.assertIn('get_redis_config', source_code,
-                      "default_settings.py 应该使用 get_redis_config")
-        self.assertIn('get_runtime_config', source_code,
-                      "default_settings.py 应该使用 get_runtime_config")
-
-    def test_env_config_tool(self):
-        """测试环境变量配置工具"""
-        # 测试获取Redis配置
-        with patch.dict(os.environ, {
-            'REDIS_HOST': 'test.redis.com',
-            'REDIS_PORT': '6380',
-            'REDIS_PASSWORD': 'test_pass',
-            'REDIS_DB': '2'
-        }):
-            redis_config = get_redis_config()
-            self.assertEqual(redis_config['REDIS_HOST'], 'test.redis.com')
-            self.assertEqual(redis_config['REDIS_PORT'], 6380)
-            self.assertEqual(redis_config['REDIS_PASSWORD'], 'test_pass')
-            self.assertEqual(redis_config['REDIS_DB'], 2)
-        
-        # 测试获取运行时配置
-        with patch.dict(os.environ, {
-            'PROJECT_NAME': 'test_project',
-            'CRAWLO_MODE': 'distributed',
-            'CONCURRENCY': '16'
-        }):
-            runtime_config = get_runtime_config()
-            self.assertEqual(runtime_config['PROJECT_NAME'], 'test_project')
-            self.assertEqual(runtime_config['CRAWLO_MODE'], 'distributed')
-            self.assertEqual(runtime_config['CONCURRENCY'], 16)
-
-    def test_settings_manager_with_env(self):
-        """测试设置管理器与环境变量的集成"""
-        # 设置环境变量
-        env_vars = {
-            'PROJECT_NAME': 'env_test_project',
-            'CONCURRENCY': '12',
-            'REDIS_HOST': 'env.redis.test',
-            'REDIS_PORT': '6381'
-        }
-        
-        with patch.dict(os.environ, env_vars):
-            # 重新导入 default_settings 模块以获取最新的环境变量
-            import importlib
-            import crawlo.settings.default_settings
-            importlib.reload(crawlo.settings.default_settings)
-            
-            # 创建设置管理器
-            settings = SettingManager()
-            settings.set_settings(crawlo.settings.default_settings)
-            
-            # 验证环境变量被正确使用
-            redis_config = get_redis_config()
-            self.assertEqual(settings.get('REDIS_HOST'), redis_config['REDIS_HOST'])
-            
-            runtime_config = get_runtime_config()
-            self.assertEqual(settings.get('PROJECT_NAME'), runtime_config['PROJECT_NAME'])
-
-    def test_mode_manager_env_usage(self):
-        """测试 mode_manager.py 中的环境变量使用"""
-        # 验证 from_env 函数现在会抛出异常
-        with self.assertRaises(RuntimeError) as context:
-            from_env()
-        
-        self.assertIn("环境变量配置已移除", str(context.exception))
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+框架环境变量使用测试
+验证整个框架中环境变量的正确使用
+"""
+import sys
+import os
+import unittest
+from unittest.mock import patch, MagicMock
+
+# 添加项目根目录到Python路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.utils.env_config import get_env_var, get_redis_config, get_runtime_config
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.settings import default_settings
+from crawlo.mode_manager import from_env
+
+
+class TestFrameworkEnvUsage(unittest.TestCase):
+    """框架环境变量使用测试"""
+
+    def test_default_settings_env_usage(self):
+        """测试 default_settings.py 中的环境变量使用"""
+        # 验证 default_settings.py 不直接使用 os.getenv
+        import inspect
+        import crawlo.settings.default_settings as default_settings_module
+        
+        source_code = inspect.getsource(default_settings_module)
+        # 检查是否还有直接使用 os.getenv 的地方
+        self.assertNotIn('os.getenv', source_code, 
+                         "default_settings.py 不应该直接使用 os.getenv")
+        
+        # 但应该使用 env_config 工具
+        self.assertIn('get_redis_config', source_code,
+                      "default_settings.py 应该使用 get_redis_config")
+        self.assertIn('get_runtime_config', source_code,
+                      "default_settings.py 应该使用 get_runtime_config")
+
+    def test_env_config_tool(self):
+        """测试环境变量配置工具"""
+        # 测试获取Redis配置
+        with patch.dict(os.environ, {
+            'REDIS_HOST': 'test.redis.com',
+            'REDIS_PORT': '6380',
+            'REDIS_PASSWORD': 'test_pass',
+            'REDIS_DB': '2'
+        }):
+            redis_config = get_redis_config()
+            self.assertEqual(redis_config['REDIS_HOST'], 'test.redis.com')
+            self.assertEqual(redis_config['REDIS_PORT'], 6380)
+            self.assertEqual(redis_config['REDIS_PASSWORD'], 'test_pass')
+            self.assertEqual(redis_config['REDIS_DB'], 2)
+        
+        # 测试获取运行时配置
+        with patch.dict(os.environ, {
+            'PROJECT_NAME': 'test_project',
+            'CRAWLO_MODE': 'distributed',
+            'CONCURRENCY': '16'
+        }):
+            runtime_config = get_runtime_config()
+            self.assertEqual(runtime_config['PROJECT_NAME'], 'test_project')
+            self.assertEqual(runtime_config['CRAWLO_MODE'], 'distributed')
+            self.assertEqual(runtime_config['CONCURRENCY'], 16)
+
+    def test_settings_manager_with_env(self):
+        """测试设置管理器与环境变量的集成"""
+        # 设置环境变量
+        env_vars = {
+            'PROJECT_NAME': 'env_test_project',
+            'CONCURRENCY': '12',
+            'REDIS_HOST': 'env.redis.test',
+            'REDIS_PORT': '6381'
+        }
+        
+        with patch.dict(os.environ, env_vars):
+            # 重新导入 default_settings 模块以获取最新的环境变量
+            import importlib
+            import crawlo.settings.default_settings
+            importlib.reload(crawlo.settings.default_settings)
+            
+            # 创建设置管理器
+            settings = SettingManager()
+            settings.set_settings(crawlo.settings.default_settings)
+            
+            # 验证环境变量被正确使用
+            redis_config = get_redis_config()
+            self.assertEqual(settings.get('REDIS_HOST'), redis_config['REDIS_HOST'])
+            
+            runtime_config = get_runtime_config()
+            self.assertEqual(settings.get('PROJECT_NAME'), runtime_config['PROJECT_NAME'])
+
+    def test_mode_manager_env_usage(self):
+        """测试 mode_manager.py 中的环境变量使用"""
+        # 验证 from_env 函数现在会抛出异常
+        with self.assertRaises(RuntimeError) as context:
+            from_env()
+        
+        self.assertIn("环境变量配置已移除", str(context.exception))
+
+
+if __name__ == '__main__':
     unittest.main()

tests/test_framework_logger.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+测试框架日志系统
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.core.framework_initializer import initialize_framework, get_framework_initializer
+from crawlo.utils.log import get_logger, LoggerManager
+
+def test_framework_logger():
+    print("=== 测试框架日志系统 ===")
+    
+    # 1. 初始化框架，传入基本配置
+    print("1. 初始化框架...")
+    custom_settings = {
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': 'test_framework.log',
+        'PROJECT_NAME': 'test_framework',
+        'RUN_MODE': 'standalone'
+    }
+    settings = initialize_framework(custom_settings)
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    
+    # 2. 获取框架初始化管理器
+    init_manager = get_framework_initializer()
+    print(f"   框架是否就绪: {init_manager.is_ready}")
+    print(f"   初始化阶段: {init_manager.phase}")
+    
+    # 3. 测试框架logger
+    framework_logger = init_manager.logger
+    if framework_logger:
+        print(f"   框架logger名称: {framework_logger.name}")
+        print(f"   框架logger级别: {framework_logger.level}")
+        print(f"   框架logger处理器数量: {len(framework_logger.handlers)}")
+        
+        for i, handler in enumerate(framework_logger.handlers):
+            handler_type = type(handler).__name__
+            print(f"     处理器{i}: {handler_type}, 级别: {handler.level}")
+    else:
+        print("   框架logger为None!")
+        framework_logger = get_logger('crawlo.framework')
+        print(f"   手动创建的框架logger: {framework_logger.name}")
+    
+    # 4. 测试日志输出
+    print("2. 测试日志输出...")
+    framework_logger.info("Crawlo框架初始化完成")
+    framework_logger.info("Crawlo Framework Started 1.3.3")
+    framework_logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+    framework_logger.info("Run Mode: standalone")
+    framework_logger.info("Starting running test_spider")
+    
+    # 5. 测试其他logger
+    print("3. 测试其他组件logger...")
+    queue_logger = get_logger('QueueManager')
+    queue_logger.info("Queue initialized successfully Type: memory")
+    
+    scheduler_logger = get_logger('Scheduler')
+    scheduler_logger.info("enabled filters: crawlo.filters.memory_filter.MemoryFilter")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
+    test_framework_logger()

tests/test_framework_startup.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+测试框架启动信息是否正确输出到日志文件
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+def test_framework_startup():
+    print("=== 测试框架启动信息输出 ===")
+    
+    # 删除旧的日志文件
+    test_log_file = '/Users/oscar/projects/Crawlo/test_startup.log'
+    if os.path.exists(test_log_file):
+        os.remove(test_log_file)
+    
+    # 准备测试设置
+    test_settings = {
+        'PROJECT_NAME': 'test_startup',
+        'LOG_LEVEL': 'INFO',
+        'LOG_FILE': test_log_file,
+        'RUN_MODE': 'standalone'
+    }
+    
+    # 初始化框架
+    from crawlo.core.framework_initializer import initialize_framework
+    settings = initialize_framework(test_settings)
+    
+    print(f"设置初始化完成: {settings.get('PROJECT_NAME')}")
+    
+    # 检查日志文件是否包含框架启动信息
+    if os.path.exists(test_log_file):
+        with open(test_log_file, 'r', encoding='utf-8') as f:
+            content = f.read()
+            print(f"日志文件内容长度: {len(content)} 字符")
+            
+            # 检查关键的框架启动信息
+            if "Crawlo框架初始化完成" in content:
+                print("✅ 发现: Crawlo框架初始化完成")
+            else:
+                print("❌ 未找到: Crawlo框架初始化完成")
+                
+            if "Crawlo Framework Started" in content:
+                print("✅ 发现: Crawlo Framework Started")
+            else:
+                print("❌ 未找到: Crawlo Framework Started")
+                
+            if "使用单机模式" in content:
+                print("✅ 发现: 使用单机模式")
+            else:
+                print("❌ 未找到: 使用单机模式")
+                
+            print("\n前50行日志内容:")
+            lines = content.split('\n')[:50]
+            for i, line in enumerate(lines, 1):
+                if any(keyword in line for keyword in ["框架", "Framework", "Started"]):
+                    print(f"{i:3d}: {line}")
+    else:
+        print("❌ 日志文件未创建")
+    
+    print("=== 测试完成 ===")
+
+if __name__ == "__main__":
+    test_framework_startup()