crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

tests/debug_log_config.py

@@ -1,127 +1,127 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-调试日志配置问题
-"""
-
-import os
-import sys
-from pathlib import Path
-
-# 添加项目根目录到Python路径
-project_root = Path(__file__).parent.parent
-sys.path.insert(0, str(project_root))
-
-from crawlo.logging import configure_logging as configure, get_logger, LogManager
-from crawlo.logging.config import LogConfig
-
-
-def debug_log_configuration():
-    """调试日志配置"""
-    print("=== 调试日志配置 ===")
-    
-    # 重置配置
-    LogManager().reset()
-    
-    # 1. 检查初始状态
-    print("1. 检查初始状态...")
-    print(f"   初始配置状态: {LogManager().is_configured}")
-    if LogManager().config:
-        print(f"   初始配置: {LogManager().config}")
-    
-    # 2. 配置日志系统
-    print("2. 配置日志系统...")
-    config = configure(
-        LOG_LEVEL='DEBUG',
-        LOG_FILE='debug_test.log',
-        LOG_MAX_BYTES=1024,
-        LOG_BACKUP_COUNT=2,
-        LOG_CONSOLE_ENABLED=True,
-        LOG_FILE_ENABLED=True
-    )
-    
-    print(f"   配置返回值: {config}")
-    print(f"   配置类型: {type(config)}")
-    print(f"   配置级别: {config.level}")
-    print(f"   配置文件路径: {config.file_path}")
-    print(f"   轮转大小: {config.max_bytes}")
-    print(f"   备份数量: {config.backup_count}")
-    print(f"   控制台启用: {config.console_enabled}")
-    print(f"   文件启用: {config.file_enabled}")
-    
-    # 3. 检查管理器状态
-    print("3. 检查管理器状态...")
-    manager = LogManager()
-    print(f"   管理器配置状态: {manager.is_configured}")
-    if manager.config:
-        print(f"   管理器配置: {manager.config}")
-        print(f"   管理器配置文件路径: {manager.config.file_path}")
-    
-    # 4. 测试Logger创建
-    print("4. 测试Logger创建...")
-    logger = get_logger('test.debug')
-    print(f"   Logger handlers数量: {len(logger.handlers)}")
-    
-    for i, handler in enumerate(logger.handlers):
-        handler_type = type(handler).__name__
-        print(f"     Handler {i}: {handler_type}")
-        if hasattr(handler, 'baseFilename'):
-            print(f"       文件名: {handler.baseFilename}")
-    
-    # 5. 测试日志输出
-    print("5. 测试日志输出...")
-    logger.info("调试测试消息")
-
-
-def test_config_from_dict():
-    """测试从字典创建配置"""
-    print("\n=== 测试从字典创建配置 ===")
-    
-    LogManager().reset()
-    
-    # 使用字典配置
-    config_dict = {
-        'level': 'DEBUG',
-        'file_path': 'dict_test.log',
-        'max_bytes': 1024,
-        'backup_count': 2,
-        'console_enabled': True,
-        'file_enabled': True
-    }
-    
-    config = LogConfig.from_dict(config_dict)
-    print(f"   字典配置: {config}")
-    print(f"   验证结果: {config.validate()}")
-    
-    # 应用配置
-    LogManager().configure(config)
-    
-    logger = get_logger('test.dict')
-    print(f"   Logger handlers数量: {len(logger.handlers)}")
-    
-    for i, handler in enumerate(logger.handlers):
-        handler_type = type(handler).__name__
-        print(f"     Handler {i}: {handler_type}")
-
-
-def main():
-    """主函数"""
-    print("开始调试日志配置问题...")
-    
-    try:
-        debug_log_configuration()
-        test_config_from_dict()
-        
-        print("\n=== 调试完成 ===")
-        
-    except Exception as e:
-        print(f"\n调试过程中出现错误: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-    
-    return 0
-
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试日志配置问题
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger, LogManager
+from crawlo.logging.config import LogConfig
+
+
+def debug_log_configuration():
+    """调试日志配置"""
+    print("=== 调试日志配置 ===")
+    
+    # 重置配置
+    LogManager().reset()
+    
+    # 1. 检查初始状态
+    print("1. 检查初始状态...")
+    print(f"   初始配置状态: {LogManager().is_configured}")
+    if LogManager().config:
+        print(f"   初始配置: {LogManager().config}")
+    
+    # 2. 配置日志系统
+    print("2. 配置日志系统...")
+    config = configure(
+        LOG_LEVEL='DEBUG',
+        LOG_FILE='debug_test.log',
+        LOG_MAX_BYTES=1024,
+        LOG_BACKUP_COUNT=2,
+        LOG_CONSOLE_ENABLED=True,
+        LOG_FILE_ENABLED=True
+    )
+    
+    print(f"   配置返回值: {config}")
+    print(f"   配置类型: {type(config)}")
+    print(f"   配置级别: {config.level}")
+    print(f"   配置文件路径: {config.file_path}")
+    print(f"   轮转大小: {config.max_bytes}")
+    print(f"   备份数量: {config.backup_count}")
+    print(f"   控制台启用: {config.console_enabled}")
+    print(f"   文件启用: {config.file_enabled}")
+    
+    # 3. 检查管理器状态
+    print("3. 检查管理器状态...")
+    manager = LogManager()
+    print(f"   管理器配置状态: {manager.is_configured}")
+    if manager.config:
+        print(f"   管理器配置: {manager.config}")
+        print(f"   管理器配置文件路径: {manager.config.file_path}")
+    
+    # 4. 测试Logger创建
+    print("4. 测试Logger创建...")
+    logger = get_logger('test.debug')
+    print(f"   Logger handlers数量: {len(logger.handlers)}")
+    
+    for i, handler in enumerate(logger.handlers):
+        handler_type = type(handler).__name__
+        print(f"     Handler {i}: {handler_type}")
+        if hasattr(handler, 'baseFilename'):
+            print(f"       文件名: {handler.baseFilename}")
+    
+    # 5. 测试日志输出
+    print("5. 测试日志输出...")
+    logger.info("调试测试消息")
+
+
+def test_config_from_dict():
+    """测试从字典创建配置"""
+    print("\n=== 测试从字典创建配置 ===")
+    
+    LogManager().reset()
+    
+    # 使用字典配置
+    config_dict = {
+        'level': 'DEBUG',
+        'file_path': 'dict_test.log',
+        'max_bytes': 1024,
+        'backup_count': 2,
+        'console_enabled': True,
+        'file_enabled': True
+    }
+    
+    config = LogConfig.from_dict(config_dict)
+    print(f"   字典配置: {config}")
+    print(f"   验证结果: {config.validate()}")
+    
+    # 应用配置
+    LogManager().configure(config)
+    
+    logger = get_logger('test.dict')
+    print(f"   Logger handlers数量: {len(logger.handlers)}")
+    
+    for i, handler in enumerate(logger.handlers):
+        handler_type = type(handler).__name__
+        print(f"     Handler {i}: {handler_type}")
+
+
+def main():
+    """主函数"""
+    print("开始调试日志配置问题...")
+    
+    try:
+        debug_log_configuration()
+        test_config_from_dict()
+        
+        print("\n=== 调试完成 ===")
+        
+    except Exception as e:
+        print(f"\n调试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
     sys.exit(main())

tests/debug_log_levels.py

@@ -1,64 +1,64 @@
-#!/usr/bin/env python
-# -*- coding: UTF-8 -*-
-"""
-调试日志级别配置脚本
-"""
-import sys
-import os
-sys.path.insert(0, '/')
-
-from crawlo.initialization import initialize_framework
-from crawlo.utils.log import LoggerManager, get_logger
-import logging
-
-def main():
-    print("=== 开始调试日志级别配置 ===")
-    
-    # 初始化框架
-    print("1. 初始化框架...")
-    settings = initialize_framework()
-    
-    # 打印配置信息
-    print(f"2. 配置信息:")
-    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
-    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
-    print(f"   LoggerManager._default_level: {LoggerManager._default_level}")
-    print(f"   LoggerManager._default_console_level: {LoggerManager._default_console_level}")
-    print(f"   LoggerManager._default_file_level: {LoggerManager._default_file_level}")
-    
-    # 测试不同组件的日志级别
-    components = [
-        'crawlo.framework',
-        'crawlo.crawler', 
-        'QueueManager',
-        'Scheduler',
-        'AioHttpDownloader',
-        'MiddlewareManager',
-        'PipelineManager',
-        'ExtensionManager',
-        'of_week_standalone'
-    ]
-    
-    print("3. 组件日志级别测试:")
-    for component_name in components:
-        logger = get_logger(component_name)
-        print(f"   {component_name}:")
-        print(f"     logger.level: {logger.level} ({logging.getLevelName(logger.level)})")
-        
-        for handler in logger.handlers:
-            handler_type = type(handler).__name__
-            print(f"     {handler_type}.level: {handler.level} ({logging.getLevelName(handler.level)})")
-    
-    # 实际测试日志输出
-    print("4. 测试日志输出:")
-    test_logger = get_logger('TestLogger')
-    
-    print("   控制台应该看到以下日志:")
-    test_logger.debug("这是DEBUG级别日志 - 控制台应该看不到")
-    test_logger.info("这是INFO级别日志 - 控制台应该能看到")
-    test_logger.warning("这是WARNING级别日志 - 控制台应该能看到")
-    
-    print("=== 调试完成 ===")
-
-if __name__ == '__main__':
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试日志级别配置脚本
+"""
+import sys
+import os
+sys.path.insert(0, '/')
+
+from crawlo.initialization import initialize_framework
+from crawlo.utils.log import LoggerManager, get_logger
+import logging
+
+def main():
+    print("=== 开始调试日志级别配置 ===")
+    
+    # 初始化框架
+    print("1. 初始化框架...")
+    settings = initialize_framework()
+    
+    # 打印配置信息
+    print(f"2. 配置信息:")
+    print(f"   LOG_LEVEL: {settings.get('LOG_LEVEL')}")
+    print(f"   LOG_FILE: {settings.get('LOG_FILE')}")
+    print(f"   LoggerManager._default_level: {LoggerManager._default_level}")
+    print(f"   LoggerManager._default_console_level: {LoggerManager._default_console_level}")
+    print(f"   LoggerManager._default_file_level: {LoggerManager._default_file_level}")
+    
+    # 测试不同组件的日志级别
+    components = [
+        'crawlo.framework',
+        'crawlo.crawler', 
+        'QueueManager',
+        'Scheduler',
+        'AioHttpDownloader',
+        'MiddlewareManager',
+        'PipelineManager',
+        'ExtensionManager',
+        'of_week_standalone'
+    ]
+    
+    print("3. 组件日志级别测试:")
+    for component_name in components:
+        logger = get_logger(component_name)
+        print(f"   {component_name}:")
+        print(f"     logger.level: {logger.level} ({logging.getLevelName(logger.level)})")
+        
+        for handler in logger.handlers:
+            handler_type = type(handler).__name__
+            print(f"     {handler_type}.level: {handler.level} ({logging.getLevelName(handler.level)})")
+    
+    # 实际测试日志输出
+    print("4. 测试日志输出:")
+    test_logger = get_logger('TestLogger')
+    
+    print("   控制台应该看到以下日志:")
+    test_logger.debug("这是DEBUG级别日志 - 控制台应该看不到")
+    test_logger.info("这是INFO级别日志 - 控制台应该能看到")
+    test_logger.warning("这是WARNING级别日志 - 控制台应该能看到")
+    
+    print("=== 调试完成 ===")
+
+if __name__ == '__main__':
     main()

tests/debug_pipelines.py

@@ -1,67 +1,67 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-调试管道配置
-查看实际的管道配置合并结果
-"""
-
-import sys
-import os
-
-# 添加项目根目录到路径
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
-
-from crawlo.settings.setting_manager import SettingManager
-
-def debug_pipelines():
-    """调试管道配置"""
-    print("调试管道配置合并...")
-    print("=" * 50)
-    
-    # 用户自定义配置
-    user_config = {
-        'PIPELINES': [
-            'myproject.pipelines.CustomPipeline',
-        ]
-    }
-    
-    settings = SettingManager(user_config)
-    
-    # 获取合并后的管道列表
-    pipelines = settings.get('PIPELINES')
-    
-    print("合并后的管道列表:")
-    for i, pipeline in enumerate(pipelines):
-        print(f"  {i}: {pipeline}")
-    
-    print()
-    print("默认去重管道:")
-    dedup_pipeline = settings.get('DEFAULT_DEDUP_PIPELINE')
-    print(f"  {dedup_pipeline}")
-    
-    print()
-    print("框架默认管道:")
-    default_pipelines = settings.get('PIPELINES', [])  # 直接获取PIPELINES，它已经包含了默认管道
-    # 从合并后的管道中移除去重管道，得到框架默认管道
-    if dedup_pipeline:
-        default_pipelines_without_dedup = [p for p in default_pipelines if p != dedup_pipeline]
-        for i, pipeline in enumerate(default_pipelines_without_dedup):
-            print(f"  {i}: {pipeline}")
-    else:
-        for i, pipeline in enumerate(default_pipelines):
-            print(f"  {i}: {pipeline}")
-    
-    print()
-    print("自定义管道:")
-    custom_pipelines = settings.get('PIPELINES')
-    # 从合并后的管道中移除默认管道，得到自定义管道
-    default_pipelines_list = [
-        'crawlo.pipelines.console_pipeline.ConsolePipeline',
-        'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
-    ]
-    custom_pipelines_list = [p for p in custom_pipelines if p not in default_pipelines_list]
-    for i, pipeline in enumerate(custom_pipelines_list):
-        print(f"  {i}: {pipeline}")
-
-if __name__ == "__main__":
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+调试管道配置
+查看实际的管道配置合并结果
+"""
+
+import sys
+import os
+
+# 添加项目根目录到路径
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
+
+from crawlo.settings.setting_manager import SettingManager
+
+def debug_pipelines():
+    """调试管道配置"""
+    print("调试管道配置合并...")
+    print("=" * 50)
+    
+    # 用户自定义配置
+    user_config = {
+        'PIPELINES': [
+            'myproject.pipelines.CustomPipeline',
+        ]
+    }
+    
+    settings = SettingManager(user_config)
+    
+    # 获取合并后的管道列表
+    pipelines = settings.get('PIPELINES')
+    
+    print("合并后的管道列表:")
+    for i, pipeline in enumerate(pipelines):
+        print(f"  {i}: {pipeline}")
+    
+    print()
+    print("默认去重管道:")
+    dedup_pipeline = settings.get('DEFAULT_DEDUP_PIPELINE')
+    print(f"  {dedup_pipeline}")
+    
+    print()
+    print("框架默认管道:")
+    default_pipelines = settings.get('PIPELINES', [])  # 直接获取PIPELINES，它已经包含了默认管道
+    # 从合并后的管道中移除去重管道，得到框架默认管道
+    if dedup_pipeline:
+        default_pipelines_without_dedup = [p for p in default_pipelines if p != dedup_pipeline]
+        for i, pipeline in enumerate(default_pipelines_without_dedup):
+            print(f"  {i}: {pipeline}")
+    else:
+        for i, pipeline in enumerate(default_pipelines):
+            print(f"  {i}: {pipeline}")
+    
+    print()
+    print("自定义管道:")
+    custom_pipelines = settings.get('PIPELINES')
+    # 从合并后的管道中移除默认管道，得到自定义管道
+    default_pipelines_list = [
+        'crawlo.pipelines.console_pipeline.ConsolePipeline',
+        'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline'
+    ]
+    custom_pipelines_list = [p for p in custom_pipelines if p not in default_pipelines_list]
+    for i, pipeline in enumerate(custom_pipelines_list):
+        print(f"  {i}: {pipeline}")
+
+if __name__ == "__main__":
     debug_pipelines()