crawlo 1.3.4__tar.gz → 1.3.5__tar.gz

{crawlo-1.3.4/crawlo.egg-info → crawlo-1.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.4
+Version: 1.3.5
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

crawlo-1.3.5/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.3.5'

{crawlo-1.3.4 → crawlo-1.3.5}/crawlo/pipelines/mysql_pipeline.py

@@ -20,7 +20,14 @@ class AsyncmyMySQLPipeline:
         self._pool_lock = asyncio.Lock()
         self._pool_initialized = False
         self.pool = None
+        
+        # 优先从爬虫的custom_settings中获取表名，如果没有则使用默认值
+        spider_table_name = None
+        if hasattr(crawler, 'spider') and crawler.spider and hasattr(crawler.spider, 'custom_settings'):
+            spider_table_name = crawler.spider.custom_settings.get('MYSQL_TABLE')
+            
         self.table_name = (
+                spider_table_name or
                 self.settings.get('MYSQL_TABLE') or
                 getattr(crawler.spider, 'mysql_table', None) or
                 f"{crawler.spider.name}_items"

{crawlo-1.3.4 → crawlo-1.3.5}/crawlo/templates/run.py.tmpl

@@ -21,8 +21,8 @@ def main():
         spider_modules = ['{{project_name}}.spiders']
         process = CrawlerProcess(spider_modules=spider_modules)
 
-        # 运行指定的爬虫
-        asyncio.run(process.crawl('{{spider_name}}'))
+        # TODO 运行指定的爬虫
+        asyncio.run(process.crawl('spider_name'))
 
     except Exception as e:
         print(f"❌ 运行失败: {e}")

crawlo-1.3.5/crawlo/utils/log.py

@@ -0,0 +1,80 @@
+# ==================== 向后兼容的日志接口 ====================
+# 主要功能已迁移到 crawlo.logging 模块
+# 本文件提供向后兼容接口，同时支持新的日志系统功能
+
+import logging
+from typing import Optional, Any
+
+# 向后兼容：导入新的日志系统
+try:
+    from crawlo.logging import get_logger as new_get_logger, configure_logging
+
+    _NEW_LOGGING_AVAILABLE = True
+except ImportError:
+    _NEW_LOGGING_AVAILABLE = False
+    new_get_logger = None
+    configure_logging = None
+
+LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
+
+
+# 向后兼容的日志函数
+def get_logger(name: str = 'default', level: Optional[int] = None):
+    """获取Logger实例 - 向后兼容函数"""
+    if _NEW_LOGGING_AVAILABLE and new_get_logger:
+        # 使用新的日志系统
+        return new_get_logger(name)
+    else:
+        # 降级到基本的Python logging
+        logger = logging.getLogger(name)
+        if not logger.handlers:
+            handler = logging.StreamHandler()
+            formatter = logging.Formatter(LOG_FORMAT)
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+            logger.setLevel(level or logging.INFO)
+        return logger
+
+
+def get_component_logger(component_class: Any, settings: Optional[Any] = None, level: Optional[str] = None):
+    """
+    获取组件Logger - 推荐的组件日志创建方式
+    
+    Args:
+        component_class: 组件类
+        settings: 配置对象，用于读取日志级别配置
+        level: 日志级别（优先级低于settings中的配置）
+        
+    Returns:
+        logging.Logger: 配置好的Logger实例
+    """
+    # 获取组件名称
+    if hasattr(component_class, '__name__'):
+        component_name = component_class.__name__
+    else:
+        component_name = str(component_class)
+    
+    # 如果新日志系统可用，使用新系统
+    if _NEW_LOGGING_AVAILABLE and new_get_logger:
+        return new_get_logger(component_name)
+    
+    # 否则使用向后兼容方式
+    # 从settings中获取日志级别（如果提供）
+    if settings is not None:
+        # 尝试从settings获取组件特定的日志级别
+        if hasattr(settings, 'get'):
+            # 检查是否有组件特定的日志级别配置
+            component_level = settings.get(f'LOG_LEVEL_{component_name}')
+            if component_level is not None:
+                level = component_level
+            else:
+                # 检查通用日志级别
+                general_level = settings.get('LOG_LEVEL')
+                if general_level is not None:
+                    level = general_level
+    
+    # 转换日志级别
+    if isinstance(level, str):
+        level = getattr(logging, level.upper(), logging.INFO)
+    
+    return get_logger(component_name, level)

{crawlo-1.3.4 → crawlo-1.3.5/crawlo.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.4
+Version: 1.3.5
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.3.4 → crawlo-1.3.5}/crawlo.egg-info/SOURCES.txt

@@ -173,8 +173,10 @@ tests/controlled_spider_example.py
 tests/date_tools_example.py
 tests/debug_configure.py
 tests/debug_framework_logger.py
+tests/debug_log_config.py
 tests/debug_log_levels.py
 tests/debug_pipelines.py
+tests/detailed_log_test.py
 tests/distributed_test.py
 tests/distributed_test_debug.py
 tests/dynamic_loading_example.py
@@ -183,8 +185,12 @@ tests/env_config_example.py
 tests/error_handling_example.py
 tests/final_command_test_report.md
 tests/final_comprehensive_test.py
+tests/final_log_test.py
 tests/final_validation_test.py
+tests/fix_log_test.py
 tests/framework_performance_test.py
+tests/log_buffering_test.py
+tests/log_generation_timing_test.py
 tests/optimized_performance_test.py
 tests/performance_comparison.py
 tests/queue_blocking_test.py
@@ -195,9 +201,11 @@ tests/response_improvements_example.py
 tests/simple_command_test.py
 tests/simple_crawlo_test.py
 tests/simple_log_test.py
+tests/simple_log_test2.py
 tests/simple_optimization_test.py
 tests/simple_spider_test.py
 tests/simple_test.py
+tests/spider_log_timing_test.py
 tests/test_advanced_tools.py
 tests/test_all_commands.py
 tests/test_all_redis_key_configs.py
@@ -232,10 +240,12 @@ tests/test_final_validation.py
 tests/test_framework_env_usage.py
 tests/test_framework_logger.py
 tests/test_framework_startup.py
+tests/test_get_component_logger.py
 tests/test_integration.py
 tests/test_item_dedup_redis_key.py
 tests/test_large_scale_config.py
 tests/test_large_scale_helper.py
+tests/test_logging_system.py
 tests/test_mode_change.py
 tests/test_mode_consistency.py
 tests/test_offsite_middleware.py

crawlo-1.3.5/tests/debug_log_config.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+调试日志配置问题
+"""
+
+import os
+import sys
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger, LogManager
+from crawlo.logging.config import LogConfig
+
+
+def debug_log_configuration():
+    """调试日志配置"""
+    print("=== 调试日志配置 ===")
+    
+    # 重置配置
+    LogManager().reset()
+    
+    # 1. 检查初始状态
+    print("1. 检查初始状态...")
+    print(f"   初始配置状态: {LogManager().is_configured}")
+    if LogManager().config:
+        print(f"   初始配置: {LogManager().config}")
+    
+    # 2. 配置日志系统
+    print("2. 配置日志系统...")
+    config = configure(
+        LOG_LEVEL='DEBUG',
+        LOG_FILE='debug_test.log',
+        LOG_MAX_BYTES=1024,
+        LOG_BACKUP_COUNT=2,
+        LOG_CONSOLE_ENABLED=True,
+        LOG_FILE_ENABLED=True
+    )
+    
+    print(f"   配置返回值: {config}")
+    print(f"   配置类型: {type(config)}")
+    print(f"   配置级别: {config.level}")
+    print(f"   配置文件路径: {config.file_path}")
+    print(f"   轮转大小: {config.max_bytes}")
+    print(f"   备份数量: {config.backup_count}")
+    print(f"   控制台启用: {config.console_enabled}")
+    print(f"   文件启用: {config.file_enabled}")
+    
+    # 3. 检查管理器状态
+    print("3. 检查管理器状态...")
+    manager = LogManager()
+    print(f"   管理器配置状态: {manager.is_configured}")
+    if manager.config:
+        print(f"   管理器配置: {manager.config}")
+        print(f"   管理器配置文件路径: {manager.config.file_path}")
+    
+    # 4. 测试Logger创建
+    print("4. 测试Logger创建...")
+    logger = get_logger('test.debug')
+    print(f"   Logger handlers数量: {len(logger.handlers)}")
+    
+    for i, handler in enumerate(logger.handlers):
+        handler_type = type(handler).__name__
+        print(f"     Handler {i}: {handler_type}")
+        if hasattr(handler, 'baseFilename'):
+            print(f"       文件名: {handler.baseFilename}")
+    
+    # 5. 测试日志输出
+    print("5. 测试日志输出...")
+    logger.info("调试测试消息")
+
+
+def test_config_from_dict():
+    """测试从字典创建配置"""
+    print("\n=== 测试从字典创建配置 ===")
+    
+    LogManager().reset()
+    
+    # 使用字典配置
+    config_dict = {
+        'level': 'DEBUG',
+        'file_path': 'dict_test.log',
+        'max_bytes': 1024,
+        'backup_count': 2,
+        'console_enabled': True,
+        'file_enabled': True
+    }
+    
+    config = LogConfig.from_dict(config_dict)
+    print(f"   字典配置: {config}")
+    print(f"   验证结果: {config.validate()}")
+    
+    # 应用配置
+    LogManager().configure(config)
+    
+    logger = get_logger('test.dict')
+    print(f"   Logger handlers数量: {len(logger.handlers)}")
+    
+    for i, handler in enumerate(logger.handlers):
+        handler_type = type(handler).__name__
+        print(f"     Handler {i}: {handler_type}")
+
+
+def main():
+    """主函数"""
+    print("开始调试日志配置问题...")
+    
+    try:
+        debug_log_configuration()
+        test_config_from_dict()
+        
+        print("\n=== 调试完成 ===")
+        
+    except Exception as e:
+        print(f"\n调试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

crawlo-1.3.5/tests/detailed_log_test.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python
+# -*- coding: UTF-8 -*-
+"""
+详细的日志系统功能测试
+"""
+
+import os
+import sys
+import tempfile
+import shutil
+from pathlib import Path
+
+# 添加项目根目录到Python路径
+project_root = Path(__file__).parent.parent
+sys.path.insert(0, str(project_root))
+
+from crawlo.logging import configure_logging as configure, get_logger, LogManager
+from crawlo.logging.config import LogConfig
+
+
+def test_log_config_creation():
+    """测试日志配置创建"""
+    print("=== 测试日志配置创建 ===")
+    
+    # 重置日志管理器
+    LogManager().reset()
+    
+    # 1. 测试通过关键字参数创建配置
+    print("1. 测试通过关键字参数创建配置...")
+    config = configure(
+        LOG_LEVEL='DEBUG',
+        LOG_FILE='test.log',
+        LOG_MAX_BYTES=1024,
+        LOG_BACKUP_COUNT=3
+    )
+    
+    print(f"   配置级别: {config.level}")
+    print(f"   配置文件路径: {config.file_path}")
+    print(f"   轮转大小: {config.max_bytes}")
+    print(f"   备份数量: {config.backup_count}")
+    
+    # 2. 测试通过字典创建配置
+    print("2. 测试通过字典创建配置...")
+    LogManager().reset()
+    config_dict = {
+        'LOG_LEVEL': 'WARNING',
+        'LOG_FILE': 'dict_test.log',
+        'LOG_CONSOLE_ENABLED': False
+    }
+    config = configure(**config_dict)
+    
+    print(f"   配置级别: {config.level}")
+    print(f"   配置文件路径: {config.file_path}")
+    print(f"   控制台启用: {config.console_enabled}")
+
+
+def test_logger_factory():
+    """测试Logger工厂"""
+    print("\n=== 测试Logger工厂 ===")
+    
+    # 重置并配置
+    LogManager().reset()
+    configure(LOG_LEVEL='INFO')
+    
+    # 1. 测试获取Logger
+    print("1. 测试获取Logger...")
+    logger1 = get_logger('test.factory1')
+    logger2 = get_logger('test.factory2')
+    logger3 = get_logger('test.factory1')  # 应该是同一个实例
+    
+    print(f"   Logger1 ID: {id(logger1)}")
+    print(f"   Logger2 ID: {id(logger2)}")
+    print(f"   Logger3 ID: {id(logger3)}")
+    print(f"   Logger1和Logger3是同一对象: {logger1 is logger3}")
+    
+    # 2. 测试Logger配置
+    print("2. 测试Logger配置...")
+    print(f"   Logger1名称: {logger1.name}")
+    print(f"   Logger1 handlers数量: {len(logger1.handlers)}")
+    
+    for i, handler in enumerate(logger1.handlers):
+        print(f"     Handler {i}: {type(handler).__name__}")
+
+
+def test_file_and_console_handlers():
+    """测试文件和控制台处理器"""
+    print("\n=== 测试文件和控制台处理器 ===")
+    
+    # 创建临时目录
+    temp_dir = tempfile.mkdtemp()
+    log_file = os.path.join(temp_dir, 'handler_test.log')
+    
+    try:
+        # 1. 测试文件和控制台都启用
+        print("1. 测试文件和控制台都启用...")
+        LogManager().reset()
+        configure(
+            LOG_LEVEL='INFO',
+            LOG_FILE=log_file,
+            LOG_CONSOLE_ENABLED=True,
+            LOG_FILE_ENABLED=True
+        )
+        
+        logger = get_logger('test.handlers')
+        print(f"   Handlers数量: {len(logger.handlers)}")
+        
+        has_file_handler = False
+        has_console_handler = False
+        
+        for handler in logger.handlers:
+            handler_type = type(handler).__name__
+            print(f"     Handler类型: {handler_type}")
+            if 'FileHandler' in handler_type:
+                has_file_handler = True
+                print(f"     文件路径: {getattr(handler, 'baseFilename', 'N/A')}")
+            elif 'StreamHandler' in handler_type:
+                has_console_handler = True
+        
+        print(f"   有文件处理器: {has_file_handler}")
+        print(f"   有控制台处理器: {has_console_handler}")
+        
+        # 输出日志
+        logger.info("测试文件和控制台处理器")
+        
+        # 检查文件是否存在
+        if os.path.exists(log_file):
+            with open(log_file, 'r', encoding='utf-8') as f:
+                content = f.read()
+                print(f"   文件内容行数: {len(content.splitlines())}")
+        else:
+            print("   文件不存在!")
+            
+    finally:
+        # 清理
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+def test_log_levels():
+    """测试日志级别"""
+    print("\n=== 测试日志级别 ===")
+    
+    # 1. 测试默认级别
+    print("1. 测试默认级别...")
+    LogManager().reset()
+    configure(LOG_LEVEL='WARNING')
+    
+    logger = get_logger('test.levels')
+    logger.debug("DEBUG消息 - 不应该显示")
+    logger.info("INFO消息 - 不应该显示")
+    logger.warning("WARNING消息 - 应该显示")
+    logger.error("ERROR消息 - 应该显示")
+    
+    # 2. 测试模块特定级别
+    print("2. 测试模块特定级别...")
+    LogManager().reset()
+    configure(
+        LOG_LEVEL='ERROR',
+        LOG_LEVELS={
+            'test.debug_module': 'DEBUG',
+            'test.info_module': 'INFO'
+        }
+    )
+    
+    # 默认模块（ERROR级别）
+    default_logger = get_logger('test.default')
+    default_logger.info("默认模块INFO消息 - 不应该显示")
+    default_logger.error("默认模块ERROR消息 - 应该显示")
+    
+    # DEBUG模块（DEBUG级别）
+    debug_logger = get_logger('test.debug_module')
+    debug_logger.debug("DEBUG模块DEBUG消息 - 应该显示")
+    debug_logger.info("DEBUG模块INFO消息 - 应该显示")
+    
+    # INFO模块（INFO级别）
+    info_logger = get_logger('test.info_module')
+    info_logger.debug("INFO模块DEBUG消息 - 不应该显示")
+    info_logger.info("INFO模块INFO消息 - 应该显示")
+
+
+def test_log_config_validation():
+    """测试日志配置验证"""
+    print("\n=== 测试日志配置验证 ===")
+    
+    # 1. 测试有效配置
+    print("1. 测试有效配置...")
+    valid_config = LogConfig(level='INFO', file_path='valid.log')
+    is_valid = valid_config.validate()
+    print(f"   有效配置验证: {is_valid}")
+    
+    # 2. 测试无效级别
+    print("2. 测试无效级别...")
+    invalid_config = LogConfig(level='INVALID', file_path='invalid.log')
+    is_valid = invalid_config.validate()
+    print(f"   无效级别验证: {is_valid}")
+    
+    # 3. 测试目录创建
+    print("3. 测试目录创建...")
+    temp_dir = tempfile.mkdtemp()
+    nested_path = os.path.join(temp_dir, 'subdir', 'nested.log')
+    
+    nested_config = LogConfig(level='INFO', file_path=nested_path)
+    is_valid = nested_config.validate()
+    print(f"   嵌套目录验证: {is_valid}")
+    print(f"   目录存在: {os.path.exists(os.path.dirname(nested_path))}")
+    
+    # 清理
+    shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+def main():
+    """主测试函数"""
+    print("开始详细测试Crawlo框架日志系统...")
+    
+    try:
+        # 运行所有测试
+        test_log_config_creation()
+        test_logger_factory()
+        test_file_and_console_handlers()
+        test_log_levels()
+        test_log_config_validation()
+        
+        print("\n=== 详细测试完成 ===")
+        
+    except Exception as e:
+        print(f"\n测试过程中出现错误: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+    
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())