crawlo 1.4.2__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/logging/config.py

@@ -1,97 +1,197 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志配置管理
-"""
-
-import os
-from dataclasses import dataclass, field
-from typing import Optional, Dict, Any
-
-
-@dataclass
-class LogConfig:
-    """日志配置数据类 - 简单明确的配置结构"""
-    
-    # 基本配置
-    level: str = "INFO"
-    format: str = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
-    encoding: str = "utf-8"
-    
-    # 文件配置
-    file_path: Optional[str] = None
-    max_bytes: int = 10 * 1024 * 1024  # 10MB
-    backup_count: int = 5
-    
-    # 控制台配置
-    console_enabled: bool = True
-    file_enabled: bool = True
-    
-    # 模块级别配置
-    module_levels: Dict[str, str] = field(default_factory=dict)
-    
-    @classmethod
-    def from_settings(cls, settings) -> 'LogConfig':
-        """从settings对象创建配置"""
-        if not settings:
-            return cls()
-            
-        # 使用settings的get方法而不是getattr
-        if hasattr(settings, 'get'):
-            get_val = settings.get
-        else:
-            get_val = lambda k, d=None: getattr(settings, k, d)
-        
-        # 获取默认值
-        format_default_value = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
-        
-        return cls(
-            level=get_val('LOG_LEVEL', 'INFO'),
-            format=get_val('LOG_FORMAT', format_default_value),
-            encoding=get_val('LOG_ENCODING', 'utf-8'),
-            file_path=get_val('LOG_FILE'),
-            max_bytes=get_val('LOG_MAX_BYTES', 10 * 1024 * 1024),
-            backup_count=get_val('LOG_BACKUP_COUNT', 5),
-            console_enabled=get_val('LOG_CONSOLE_ENABLED', True),
-            file_enabled=get_val('LOG_FILE_ENABLED', True),
-            module_levels=get_val('LOG_LEVELS', {})
-        )
-    
-    @classmethod
-    def from_dict(cls, config_dict: Dict[str, Any]) -> 'LogConfig':
-        """从字典创建配置"""
-        return cls(**{k: v for k, v in config_dict.items() if k in cls.__annotations__})
-    
-    def get_module_level(self, module_name: str) -> str:
-        """获取模块的日志级别"""
-        # 先查找精确匹配
-        if module_name in self.module_levels:
-            return self.module_levels[module_name]
-        
-        # 查找父模块匹配
-        parts = module_name.split('.')
-        for i in range(len(parts) - 1, 0, -1):
-            parent_module = '.'.join(parts[:i])
-            if parent_module in self.module_levels:
-                return self.module_levels[parent_module]
-        
-        # 返回默认级别
-        return self.level
-    
-    def validate(self) -> bool:
-        """验证配置有效性"""
-        valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}
-        
-        if self.level.upper() not in valid_levels:
-            return False
-        
-        # 确保日志目录存在
-        if self.file_path and self.file_enabled:
-            try:
-                log_dir = os.path.dirname(self.file_path)
-                if log_dir and not os.path.exists(log_dir):
-                    os.makedirs(log_dir, exist_ok=True)
-            except (OSError, PermissionError):
-                return False
-        
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志配置管理
+"""
+
+import os
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+
+
+@dataclass
+class LogConfig:
+    """日志配置数据类 - 简单明确的配置结构"""
+    
+    # 基本配置
+    level: str = "INFO"
+    format: str = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+    encoding: str = "utf-8"
+    
+    # 文件配置
+    file_path: Optional[str] = None
+    max_bytes: int = 10 * 1024 * 1024  # 10MB
+    backup_count: int = 5
+    
+    # 控制台配置
+    console_enabled: bool = True
+    file_enabled: bool = True
+    
+    # 分别控制台和文件的日志级别
+    console_level: Optional[str] = None
+    file_level: Optional[str] = None
+    
+    # 上下文信息配置
+    include_thread_id: bool = False
+    include_process_id: bool = False
+    include_module_path: bool = False
+    
+    # 模块级别配置
+    module_levels: Dict[str, str] = field(default_factory=dict)
+    
+    @classmethod
+    def from_settings(cls, settings) -> 'LogConfig':
+        """从settings对象创建配置"""
+        if not settings:
+            return cls()
+            
+        # 使用settings的get方法而不是getattr
+        if hasattr(settings, 'get'):
+            get_val = settings.get
+        else:
+            get_val = lambda k, d=None: getattr(settings, k, d)
+        
+        # 获取默认值
+        format_default_value = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+        
+        return cls(
+            level=get_val('LOG_LEVEL', 'INFO'),
+            format=get_val('LOG_FORMAT', format_default_value),
+            encoding=get_val('LOG_ENCODING', 'utf-8'),
+            file_path=get_val('LOG_FILE'),
+            max_bytes=get_val('LOG_MAX_BYTES', 10 * 1024 * 1024),
+            backup_count=get_val('LOG_BACKUP_COUNT', 5),
+            console_enabled=get_val('LOG_CONSOLE_ENABLED', True),
+            file_enabled=get_val('LOG_FILE_ENABLED', True),
+            console_level=get_val('LOG_CONSOLE_LEVEL'),  # 允许单独设置控制台级别
+            file_level=get_val('LOG_FILE_LEVEL'),  # 允许单独设置文件级别
+            include_thread_id=get_val('LOG_INCLUDE_THREAD_ID', False),
+            include_process_id=get_val('LOG_INCLUDE_PROCESS_ID', False),
+            include_module_path=get_val('LOG_INCLUDE_MODULE_PATH', False),
+            module_levels=get_val('LOG_LEVELS', {})
+        )
+    
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any]) -> 'LogConfig':
+        """从字典创建配置"""
+        # 映射字典键到类属性名
+        key_mapping = {
+            'LOG_LEVEL': 'level',
+            'LOG_FORMAT': 'format',
+            'LOG_ENCODING': 'encoding',
+            'LOG_FILE': 'file_path',
+            'LOG_MAX_BYTES': 'max_bytes',
+            'LOG_BACKUP_COUNT': 'backup_count',
+            'LOG_CONSOLE_ENABLED': 'console_enabled',
+            'LOG_FILE_ENABLED': 'file_enabled',
+            'LOG_CONSOLE_LEVEL': 'console_level',
+            'LOG_FILE_LEVEL': 'file_level',
+            'LOG_INCLUDE_THREAD_ID': 'include_thread_id',
+            'LOG_INCLUDE_PROCESS_ID': 'include_process_id',
+            'LOG_INCLUDE_MODULE_PATH': 'include_module_path',
+            'LOG_LEVELS': 'module_levels'
+        }
+        
+        # 应用键映射
+        mapped_dict = {}
+        for k, v in config_dict.items():
+            mapped_key = key_mapping.get(k, k)
+            if mapped_key in cls.__annotations__:
+                mapped_dict[mapped_key] = v
+                
+        return cls(**mapped_dict)
+    
+    def get_module_level(self, module_name: str) -> str:
+        """获取模块的日志级别"""
+        # 先查找精确匹配
+        if module_name in self.module_levels:
+            return self.module_levels[module_name]
+        
+        # 查找父模块匹配
+        parts = module_name.split('.')
+        for i in range(len(parts) - 1, 0, -1):
+            parent_module = '.'.join(parts[:i])
+            if parent_module in self.module_levels:
+                return self.module_levels[parent_module]
+        
+        # 返回默认级别
+        return self.level
+    
+    def get_console_level(self) -> str:
+        """获取控制台日志级别"""
+        return self.console_level or self.level
+    
+    def get_file_level(self) -> str:
+        """获取文件日志级别"""
+        return self.file_level or self.level
+    
+    def get_format(self) -> str:
+        """
+        获取日志格式，包含上下文信息
+        
+        Returns:
+            日志格式字符串
+        """
+        base_format = self.format
+        
+        # 添加线程ID
+        if self.include_thread_id:
+            if '[%(thread)d]' not in base_format:
+                # 在时间戳后添加线程ID
+                base_format = base_format.replace(
+                    '%(asctime)s', 
+                    '%(asctime)s [%(thread)d]'
+                )
+                
+        # 添加进程ID
+        if self.include_process_id:
+            if '[%(process)d]' not in base_format:
+                # 在时间戳后添加进程ID（如果已经有线程ID，则在线程ID后添加）
+                if '[%(thread)d]' in base_format:
+                    base_format = base_format.replace(
+                        '%(asctime)s [%(thread)d]', 
+                        '%(asctime)s [%(thread)d] [%(process)d]'
+                    )
+                else:
+                    base_format = base_format.replace(
+                        '%(asctime)s', 
+                        '%(asctime)s [%(process)d]'
+                    )
+                
+        # 添加模块路径
+        if self.include_module_path:
+            if '%(pathname)s:%(lineno)d' not in base_format:
+                # 在消息前添加文件路径和行号
+                base_format = base_format.replace(
+                    '%(message)s', 
+                    '%(pathname)s:%(lineno)d - %(message)s'
+                )
+                
+        return base_format
+    
+    def validate(self) -> bool:
+        """验证配置有效性"""
+        valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}
+        
+        # 验证主级别
+        if self.level.upper() not in valid_levels:
+            return False
+        
+        # 验证控制台级别
+        if self.console_level and self.console_level.upper() not in valid_levels:
+            return False
+        
+        # 验证文件级别
+        if self.file_level and self.file_level.upper() not in valid_levels:
+            return False
+        
+        # 确保日志目录存在
+        if self.file_path and self.file_enabled:
+            try:
+                log_dir = os.path.dirname(self.file_path)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+            except (OSError, PermissionError):
+                return False
+        
         return True

crawlo/logging/factory.py

@@ -1,129 +1,172 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-日志器工厂 - 创建和缓存Logger实例
-"""
-
-import logging
-import os
-import threading
-from logging.handlers import RotatingFileHandler
-from typing import Dict, Optional
-from weakref import WeakValueDictionary
-
-from .manager import get_config, is_configured, configure
-from .config import LogConfig
-
-
-class LoggerFactory:
-    """
-    Logger工厂类 - 负责创建和缓存Logger实例
-    
-    特点：
-    1. 使用WeakValueDictionary避免内存泄漏
-    2. 线程安全的Logger创建
-    3. 自动配置管理
-    4. 简单的缓存策略
-    """
-    
-    # Logger缓存 - 使用弱引用避免内存泄漏
-    _logger_cache: WeakValueDictionary = WeakValueDictionary()
-    _cache_lock = threading.RLock()
-    
-    @classmethod
-    def get_logger(cls, name: str = 'crawlo') -> logging.Logger:
-        """
-        获取Logger实例
-        
-        Args:
-            name: Logger名称
-            
-        Returns:
-            logging.Logger: 配置好的Logger实例
-        """
-        # 确保日志系统已配置
-        if not is_configured():
-            configure()  # 使用默认配置
-        
-        # 检查缓存
-        with cls._cache_lock:
-            if name in cls._logger_cache:
-                return cls._logger_cache[name]
-            
-            # 创建新的Logger
-            logger = cls._create_logger(name)
-            cls._logger_cache[name] = logger
-            return logger
-    
-    @classmethod
-    def _create_logger(cls, name: str) -> logging.Logger:
-        """创建新的Logger实例"""
-        config = get_config()
-        if not config:
-            raise RuntimeError("Log system not configured")
-        
-        # 创建Logger
-        logger = logging.getLogger(name)
-        logger.setLevel(logging.DEBUG)  # Logger本身设为最低级别
-        
-        # 清除现有handlers（避免重复添加）
-        logger.handlers.clear()
-        
-        # 获取模块级别
-        module_level = config.get_module_level(name)
-        level = getattr(logging, module_level.upper(), logging.INFO)
-        
-        # 创建formatter
-        formatter = logging.Formatter(config.format)
-        
-        # 添加控制台Handler
-        if config.console_enabled:
-            console_handler = logging.StreamHandler()
-            console_handler.setFormatter(formatter)
-            console_handler.setLevel(level)
-            logger.addHandler(console_handler)
-        
-        # 添加文件Handler
-        if config.file_enabled and config.file_path:
-            try:
-                # 确保日志目录存在
-                log_dir = os.path.dirname(config.file_path)
-                if log_dir and not os.path.exists(log_dir):
-                    os.makedirs(log_dir, exist_ok=True)
-                
-                file_handler = RotatingFileHandler(
-                    filename=config.file_path,
-                    maxBytes=config.max_bytes,
-                    backupCount=config.backup_count,
-                    encoding=config.encoding
-                )
-                file_handler.setFormatter(formatter)
-                file_handler.setLevel(level)
-                logger.addHandler(file_handler)
-            except Exception as e:
-                # 文件Handler创建失败时，至少保证控制台输出
-                pass
-        
-        # 防止向上传播（避免重复输出）
-        logger.propagate = False
-        
-        return logger
-    
-    @classmethod
-    def clear_cache(cls):
-        """清空Logger缓存"""
-        with cls._cache_lock:
-            cls._logger_cache.clear()
-    
-    @classmethod
-    def refresh_loggers(cls, new_config: LogConfig):
-        """刷新所有缓存的Logger（配置更新时使用）"""
-        with cls._cache_lock:
-            # 清空缓存，强制重新创建
-            cls._logger_cache.clear()
-
-
-# 便捷函数
-def get_logger(name: str = 'crawlo') -> logging.Logger:
-    """获取Logger实例的便捷函数"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志器工厂 - 创建和缓存Logger实例
+"""
+
+import logging
+import os
+import sys
+import threading
+from typing import Dict, Optional
+from weakref import WeakValueDictionary
+
+# 尝试导入concurrent-log-handler，如果不可用则回退到标准库
+try:
+    from concurrent_log_handler import ConcurrentRotatingFileHandler
+    USE_CONCURRENT_HANDLER = True
+except ImportError:
+    from logging.handlers import RotatingFileHandler
+    USE_CONCURRENT_HANDLER = False
+
+from .manager import get_config, is_configured, configure
+from .config import LogConfig
+
+
+class LoggerFactory:
+    """
+    Logger工厂类 - 负责创建和缓存Logger实例
+    
+    特点：
+    1. 使用WeakValueDictionary避免内存泄漏
+    2. 线程安全的Logger创建
+    3. 自动配置管理
+    4. 简单的缓存策略
+    5. Windows兼容的日志轮转处理
+    """
+    
+    # Logger缓存 - 使用弱引用避免内存泄漏
+    _logger_cache: WeakValueDictionary = WeakValueDictionary()
+    _cache_lock = threading.RLock()
+    
+    @classmethod
+    def get_logger(cls, name: str = 'crawlo') -> logging.Logger:
+        """
+        获取Logger实例
+        
+        Args:
+            name: Logger名称
+            
+        Returns:
+            logging.Logger: 配置好的Logger实例
+        """
+        # 确保日志系统已配置
+        if not is_configured():
+            configure()  # 使用默认配置
+        
+        # 检查缓存
+        with cls._cache_lock:
+            if name in cls._logger_cache:
+                return cls._logger_cache[name]
+            
+            # 创建新的Logger
+            logger = cls._create_logger(name)
+            cls._logger_cache[name] = logger
+            return logger
+    
+    @classmethod
+    def _create_logger(cls, name: str) -> logging.Logger:
+        """创建新的Logger实例"""
+        config = get_config()
+        if not config:
+            raise RuntimeError("Log system not configured")
+        
+        # 创建Logger
+        logger = logging.getLogger(name)
+        logger.setLevel(logging.DEBUG)  # Logger本身设为最低级别
+        
+        # 清除现有handlers（避免重复添加）
+        logger.handlers.clear()
+        
+        # 获取模块级别
+        module_level = config.get_module_level(name)
+        
+        # 创建formatter
+        formatter = logging.Formatter(config.get_format())
+        
+        # 添加控制台Handler
+        if config.console_enabled:
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            # 使用专门的控制台级别或模块级别
+            console_level = config.get_console_level()
+            level = getattr(logging, console_level.upper(), logging.INFO)
+            console_handler.setLevel(level)
+            logger.addHandler(console_handler)
+        
+        # 添加文件Handler
+        if config.file_enabled and config.file_path:
+            try:
+                # 确保日志目录存在
+                log_dir = os.path.dirname(config.file_path)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+                
+                # 根据平台选择合适的Handler
+                if USE_CONCURRENT_HANDLER:
+                    file_handler = ConcurrentRotatingFileHandler(
+                        filename=config.file_path,
+                        maxBytes=config.max_bytes,
+                        backupCount=config.backup_count,
+                        encoding=config.encoding
+                    )
+                else:
+                    # 在Windows上给出警告信息
+                    if sys.platform.startswith('win'):
+                        # 检查是否已经有同名的日志文件被其他进程使用
+                        try:
+                            # 尝试以独占模式打开文件来检查是否被占用
+                            with open(config.file_path, 'a'):
+                                pass
+                        except (PermissionError, OSError):
+                            # 如果文件被占用，记录警告信息
+                            console_handler = logging.StreamHandler()
+                            console_handler.setFormatter(formatter)
+                            console_handler.setLevel(logging.WARNING)
+                            logger.addHandler(console_handler)
+                            logger.warning(f"日志文件 {config.file_path} 可能正在被其他进程使用，这可能导致日志轮转失败。建议安装 concurrent-log-handler 库以获得更好的Windows兼容性。")
+                    
+                    file_handler = RotatingFileHandler(
+                        filename=config.file_path,
+                        maxBytes=config.max_bytes,
+                        backupCount=config.backup_count,
+                        encoding=config.encoding
+                    )
+                
+                file_handler.setFormatter(formatter)
+                # 使用专门的文件级别或模块级别
+                file_level = config.get_file_level()
+                level = getattr(logging, file_level.upper(), logging.INFO)
+                file_handler.setLevel(level)
+                logger.addHandler(file_handler)
+            except Exception as e:
+                # 文件Handler创建失败时，至少保证控制台输出
+                console_handler = logging.StreamHandler()
+                console_handler.setFormatter(formatter)
+                console_handler.setLevel(logging.WARNING)
+                logger.addHandler(console_handler)
+                logger.warning(f"无法创建文件日志处理器: {e}，仅使用控制台输出。")
+        
+        # 防止向上传播（避免重复输出）
+        logger.propagate = False
+        
+        return logger
+    
+    @classmethod
+    def clear_cache(cls):
+        """清空Logger缓存"""
+        with cls._cache_lock:
+            cls._logger_cache.clear()
+    
+    @classmethod
+    def refresh_loggers(cls, new_config: LogConfig):
+        """刷新所有缓存的Logger（配置更新时使用）"""
+        with cls._cache_lock:
+            # 清空缓存，强制重新创建
+            cls._logger_cache.clear()
+
+
+# 便捷函数
+def get_logger(name: str = 'crawlo') -> logging.Logger:
+    """获取Logger实例的便捷函数"""
     return LoggerFactory.get_logger(name)