crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/initialization/core.py

@@ -6,14 +6,18 @@
 
 import threading
 import time
+import signal
 from typing import Optional, Any
 
 from .built_in import register_built_in_initializers
 from .context import InitializationContext
-from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
+from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition, validate_phase_dependencies
 from .registry import get_global_registry
 
 
+from crawlo.utils.singleton import singleton
+
+@singleton
 class CoreInitializer:
     """
     核心初始化器 - 协调整个框架的初始化过程
@@ -25,29 +29,18 @@ class CoreInitializer:
     4. 错误处理和降级策略
     """
     
-    _instance: Optional['CoreInitializer'] = None
-    _lock = threading.Lock()
-    
-    def __new__(cls) -> 'CoreInitializer':
-        if cls._instance is None:
-            with cls._lock:
-                if cls._instance is None:
-                    cls._instance = super(CoreInitializer, cls).__new__(cls)
-                    cls._instance._initialized = False
-        return cls._instance
-    
     def __init__(self):
-        if hasattr(self, '_initialized') and self._initialized:
-            return
-            
         self._context: Optional[InitializationContext] = None
         self._is_ready = False
         self._init_lock = threading.RLock()
         
+        # 在注册内置初始化器之前，先验证阶段依赖关系
+        is_valid, error_msg = validate_phase_dependencies()
+        if not is_valid:
+            raise RuntimeError(f"初始化阶段配置错误: {error_msg}")
+        
         # 注册内置初始化器
         register_built_in_initializers()
-        
-        self._initialized = True
     
     @property
     def context(self) -> Optional[InitializationContext]:
@@ -128,10 +121,10 @@ class CoreInitializer:
                     # 可选阶段，跳过
                     continue
             
-            # 执行阶段
+            # 执行阶段（带超时控制）
             start_time = time.time()
             try:
-                result = registry.execute_phase(phase, context)
+                result = self._execute_phase_with_timeout(phase, context, registry)
                 result.duration = time.time() - start_time
                 
                 context.mark_phase_completed(phase, result)
@@ -152,6 +145,59 @@ class CoreInitializer:
                 if not self._is_phase_optional(phase):
                     raise
     
+    def _execute_phase_with_timeout(self, phase: InitializationPhase, 
+                                    context: InitializationContext,
+                                    registry) -> PhaseResult:
+        """
+        执行阶段并支持超时控制
+        
+        Args:
+            phase: 初始化阶段
+            context: 初始化上下文
+            registry: 初始化器注册表
+            
+        Returns:
+            PhaseResult: 阶段执行结果
+            
+        Raises:
+            TimeoutError: 阶段执行超时
+        """
+        phase_def = get_phase_definition(phase)
+        timeout = phase_def.timeout if phase_def else 30.0
+        
+        # 使用线程执行，支持超时
+        result_container: list[Optional[PhaseResult]] = [None]
+        exception_container: list[Optional[Exception]] = [None]
+        
+        def execute_in_thread():
+            try:
+                result_container[0] = registry.execute_phase(phase, context)
+            except Exception as e:
+                exception_container[0] = e
+        
+        thread = threading.Thread(target=execute_in_thread, daemon=True)
+        thread.start()
+        thread.join(timeout=timeout)
+        
+        if thread.is_alive():
+            # 超时了
+            error_msg = f"Phase {phase.value} execution timeout after {timeout} seconds"
+            context.add_warning(error_msg)
+            return PhaseResult(
+                phase=phase,
+                success=False,
+                error=TimeoutError(error_msg)
+            )
+        
+        # 检查是否有异常
+        if exception_container[0]:
+            raise exception_container[0]
+        
+        # 返回结果（已经确保不为None）
+        if result_container[0] is None:
+            raise RuntimeError(f"Phase {phase.value} returned None result")
+        return result_container[0]
+    
     def _check_dependencies(self, phase: InitializationPhase, 
                           context: InitializationContext) -> bool:
         """检查阶段依赖关系"""

crawlo/initialization/phases.py

@@ -6,7 +6,7 @@
 
 from enum import Enum
 from dataclasses import dataclass
-from typing import List, Optional
+from typing import List, Optional, Dict
 
 
 class InitializationPhase(Enum):
@@ -146,4 +146,85 @@ def validate_dependencies() -> bool:
             if dependency not in phases:
                 return False
     
-    return True
+    return True
+
+
+def detect_circular_dependencies() -> Optional[List[InitializationPhase]]:
+    """
+    检测循环依赖
+    
+    使用DFS（深度优先搜索）算法检测初始化阶段的循环依赖。
+    
+    Returns:
+        Optional[List[InitializationPhase]]: 如果存在循环，返回循环路径；否则返回None
+    
+    算法说明：
+        使用三色标记法：
+        - 白色(0)：未访问
+        - 灰色(1)：正在访问（在当前DFS路径中）
+        - 黑色(2)：已完成访问
+        
+        如果在DFS过程中遇到灰色节点，说明存在循环依赖。
+    """
+    # 构建依赖图
+    dependency_graph: Dict[InitializationPhase, List[InitializationPhase]] = {}
+    for definition in PHASE_DEFINITIONS:
+        dependency_graph[definition.phase] = definition.dependencies.copy()
+    
+    # 三色标记：0-白色(未访问)，1-灰色(访问中)，2-黑色(已完成)
+    color: Dict[InitializationPhase, int] = {phase: 0 for phase in dependency_graph}
+    parent: Dict[InitializationPhase, Optional[InitializationPhase]] = {phase: None for phase in dependency_graph}
+    
+    def dfs(node: InitializationPhase) -> Optional[List[InitializationPhase]]:
+        """DFS遍历检测循环"""
+        color[node] = 1  # 标记为灰色（访问中）
+        
+        for neighbor in dependency_graph.get(node, []):
+            if color[neighbor] == 1:  # 遇到灰色节点，发现循环
+                # 重建循环路径
+                cycle = [neighbor]
+                current: Optional[InitializationPhase] = node
+                while current is not None and current != neighbor:
+                    cycle.append(current)
+                    current = parent.get(current)
+                cycle.append(neighbor)
+                cycle.reverse()
+                return cycle
+            
+            if color[neighbor] == 0:  # 未访问的节点
+                parent[neighbor] = node
+                result = dfs(neighbor)
+                if result:
+                    return result
+        
+        color[node] = 2  # 标记为黑色（已完成）
+        return None
+    
+    # 对所有未访问的节点执行DFS
+    for phase in dependency_graph:
+        if color[phase] == 0:
+            cycle = dfs(phase)
+            if cycle:
+                return cycle
+    
+    return None
+
+
+def validate_phase_dependencies() -> tuple[bool, Optional[str]]:
+    """
+    全面验证阶段依赖关系
+    
+    Returns:
+        tuple[bool, Optional[str]]: (是否有效, 错误信息)
+    """
+    # 1. 检查依赖是否存在
+    if not validate_dependencies():
+        return False, "存在未定义的依赖阶段"
+    
+    # 2. 检查循环依赖
+    cycle = detect_circular_dependencies()
+    if cycle:
+        cycle_path = ' -> '.join([phase.value for phase in cycle])
+        return False, f"检测到循环依赖: {cycle_path}"
+    
+    return True, None

crawlo/initialization/registry.py

@@ -35,11 +35,12 @@ class BaseInitializer(Initializer):
     def _create_result(self, success: bool, duration: float = 0.0, 
                       artifacts: Optional[Dict] = None, error: Optional[Exception] = None) -> PhaseResult:
         """创建初始化结果"""
-        return PhaseResult(
+        from .utils import create_initialization_result
+        return create_initialization_result(
             phase=self.phase,
             success=success,
             duration=duration,
-            artifacts=artifacts or {},
+            artifacts=artifacts,
             error=error
         )
 
@@ -70,15 +71,12 @@ class InitializerRegistry:
                          init_func: Callable[[InitializationContext], PhaseResult]):
         """注册函数式初始化器"""
         
-        class FunctionInitializer:
+        class FunctionInitializer(Initializer):
             def __init__(self, phase: InitializationPhase, func: Callable):
+                super().__init__(phase)
                 self._phase = phase
                 self._func = func
             
-            @property  
-            def phase(self) -> InitializationPhase:
-                return self._phase
-            
             def initialize(self, context: InitializationContext) -> PhaseResult:
                 return self._func(context)
         

crawlo/initialization/utils.py

@@ -0,0 +1,49 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+初始化工具模块 - 提供通用的初始化工具函数
+"""
+
+import time
+from typing import Optional, Dict, Any
+from .phases import PhaseResult, InitializationPhase
+
+
+def create_initialization_result(
+    phase: 'InitializationPhase',
+    success: bool, 
+    duration: float = 0.0,
+    artifacts: Optional[Dict[str, Any]] = None, 
+    error: Optional[Exception] = None
+) -> PhaseResult:
+    """
+    创建标准化的初始化结果
+    
+    Args:
+        phase: 初始化阶段
+        success: 是否成功
+        duration: 执行时长
+        artifacts: 产生的工件数据
+        error: 异常对象
+        
+    Returns:
+        PhaseResult: 标准化的初始化结果
+    """
+    return PhaseResult(
+        phase=phase,
+        success=success,
+        duration=duration,
+        artifacts=artifacts or {},
+        error=error
+    )
+
+
+class InitializationTimer:
+    """初始化计时器"""
+    
+    def __init__(self):
+        self.start_time = time.time()
+    
+    def get_duration(self) -> float:
+        """获取经过的时间"""
+        return time.time() - self.start_time

crawlo/logging/__init__.py

@@ -14,33 +14,29 @@ Crawlo统一日志系统
 from .manager import LogManager
 from .factory import LoggerFactory
 from .config import LogConfig
-from .monitor import LogPerformanceMonitor
+
 
 # 统一的公共接口
 def get_logger(name: str = 'default'):
     """获取logger实例"""
     return LoggerFactory.get_logger(name)
 
+
 def configure_logging(settings=None, **kwargs):
     """配置日志系统"""
     return LogManager().configure(settings, **kwargs)
 
+
 def is_configured() -> bool:
     """检查日志系统是否已配置"""
     return LogManager().is_configured
 
-def get_monitor() -> LogPerformanceMonitor:
-    """获取日志性能监控器"""
-    from .monitor import get_monitor as _get_monitor
-    return _get_monitor()
 
 __all__ = [
     'LogManager',
-    'LoggerFactory', 
+    'LoggerFactory',
     'LogConfig',
-    'LogPerformanceMonitor',
     'get_logger',
     'configure_logging',
-    'is_configured',
-    'get_monitor'
-]
+    'is_configured'
+]

crawlo/logging/config.py

@@ -13,6 +13,45 @@ from typing import Optional, Dict, Any
 class LogConfig:
     """日志配置数据类 - 简单明确的配置结构"""
     
+    # 预设配置模板
+    TEMPLATES = {
+        'minimal': {
+            'level': 'INFO',
+            'format': '%(asctime)s - %(levelname)s: %(message)s',
+            'console_enabled': True,
+            'file_enabled': False
+        },
+        'standard': {
+            'level': 'INFO',
+            'format': '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s',
+            'console_enabled': True,
+            'file_enabled': True,
+            'file_path': 'logs/crawlo.log',
+            # 注意：standard模板未指定max_bytes和backup_count，
+            # 将使用类定义的默认值(10MB, 5个备份)或用户在settings.py中设置的值
+            # 如果用户不想要日志轮转，可以在settings.py中设置LOG_MAX_BYTES=0
+            # 当max_bytes或backup_count为0时，日志轮转将被禁用，文件会持续增长
+        },
+        'detailed': {
+            'level': 'DEBUG',
+            'format': '%(asctime)s - [%(name)s] - %(levelname)s - %(pathname)s:%(lineno)d: %(message)s',
+            'console_enabled': True,
+            'file_enabled': True,
+            'file_path': 'logs/crawlo.log',
+            'max_bytes': 20 * 1024 * 1024,  # 20MB，适用于大多数生产环境
+            'backup_count': 10  # 10个备份文件，可保留约10次轮转的历史
+        },
+        'production': {
+            'level': 'WARNING',
+            'format': '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s',
+            'console_enabled': False,  # 生产环境通常禁用控制台输出
+            'file_enabled': True,
+            'file_path': 'logs/crawlo.log',
+            'max_bytes': 50 * 1024 * 1024,  # 50MB，适用于高负载生产环境
+            'backup_count': 20  # 20个备份文件，可保留较长时间的历史记录
+        }
+    }
+    
     # 基本配置
     level: str = "INFO"
     format: str = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
@@ -54,21 +93,45 @@ class LogConfig:
         # 获取默认值
         format_default_value = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
         
+        # 确保类型安全
+        def safe_get_str(key: str, default: str = '') -> str:
+            value = get_val(key, default)
+            return str(value) if value is not None else default
+        
+        def safe_get_int(key: str, default: int) -> int:
+            value = get_val(key, default)
+            try:
+                return int(value) if value is not None else default
+            except (ValueError, TypeError):
+                return default
+        
+        def safe_get_bool(key: str, default: bool) -> bool:
+            value = get_val(key, default)
+            if isinstance(value, bool):
+                return value
+            if isinstance(value, str):
+                return value.lower() in ('1', 'true', 'yes', 'on')
+            return bool(value) if value is not None else default
+        
+        def safe_get_dict(key: str, default: dict) -> dict:
+            value = get_val(key, default)
+            return value if isinstance(value, dict) else default
+        
         return cls(
-            level=get_val('LOG_LEVEL', 'INFO'),
-            format=get_val('LOG_FORMAT', format_default_value),
-            encoding=get_val('LOG_ENCODING', 'utf-8'),
-            file_path=get_val('LOG_FILE'),
-            max_bytes=get_val('LOG_MAX_BYTES', 10 * 1024 * 1024),
-            backup_count=get_val('LOG_BACKUP_COUNT', 5),
-            console_enabled=get_val('LOG_CONSOLE_ENABLED', True),
-            file_enabled=get_val('LOG_FILE_ENABLED', True),
-            console_level=get_val('LOG_CONSOLE_LEVEL'),  # 允许单独设置控制台级别
-            file_level=get_val('LOG_FILE_LEVEL'),  # 允许单独设置文件级别
-            include_thread_id=get_val('LOG_INCLUDE_THREAD_ID', False),
-            include_process_id=get_val('LOG_INCLUDE_PROCESS_ID', False),
-            include_module_path=get_val('LOG_INCLUDE_MODULE_PATH', False),
-            module_levels=get_val('LOG_LEVELS', {})
+            level=safe_get_str('LOG_LEVEL', 'INFO'),
+            format=safe_get_str('LOG_FORMAT', format_default_value),
+            encoding=safe_get_str('LOG_ENCODING', 'utf-8'),
+            file_path=safe_get_str('LOG_FILE'),
+            max_bytes=safe_get_int('LOG_MAX_BYTES', 10 * 1024 * 1024),  # 从200MB改为10MB以保持一致性
+            backup_count=safe_get_int('LOG_BACKUP_COUNT', 5),
+            console_enabled=safe_get_bool('LOG_CONSOLE_ENABLED', True),
+            file_enabled=safe_get_bool('LOG_FILE_ENABLED', True),
+            console_level=safe_get_str('LOG_CONSOLE_LEVEL'),  # 允许单独设置控制台级别
+            file_level=safe_get_str('LOG_FILE_LEVEL'),  # 允许单独设置文件级别
+            include_thread_id=safe_get_bool('LOG_INCLUDE_THREAD_ID', False),
+            include_process_id=safe_get_bool('LOG_INCLUDE_PROCESS_ID', False),
+            include_module_path=safe_get_bool('LOG_INCLUDE_MODULE_PATH', False),
+            module_levels=safe_get_dict('LOG_LEVELS', {})
         )
     
     @classmethod
@@ -101,6 +164,22 @@ class LogConfig:
                 
         return cls(**mapped_dict)
     
+    @classmethod
+    def from_template(cls, template_name: str) -> 'LogConfig':
+        """从模板创建配置
+        
+        Args:
+            template_name: 模板名称 (minimal, standard, detailed, production)
+            
+        Returns:
+            LogConfig: 配置对象
+        """
+        if template_name not in cls.TEMPLATES:
+            raise ValueError(f"未知的模板名称: {template_name}，可用模板: {', '.join(cls.TEMPLATES.keys())}")
+            
+        template_config = cls.TEMPLATES[template_name]
+        return cls(**template_config)
+    
     def get_module_level(self, module_name: str) -> str:
         """获取模块的日志级别"""
         # 先查找精确匹配
@@ -169,21 +248,25 @@ class LogConfig:
                 
         return base_format
     
-    def validate(self) -> bool:
-        """验证配置有效性"""
+    def validate(self) -> tuple[bool, str]:
+        """验证配置有效性
+        
+        Returns:
+            tuple[bool, str]: (是否有效, 错误信息)
+        """
         valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}
         
         # 验证主级别
         if self.level.upper() not in valid_levels:
-            return False
+            return False, f"无效的日志级别: {self.level}，有效级别为: {', '.join(valid_levels)}"
         
         # 验证控制台级别
         if self.console_level and self.console_level.upper() not in valid_levels:
-            return False
+            return False, f"无效的控制台日志级别: {self.console_level}，有效级别为: {', '.join(valid_levels)}"
         
         # 验证文件级别
         if self.file_level and self.file_level.upper() not in valid_levels:
-            return False
+            return False, f"无效的文件日志级别: {self.file_level}，有效级别为: {', '.join(valid_levels)}"
         
         # 确保日志目录存在
         if self.file_path and self.file_enabled:
@@ -191,7 +274,8 @@ class LogConfig:
                 log_dir = os.path.dirname(self.file_path)
                 if log_dir and not os.path.exists(log_dir):
                     os.makedirs(log_dir, exist_ok=True)
-            except (OSError, PermissionError):
-                return False
+            except (OSError, PermissionError) as e:
+                log_dir = os.path.dirname(self.file_path) if self.file_path else "未知"
+                return False, f"无法创建日志目录 {log_dir}: {e}"
         
-        return True
+        return True, "配置有效"

crawlo/logging/factory.py

@@ -8,16 +8,17 @@ import logging
 import os
 import sys
 import threading
-from typing import Dict, Optional
 from weakref import WeakValueDictionary
 
 # 尝试导入concurrent-log-handler，如果不可用则回退到标准库
 try:
     from concurrent_log_handler import ConcurrentRotatingFileHandler
     USE_CONCURRENT_HANDLER = True
+    RotatingFileHandler = ConcurrentRotatingFileHandler  # 别名以避免未绑定错误
 except ImportError:
     from logging.handlers import RotatingFileHandler
     USE_CONCURRENT_HANDLER = False
+    ConcurrentRotatingFileHandler = RotatingFileHandler  # 别名以避免未绑定错误
 
 from .manager import get_config, is_configured, configure
 from .config import LogConfig
@@ -69,7 +70,7 @@ class LoggerFactory:
         """创建新的Logger实例"""
         config = get_config()
         if not config:
-            raise RuntimeError("Log system not configured")
+            raise RuntimeError("日志系统未配置，请先调用 configure_logging() 进行配置")
         
         # 创建Logger
         logger = logging.getLogger(name)
@@ -103,6 +104,7 @@ class LoggerFactory:
                     os.makedirs(log_dir, exist_ok=True)
                 
                 # 根据平台选择合适的Handler
+                file_handler = None
                 if USE_CONCURRENT_HANDLER:
                     file_handler = ConcurrentRotatingFileHandler(
                         filename=config.file_path,
@@ -133,12 +135,14 @@ class LoggerFactory:
                         encoding=config.encoding
                     )
                 
-                file_handler.setFormatter(formatter)
-                # 使用专门的文件级别或模块级别
-                file_level = config.get_file_level()
-                level = getattr(logging, file_level.upper(), logging.INFO)
-                file_handler.setLevel(level)
-                logger.addHandler(file_handler)
+                # 添加文件处理器（如果创建成功）
+                if file_handler is not None:
+                    file_handler.setFormatter(formatter)
+                    # 使用专门的文件级别或模块级别
+                    file_level = config.get_file_level()
+                    level = getattr(logging, file_level.upper(), logging.INFO)
+                    file_handler.setLevel(level)
+                    logger.addHandler(file_handler)
             except Exception as e:
                 # 文件Handler创建失败时，至少保证控制台输出
                 console_handler = logging.StreamHandler()

crawlo/logging/manager.py

@@ -5,10 +5,12 @@
 """
 
 import threading
-from typing import Optional, Any
+from typing import Optional
 from .config import LogConfig
+from crawlo.utils.singleton import singleton
 
 
+@singleton
 class LogManager:
     """
     日志管理器 - 单例模式
@@ -18,38 +20,23 @@ class LogManager:
     2. 配置状态跟踪
     3. 线程安全的配置更新
     """
-    
-    _instance: Optional['LogManager'] = None
-    _lock = threading.Lock()
-    
-    def __new__(cls) -> 'LogManager':
-        if cls._instance is None:
-            with cls._lock:
-                if cls._instance is None:
-                    cls._instance = super(LogManager, cls).__new__(cls)
-                    cls._instance._initialized = False
-        return cls._instance
-    
+
     def __init__(self):
-        if hasattr(self, '_initialized') and self._initialized:
-            return
-            
         self._config: Optional[LogConfig] = None
         self._configured = False
         self._config_lock = threading.RLock()
-        self._initialized = True
-    
+
     @property
     def config(self) -> Optional[LogConfig]:
         """获取当前配置"""
         with self._config_lock:
             return self._config
-    
+
     @property
     def is_configured(self) -> bool:
         """检查是否已配置"""
         return self._configured
-    
+
     def configure(self, settings=None, **kwargs) -> LogConfig:
         """
         配置日志系统
@@ -74,16 +61,17 @@ class LogManager:
                 config = LogConfig.from_dict(kwargs)
             else:
                 config = LogConfig()  # 使用默认配置
-            
+
             # 验证配置
-            if not config.validate():
-                raise ValueError("Invalid log configuration")
-            
+            is_valid, error_msg = config.validate()
+            if not is_valid:
+                raise ValueError(f"Invalid log configuration: {error_msg}")
+
             self._config = config
             self._configured = True
-            
+
             return config
-    
+
     def reset(self):
         """重置配置（主要用于测试）"""
         with self._config_lock:
@@ -94,19 +82,23 @@ class LogManager:
 # 全局实例
 _log_manager = LogManager()
 
+
 # 模块级便捷函数
 def configure(settings=None, **kwargs) -> LogConfig:
     """配置日志系统"""
     return _log_manager.configure(settings, **kwargs)
 
+
 def is_configured() -> bool:
     """检查是否已配置"""
     return _log_manager.is_configured
 
+
 def get_config() -> Optional[LogConfig]:
     """获取当前配置"""
     return _log_manager.config
 
+
 def reset():
     """重置配置"""
-    _log_manager.reset()
+    _log_manager.reset()