crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/initialization/core.py

@@ -0,0 +1,194 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+核心初始化器 - 协调整个初始化过程
+"""
+
+import time
+import threading
+from typing import Optional, Any
+
+from .context import InitializationContext  
+from .phases import InitializationPhase, PhaseResult, get_execution_order, get_phase_definition
+from .registry import get_global_registry, BaseInitializer, register_initializer
+from .built_in import register_built_in_initializers
+
+
+class CoreInitializer:
+    """
+    核心初始化器 - 协调整个框架的初始化过程
+    
+    职责：
+    1. 管理初始化阶段的执行顺序
+    2. 处理阶段间的依赖关系
+    3. 提供统一的初始化入口
+    4. 错误处理和降级策略
+    """
+    
+    _instance: Optional['CoreInitializer'] = None
+    _lock = threading.Lock()
+    
+    def __new__(cls) -> 'CoreInitializer':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super(CoreInitializer, cls).__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        if hasattr(self, '_initialized') and self._initialized:
+            return
+            
+        self._context: Optional[InitializationContext] = None
+        self._is_ready = False
+        self._init_lock = threading.RLock()
+        
+        # 注册内置初始化器
+        register_built_in_initializers()
+        
+        self._initialized = True
+    
+    @property
+    def context(self) -> Optional[InitializationContext]:
+        """获取初始化上下文"""
+        return self._context
+    
+    @property  
+    def is_ready(self) -> bool:
+        """检查框架是否已准备就绪"""
+        return self._is_ready
+    
+    def initialize(self, settings=None, **kwargs) -> Any:
+        """
+        执行框架初始化
+        
+        Args:
+            settings: 配置对象
+            **kwargs: 额外的配置参数
+            
+        Returns:
+            初始化后的配置管理器
+        """
+        with self._init_lock:
+            # 如果已经初始化完成，直接返回
+            if self._is_ready and self._context and self._context.settings:
+                return self._context.settings
+            
+            # 创建初始化上下文
+            context = InitializationContext()
+            context.custom_settings = kwargs
+            self._context = context
+            
+            try:
+                # 执行初始化阶段
+                self._execute_initialization_phases(context)
+                
+                # 检查关键阶段是否完成
+                if not context.is_phase_completed(InitializationPhase.SETTINGS):
+                    raise RuntimeError("Settings initialization failed")
+                
+                self._is_ready = True
+                context.finish()
+                
+                return context.settings
+                
+            except Exception as e:
+                context.add_error(f"Framework initialization failed: {e}")
+                context.finish()
+                
+                # 降级策略
+                return self._fallback_initialization(settings, **kwargs)
+    
+    def _execute_initialization_phases(self, context: InitializationContext):
+        """执行初始化阶段"""
+        registry = get_global_registry()
+        execution_order = get_execution_order()
+        
+        # 只执行已注册的阶段
+        registered_phases = set(registry.get_all_phases())
+        
+        for phase in execution_order:
+            if phase == InitializationPhase.ERROR:
+                continue
+                
+            # 只执行已注册的阶段
+            if phase not in registered_phases:
+                continue
+                
+            context.set_current_phase(phase)
+            
+            # 检查依赖关系
+            if not self._check_dependencies(phase, context):
+                phase_def = get_phase_definition(phase)
+                if not (phase_def and phase_def.optional):
+                    raise RuntimeError(f"Dependencies not satisfied for phase {phase}")
+                else:
+                    # 可选阶段，跳过
+                    continue
+            
+            # 执行阶段
+            start_time = time.time()
+            try:
+                result = registry.execute_phase(phase, context)
+                result.duration = time.time() - start_time
+                
+                context.mark_phase_completed(phase, result)
+                
+                if not result.success and not self._is_phase_optional(phase):
+                    raise RuntimeError(f"Phase {phase} failed: {result.error}")
+                    
+            except Exception as e:
+                duration = time.time() - start_time
+                result = PhaseResult(
+                    phase=phase,
+                    success=False,
+                    duration=duration,
+                    error=e
+                )
+                context.mark_phase_completed(phase, result)
+                
+                if not self._is_phase_optional(phase):
+                    raise
+    
+    def _check_dependencies(self, phase: InitializationPhase, 
+                          context: InitializationContext) -> bool:
+        """检查阶段依赖关系"""
+        phase_def = get_phase_definition(phase)
+        if not phase_def:
+            return True
+        
+        for dependency in phase_def.dependencies:
+            if not context.is_phase_completed(dependency):
+                return False
+        
+        return True
+    
+    def _is_phase_optional(self, phase: InitializationPhase) -> bool:
+        """检查阶段是否可选"""
+        phase_def = get_phase_definition(phase)
+        return phase_def.optional if phase_def else False
+    
+    def _fallback_initialization(self, settings=None, **kwargs):
+        """降级初始化策略"""
+        try:
+            # 尝试创建基本的配置管理器
+            from crawlo.settings.setting_manager import SettingManager
+            
+            if settings:
+                return settings
+            else:
+                fallback_settings = SettingManager()
+                if kwargs:
+                    fallback_settings.update_attributes(kwargs)
+                return fallback_settings
+                
+        except Exception:
+            # 如果连降级都失败，返回None
+            return None
+    
+    def reset(self):
+        """重置初始化状态（主要用于测试）"""
+        with self._init_lock:
+            self._context = None
+            self._is_ready = False

crawlo/initialization/phases.py

@@ -0,0 +1,149 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+初始化阶段定义
+"""
+
+from enum import Enum
+from dataclasses import dataclass
+from typing import List, Optional
+
+
+class InitializationPhase(Enum):
+    """初始化阶段枚举"""
+    
+    # 阶段0：准备阶段
+    PREPARING = "preparing"
+    
+    # 阶段1：日志系统初始化
+    LOGGING = "logging"
+    
+    # 阶段2：配置系统初始化  
+    SETTINGS = "settings"
+    
+    # 阶段3：核心组件初始化
+    CORE_COMPONENTS = "core_components"
+    
+    # 阶段4：扩展组件初始化
+    EXTENSIONS = "extensions"
+    
+    # 阶段5：框架启动日志记录
+    FRAMEWORK_STARTUP_LOG = "framework_startup_log"
+    
+    # 阶段6：完成
+    COMPLETED = "completed"
+    
+    # 错误状态
+    ERROR = "error"
+
+
+@dataclass
+class PhaseResult:
+    """阶段执行结果"""
+    phase: InitializationPhase
+    success: bool
+    duration: float = 0.0
+    error: Optional[Exception] = None
+    artifacts: dict = None  # 阶段产生的工件
+    
+    def __post_init__(self):
+        if self.artifacts is None:
+            self.artifacts = {}
+
+
+@dataclass 
+class PhaseDefinition:
+    """阶段定义"""
+    phase: InitializationPhase
+    name: str
+    description: str
+    dependencies: List[InitializationPhase] = None
+    optional: bool = False
+    timeout: float = 30.0  # 超时时间（秒）
+    
+    def __post_init__(self):
+        if self.dependencies is None:
+            self.dependencies = []
+
+
+# 预定义的初始化阶段
+PHASE_DEFINITIONS = [
+    PhaseDefinition(
+        phase=InitializationPhase.PREPARING,
+        name="准备阶段",
+        description="初始化基础环境和检查前置条件",
+        dependencies=[],
+        timeout=5.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.LOGGING,
+        name="日志系统",
+        description="配置和初始化日志系统",
+        dependencies=[],  # 移除对PREPARING的依赖
+        timeout=10.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.SETTINGS,
+        name="配置系统", 
+        description="加载和验证配置",
+        dependencies=[InitializationPhase.LOGGING],
+        timeout=15.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.CORE_COMPONENTS,
+        name="核心组件",
+        description="初始化框架核心组件",
+        dependencies=[InitializationPhase.SETTINGS],
+        timeout=20.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.EXTENSIONS,
+        name="扩展组件",
+        description="加载和初始化扩展组件",
+        dependencies=[InitializationPhase.CORE_COMPONENTS],
+        optional=True,
+        timeout=15.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.FRAMEWORK_STARTUP_LOG,
+        name="框架启动日志",
+        description="记录框架启动相关信息",
+        dependencies=[InitializationPhase.LOGGING, InitializationPhase.SETTINGS],
+        timeout=5.0
+    ),
+    PhaseDefinition(
+        phase=InitializationPhase.COMPLETED,
+        name="初始化完成",
+        description="框架初始化完成",
+        dependencies=[
+            InitializationPhase.CORE_COMPONENTS,
+            InitializationPhase.FRAMEWORK_STARTUP_LOG
+        ],  # Extensions是可选的
+        timeout=5.0
+    )
+]
+
+
+def get_phase_definition(phase: InitializationPhase) -> Optional[PhaseDefinition]:
+    """获取阶段定义"""
+    for definition in PHASE_DEFINITIONS:
+        if definition.phase == phase:
+            return definition
+    return None
+
+
+def get_execution_order() -> List[InitializationPhase]:
+    """获取执行顺序"""
+    return [definition.phase for definition in PHASE_DEFINITIONS]
+
+
+def validate_dependencies() -> bool:
+    """验证阶段依赖关系的正确性"""
+    phases = {definition.phase for definition in PHASE_DEFINITIONS}
+    
+    for definition in PHASE_DEFINITIONS:
+        for dependency in definition.dependencies:
+            if dependency not in phases:
+                return False
+    
+    return True

crawlo/initialization/registry.py

@@ -0,0 +1,146 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+初始化器注册表 - 管理所有初始化器的注册和执行
+"""
+
+import threading
+from typing import Dict, Optional, Callable, List
+from .context import InitializationContext
+from .phases import InitializationPhase, PhaseResult
+
+
+class Initializer:
+    """初始化器基类"""
+    
+    def __init__(self, phase: InitializationPhase):
+        self._phase = phase
+    
+    @property
+    def phase(self) -> InitializationPhase:
+        """获取初始化阶段"""
+        return self._phase
+    
+    def initialize(self, context: InitializationContext) -> PhaseResult:
+        """执行初始化 - 子类必须实现"""
+        raise NotImplementedError("Subclasses must implement initialize method")
+
+
+class BaseInitializer(Initializer):
+    """基础初始化器类 - 为向后兼容保留"""
+    
+    def __init__(self, phase: InitializationPhase):
+        super().__init__(phase)
+    
+    def _create_result(self, success: bool, duration: float = 0.0, 
+                      artifacts: Optional[Dict] = None, error: Optional[Exception] = None) -> PhaseResult:
+        """创建初始化结果"""
+        return PhaseResult(
+            phase=self.phase,
+            success=success,
+            duration=duration,
+            artifacts=artifacts or {},
+            error=error
+        )
+
+
+class InitializerRegistry:
+    """
+    初始化器注册表 - 管理所有初始化器的注册和执行
+    
+    特点：
+    1. 线程安全的注册和执行
+    2. 支持函数式和类式初始化器
+    3. 统一的结果处理
+    """
+    
+    def __init__(self):
+        self._initializers: Dict[InitializationPhase, Initializer] = {}
+        self._lock = threading.RLock()
+    
+    def register(self, initializer: Initializer):
+        """注册初始化器"""
+        with self._lock:
+            phase = initializer.phase
+            if phase in self._initializers:
+                raise ValueError(f"Initializer for phase {phase} already registered")
+            self._initializers[phase] = initializer
+    
+    def register_function(self, phase: InitializationPhase, 
+                         init_func: Callable[[InitializationContext], PhaseResult]):
+        """注册函数式初始化器"""
+        
+        class FunctionInitializer:
+            def __init__(self, phase: InitializationPhase, func: Callable):
+                self._phase = phase
+                self._func = func
+            
+            @property  
+            def phase(self) -> InitializationPhase:
+                return self._phase
+            
+            def initialize(self, context: InitializationContext) -> PhaseResult:
+                return self._func(context)
+        
+        self.register(FunctionInitializer(phase, init_func))
+    
+    def get_initializer(self, phase: InitializationPhase) -> Optional[Initializer]:
+        """获取指定阶段的初始化器"""
+        with self._lock:
+            return self._initializers.get(phase)
+    
+    def get_all_phases(self) -> List[InitializationPhase]:
+        """获取所有已注册的阶段"""
+        with self._lock:
+            return list(self._initializers.keys())
+    
+    def has_initializer(self, phase: InitializationPhase) -> bool:
+        """检查是否有指定阶段的初始化器"""
+        with self._lock:
+            return phase in self._initializers
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._initializers.clear()
+    
+    def execute_phase(self, phase: InitializationPhase, 
+                     context: InitializationContext) -> PhaseResult:
+        """执行指定阶段的初始化"""
+        initializer = self.get_initializer(phase)
+        if not initializer:
+            error = ValueError(f"No initializer registered for phase {phase}")
+            return PhaseResult(
+                phase=phase,
+                success=False,
+                error=error
+            )
+        
+        try:
+            return initializer.initialize(context)
+        except Exception as e:
+            return PhaseResult(
+                phase=phase,
+                success=False,
+                error=e
+            )
+
+
+# 全局注册表实例
+_global_registry = InitializerRegistry()
+
+
+def get_global_registry() -> InitializerRegistry:
+    """获取全局注册表"""
+    return _global_registry
+
+
+def register_initializer(initializer: Initializer):
+    """注册初始化器到全局注册表"""
+    _global_registry.register(initializer)
+
+
+def register_phase_function(phase: InitializationPhase, 
+                           init_func: Callable[[InitializationContext], PhaseResult]):
+    """注册函数式初始化器到全局注册表"""
+    _global_registry.register_function(phase, init_func)

crawlo/items/base.py

@@ -6,6 +6,7 @@
 from abc import ABCMeta
 from .fields import Field
 
+
 class ItemMeta(ABCMeta):
     def __new__(mcs, name, bases, attrs):
         fields = {}
@@ -19,4 +20,4 @@ class ItemMeta(ABCMeta):
 
         cls_instance = super().__new__(mcs, name, bases, cls_attrs)
         cls_instance.FIELDS = fields
-        return cls_instance
+        return cls_instance

crawlo/logging/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo统一日志系统
+=================
+
+设计原则：
+1. 简单优先 - 避免过度设计
+2. 性能优先 - 减少锁竞争和复杂逻辑  
+3. 一致性 - 统一的日志接口
+4. 可靠性 - 确保日志始终可用
+"""
+
+from .manager import LogManager
+from .factory import LoggerFactory
+from .config import LogConfig
+
+# 统一的公共接口
+def get_logger(name: str = 'default'):
+    """获取logger实例"""
+    return LoggerFactory.get_logger(name)
+
+def configure_logging(settings=None, **kwargs):
+    """配置日志系统"""
+    return LogManager().configure(settings, **kwargs)
+
+def is_configured() -> bool:
+    """检查日志系统是否已配置"""
+    return LogManager().is_configured
+
+__all__ = [
+    'LogManager',
+    'LoggerFactory', 
+    'LogConfig',
+    'get_logger',
+    'configure_logging',
+    'is_configured'
+]

crawlo/logging/config.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志配置管理
+"""
+
+import os
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+
+
+@dataclass
+class LogConfig:
+    """日志配置数据类 - 简单明确的配置结构"""
+    
+    # 基本配置
+    level: str = "INFO"
+    format: str = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+    encoding: str = "utf-8"
+    
+    # 文件配置
+    file_path: Optional[str] = None
+    max_bytes: int = 10 * 1024 * 1024  # 10MB
+    backup_count: int = 5
+    
+    # 控制台配置
+    console_enabled: bool = True
+    file_enabled: bool = True
+    
+    # 模块级别配置
+    module_levels: Dict[str, str] = field(default_factory=dict)
+    
+    @classmethod
+    def from_settings(cls, settings) -> 'LogConfig':
+        """从settings对象创建配置"""
+        if not settings:
+            return cls()
+            
+        # 使用settings的get方法而不是getattr
+        if hasattr(settings, 'get'):
+            get_val = settings.get
+        else:
+            get_val = lambda k, d=None: getattr(settings, k, d)
+        
+        # 获取默认值
+        format_default_value = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+        
+        return cls(
+            level=get_val('LOG_LEVEL', 'INFO'),
+            format=get_val('LOG_FORMAT', format_default_value),
+            encoding=get_val('LOG_ENCODING', 'utf-8'),
+            file_path=get_val('LOG_FILE'),
+            max_bytes=get_val('LOG_MAX_BYTES', 10 * 1024 * 1024),
+            backup_count=get_val('LOG_BACKUP_COUNT', 5),
+            console_enabled=get_val('LOG_CONSOLE_ENABLED', True),
+            file_enabled=get_val('LOG_FILE_ENABLED', True),
+            module_levels=get_val('LOG_LEVELS', {})
+        )
+    
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any]) -> 'LogConfig':
+        """从字典创建配置"""
+        return cls(**{k: v for k, v in config_dict.items() if k in cls.__annotations__})
+    
+    def get_module_level(self, module_name: str) -> str:
+        """获取模块的日志级别"""
+        # 先查找精确匹配
+        if module_name in self.module_levels:
+            return self.module_levels[module_name]
+        
+        # 查找父模块匹配
+        parts = module_name.split('.')
+        for i in range(len(parts) - 1, 0, -1):
+            parent_module = '.'.join(parts[:i])
+            if parent_module in self.module_levels:
+                return self.module_levels[parent_module]
+        
+        # 返回默认级别
+        return self.level
+    
+    def validate(self) -> bool:
+        """验证配置有效性"""
+        valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}
+        
+        if self.level.upper() not in valid_levels:
+            return False
+        
+        # 确保日志目录存在
+        if self.file_path and self.file_enabled:
+            try:
+                log_dir = os.path.dirname(self.file_path)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+            except (OSError, PermissionError):
+                return False
+        
+        return True