crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/utils/enhanced_error_handler.py

@@ -0,0 +1,360 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+增强版错误处理工具
+提供更详细、更一致的错误处理和日志记录机制
+"""
+import traceback
+import logging
+import sys
+import os
+from typing import Optional, Callable, Any, Dict, List
+from functools import wraps
+from datetime import datetime
+
+from crawlo.utils.log import get_logger
+
+
+class ErrorContext:
+    """错误上下文信息"""
+    
+    def __init__(self, context: str = "", module: str = "", function: str = ""):
+        self.context = context
+        self.module = module
+        self.function = function
+        self.timestamp = datetime.now()
+        
+    def __str__(self):
+        parts = []
+        if self.module:
+            parts.append(f"Module: {self.module}")
+        if self.function:
+            parts.append(f"Function: {self.function}")
+        if self.context:
+            parts.append(f"Context: {self.context}")
+        parts.append(f"Time: {self.timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
+        return " | ".join(parts)
+
+
+class DetailedException(Exception):
+    """带有详细信息的异常基类"""
+    
+    def __init__(self, message: str, context: Optional[ErrorContext] = None, 
+                 error_code: Optional[str] = None, **kwargs):
+        super().__init__(message)
+        self.context = context
+        self.error_code = error_code
+        self.details = kwargs
+        self.timestamp = datetime.now()
+        
+    def __str__(self):
+        base_msg = super().__str__()
+        if self.context:
+            return f"{base_msg} ({self.context})"
+        return base_msg
+    
+    def get_full_details(self) -> Dict:
+        """获取完整的错误详情"""
+        return {
+            "message": str(self),
+            "error_code": self.error_code,
+            "context": str(self.context) if self.context else None,
+            "details": self.details,
+            "timestamp": self.timestamp.isoformat(),
+            "exception_type": self.__class__.__name__
+        }
+
+
+class EnhancedErrorHandler:
+    """增强版错误处理器"""
+    
+    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
+        self.logger = get_logger(logger_name, log_level)
+        self.error_history: List[Dict] = []  # 错误历史记录
+        self.max_history_size = 100  # 最大历史记录数
+    
+    def handle_error(self, exception: Exception, context: Optional[ErrorContext] = None, 
+                     raise_error: bool = True, log_error: bool = True,
+                     extra_info: Optional[Dict] = None) -> Dict:
+        """
+        增强版错误处理
+        
+        Args:
+            exception: 异常对象
+            context: 错误上下文信息
+            raise_error: 是否重新抛出异常
+            log_error: 是否记录错误日志
+            extra_info: 额外的错误信息
+            
+        Returns:
+            包含错误详情的字典
+        """
+        # 构建错误详情
+        error_details = {
+            "exception": exception,
+            "exception_type": type(exception).__name__,
+            "message": str(exception),
+            "context": str(context) if context else None,
+            "timestamp": datetime.now().isoformat(),
+            "traceback": traceback.format_exc() if log_error else None,
+            "extra_info": extra_info or {}
+        }
+        
+        # 记录到历史
+        self._record_error(error_details)
+        
+        # 记录日志
+        if log_error:
+            self._log_error(error_details)
+        
+        # 重新抛出异常
+        if raise_error:
+            raise exception
+        
+        return error_details
+    
+    def _log_error(self, error_details: Dict):
+        """记录错误日志"""
+        # 基本错误信息
+        context_info = error_details.get("context", "")
+        message = error_details["message"]
+        error_msg = f"{message} [{context_info}]" if context_info else message
+        
+        # 记录错误
+        self.logger.error(error_msg)
+        
+        # 记录详细信息
+        if error_details.get("traceback"):
+            self.logger.debug(f"详细错误信息:\n{error_details['traceback']}")
+        
+        # 记录额外信息
+        if error_details.get("extra_info"):
+            self.logger.debug(f"额外信息: {error_details['extra_info']}")
+    
+    def _record_error(self, error_details: Dict):
+        """记录错误到历史"""
+        self.error_history.append(error_details)
+        # 限制历史记录大小
+        if len(self.error_history) > self.max_history_size:
+            self.error_history.pop(0)
+    
+    def safe_call(self, func: Callable, *args, default_return=None, 
+                  context: Optional[ErrorContext] = None, **kwargs) -> Any:
+        """
+        安全调用函数，捕获并处理异常
+        
+        Args:
+            func: 要调用的函数
+            *args: 函数参数
+            default_return: 默认返回值
+            context: 错误上下文
+            **kwargs: 函数关键字参数
+            
+        Returns:
+            函数返回值或默认值
+        """
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            self.handle_error(e, context=context, raise_error=False)
+            return default_return
+    
+    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
+                         exceptions: tuple = (Exception,), backoff_factor: float = 1.0,
+                         context: Optional[ErrorContext] = None):
+        """
+        装饰器：失败时重试（增强版）
+        
+        Args:
+            max_retries: 最大重试次数
+            delay: 初始重试间隔（秒）
+            exceptions: 需要重试的异常类型
+            backoff_factor: 退避因子（每次重试间隔乘以此因子）
+            context: 错误上下文
+        """
+        def decorator(func):
+            @wraps(func)
+            async def async_wrapper(*args, **kwargs):
+                last_exception = None
+                current_delay = delay
+                
+                for attempt in range(max_retries + 1):
+                    try:
+                        return await func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_retries:
+                            # 记录重试信息
+                            retry_context = ErrorContext(
+                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
+                                module=context.module if context else "",
+                                function=func.__name__
+                            ) if context else None
+                            
+                            self.logger.warning(
+                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
+                            )
+                            
+                            import asyncio
+                            await asyncio.sleep(current_delay)
+                            current_delay *= backoff_factor  # 指数退避
+                        else:
+                            # 最后一次尝试失败
+                            final_context = ErrorContext(
+                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
+                                module=context.module if context else "",
+                                function=func.__name__
+                            ) if context else None
+                            
+                            self.logger.error(
+                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
+                            )
+                raise last_exception
+            
+            @wraps(func)
+            def sync_wrapper(*args, **kwargs):
+                last_exception = None
+                current_delay = delay
+                
+                for attempt in range(max_retries + 1):
+                    try:
+                        return func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_retries:
+                            # 记录重试信息
+                            retry_context = ErrorContext(
+                                context=f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1})",
+                                module=context.module if context else "",
+                                function=func.__name__
+                            ) if context else None
+            
+                            self.logger.warning(
+                                f"函数 {func.__name__} 执行失败 (尝试 {attempt + 1}/{max_retries + 1}): {e}"
+                            )
+                            
+                            import time
+                            time.sleep(current_delay)
+                            current_delay *= backoff_factor  # 指数退避
+                        else:
+                            # 最后一次尝试失败
+                            final_context = ErrorContext(
+                                context=f"函数 {func.__name__} 执行失败，已达到最大重试次数",
+                                module=context.module if context else "",
+                                function=func.__name__
+                            ) if context else None
+                            
+                            self.logger.error(
+                                f"函数 {func.__name__} 执行失败，已达到最大重试次数: {e}"
+                            )
+                raise last_exception
+            
+            # 根据函数是否为异步函数返回相应的包装器
+            import inspect
+            if inspect.iscoroutinefunction(func):
+                return async_wrapper
+            else:
+                return sync_wrapper
+        
+        return decorator
+    
+    def get_error_history(self) -> List[Dict]:
+        """获取错误历史记录"""
+        return self.error_history.copy()
+    
+    def clear_error_history(self):
+        """清空错误历史记录"""
+        self.error_history.clear()
+
+
+# 全局增强错误处理器实例
+enhanced_error_handler = EnhancedErrorHandler()
+
+
+def handle_exception(context: str = "", module: str = "", function: str = "",
+                     raise_error: bool = True, log_error: bool = True,
+                     error_code: Optional[str] = None):
+    """
+    装饰器：处理函数异常（增强版）
+    
+    Args:
+        context: 错误上下文描述
+        module: 模块名称
+        function: 函数名称
+        raise_error: 是否重新抛出异常
+        log_error: 是否记录错误日志
+        error_code: 错误代码
+    """
+    def decorator(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            try:
+                return await func(*args, **kwargs)
+            except Exception as e:
+                error_context = ErrorContext(
+                    context=f"{context} - {func.__name__}",
+                    module=module,
+                    function=func.__name__
+                )
+                
+                # 如果是详细异常，保留原有信息
+                if isinstance(e, DetailedException):
+                    # 确保上下文信息完整
+                    if not e.context:
+                        e.context = error_context
+                    enhanced_error_handler.handle_error(
+                        e, context=e.context,
+                        raise_error=raise_error, log_error=log_error
+                    )
+                else:
+                    # 包装为详细异常
+                    detailed_e = DetailedException(
+                        str(e), context=error_context, error_code=error_code
+                    )
+                    enhanced_error_handler.handle_error(
+                        detailed_e, context=error_context,
+                        raise_error=raise_error, log_error=log_error
+                    )
+                if not raise_error:
+                    return None
+        
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                error_context = ErrorContext(
+                    context=f"{context} - {func.__name__}",
+                    module=module,
+                    function=func.__name__
+                )
+                
+                # 如果是详细异常，保留原有信息
+                if isinstance(e, DetailedException):
+                    # 确保上下文信息完整
+                    if not e.context:
+                        e.context = error_context
+                    enhanced_error_handler.handle_error(
+                        e, context=e.context,
+                        raise_error=raise_error, log_error=log_error
+                    )
+                else:
+                    # 包装为详细异常
+                    detailed_e = DetailedException(
+                        str(e), context=error_context, error_code=error_code
+                    )
+                    enhanced_error_handler.handle_error(
+                        detailed_e, context=error_context,
+                        raise_error=raise_error, log_error=log_error
+                    )
+                if not raise_error:
+                    return None
+        
+        # 根据函数是否为异步函数返回相应的包装器
+        import inspect
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator

crawlo/utils/env_config.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+环境变量配置工具
+提供统一的环境变量读取和配置管理机制
+"""
+import os
+from typing import Optional, Union, Any
+
+
+class EnvConfigManager:
+    """环境变量配置管理器"""
+    
+    @staticmethod
+    def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
+        """
+        获取环境变量值
+        
+        Args:
+            var_name: 环境变量名称
+            default: 默认值
+            var_type: 变量类型 (str, int, float, bool)
+            
+        Returns:
+            环境变量值或默认值
+        """
+        value = os.getenv(var_name)
+        if value is None:
+            return default
+        
+        try:
+            if var_type == bool:
+                return value.lower() in ('1', 'true', 'yes', 'on')
+            elif var_type == int:
+                return int(value)
+            elif var_type == float:
+                return float(value)
+            else:
+                return value
+        except (ValueError, TypeError):
+            return default
+    
+    @staticmethod
+    def get_redis_config() -> dict:
+        """
+        获取 Redis 配置
+        
+        Returns:
+            Redis 配置字典
+        """
+        return {
+            'REDIS_HOST': EnvConfigManager.get_env_var('REDIS_HOST', '127.0.0.1', str),
+            'REDIS_PORT': EnvConfigManager.get_env_var('REDIS_PORT', 6379, int),
+            'REDIS_PASSWORD': EnvConfigManager.get_env_var('REDIS_PASSWORD', '', str),
+            'REDIS_DB': EnvConfigManager.get_env_var('REDIS_DB', 0, int),
+        }
+    
+    @staticmethod
+    def get_runtime_config() -> dict:
+        """
+        获取运行时配置
+        
+        Returns:
+            运行时配置字典
+        """
+        return {
+            'CRAWLO_MODE': EnvConfigManager.get_env_var('CRAWLO_MODE', 'standalone', str),
+            'PROJECT_NAME': EnvConfigManager.get_env_var('PROJECT_NAME', 'crawlo', str),
+            'CONCURRENCY': EnvConfigManager.get_env_var('CONCURRENCY', 8, int),
+        }
+
+
+# 便捷函数
+def get_env_var(var_name: str, default: Any = None, var_type: type = str) -> Any:
+    """
+    便捷函数：获取环境变量值
+    
+    Args:
+        var_name: 环境变量名称
+        default: 默认值
+        var_type: 变量类型 (str, int, float, bool)
+        
+    Returns:
+        环境变量值或默认值
+    """
+    return EnvConfigManager.get_env_var(var_name, default, var_type)
+
+
+def get_redis_config() -> dict:
+    """
+    便捷函数：获取 Redis 配置
+    
+    Returns:
+        Redis 配置字典
+    """
+    return EnvConfigManager.get_redis_config()
+
+
+def get_runtime_config() -> dict:
+    """
+    便捷函数：获取运行时配置
+    
+    Returns:
+        运行时配置字典
+    """
+    return EnvConfigManager.get_runtime_config()

crawlo/utils/error_handler.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+统一错误处理工具
+提供一致的错误处理和日志记录机制
+"""
+import traceback
+import logging
+from typing import Optional, Callable, Any
+from functools import wraps
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.enhanced_error_handler import EnhancedErrorHandler, ErrorContext
+
+
+class ErrorHandler:
+    """统一错误处理器（简化版，使用增强版作为后端实现）"""
+    
+    def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
+        self.logger = get_logger(logger_name, log_level)
+        # 使用增强版错误处理器作为后端
+        self._enhanced_handler = EnhancedErrorHandler(logger_name, log_level)
+    
+    def handle_error(self, exception: Exception, context: str = "", 
+                     raise_error: bool = True, log_error: bool = True) -> None:
+        """
+        统一处理错误
+        
+        Args:
+            exception: 异常对象
+            context: 错误上下文描述
+            raise_error: 是否重新抛出异常
+            log_error: 是否记录错误日志
+        """
+        # 转换为增强版错误上下文
+        error_context = ErrorContext(context=context) if context else None
+        self._enhanced_handler.handle_error(
+            exception, context=error_context, 
+            raise_error=raise_error, log_error=log_error
+        )
+    
+    def safe_call(self, func: Callable, *args, default_return=None, 
+                  context: str = "", **kwargs) -> Any:
+        """
+        安全调用函数，捕获并处理异常
+        
+        Args:
+            func: 要调用的函数
+            *args: 函数参数
+            default_return: 默认返回值
+            context: 错误上下文描述
+            **kwargs: 函数关键字参数
+            
+        Returns:
+            函数返回值或默认值
+        """
+        error_context = ErrorContext(context=context) if context else None
+        return self._enhanced_handler.safe_call(
+            func, *args, default_return=default_return, 
+            context=error_context, **kwargs
+        )
+    
+    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
+                         exceptions: tuple = (Exception,)):
+        """
+        装饰器：失败时重试
+        
+        Args:
+            max_retries: 最大重试次数
+            delay: 重试间隔（秒）
+            exceptions: 需要重试的异常类型
+        """
+        def decorator(func):
+            # 直接使用增强版处理器的重试装饰器
+            return self._enhanced_handler.retry_on_failure(
+                max_retries=max_retries, delay=delay, exceptions=exceptions
+            )(func)
+        return decorator
+
+
+# 全局错误处理器实例
+default_error_handler = ErrorHandler()
+
+
+def handle_exception(context: str = "", raise_error: bool = True, log_error: bool = True):
+    """
+    装饰器：处理函数异常
+    
+    Args:
+        context: 错误上下文描述
+        raise_error: 是否重新抛出异常
+        log_error: 是否记录错误日志
+    """
+    def decorator(func):
+        @wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            try:
+                return await func(*args, **kwargs)
+            except Exception as e:
+                default_error_handler.handle_error(
+                    e, context=f"{context} - {func.__name__}", 
+                    raise_error=raise_error, log_error=log_error
+                )
+                if not raise_error:
+                    return None
+        
+        @wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                default_error_handler.handle_error(
+                    e, context=f"{context} - {func.__name__}", 
+                    raise_error=raise_error, log_error=log_error
+                )
+                if not raise_error:
+                    return None
+        
+        # 根据函数是否为异步函数返回相应的包装器
+        import inspect
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+    
+    return decorator