crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/utils/class_loader.py

@@ -0,0 +1,26 @@
+# -*- coding: UTF-8 -*-
+"""
+类加载器工具模块
+==============
+提供动态类加载功能，避免循环依赖问题。
+"""
+import importlib
+from typing import Any
+
+
+def load_class(path: str) -> Any:
+    """
+    动态加载类
+    
+    Args:
+        path: 类的完整路径，如 'package.module.ClassName'
+        
+    Returns:
+        加载的类对象
+    """
+    try:
+        module_path, class_name = path.rsplit('.', 1)
+        module = importlib.import_module(module_path)
+        return getattr(module, class_name)
+    except (ValueError, ImportError, AttributeError) as e:
+        raise ImportError(f"无法加载类 '{path}': {e}")

crawlo/utils/error_handler.py

@@ -4,22 +4,30 @@
 统一错误处理工具
 提供一致的错误处理和日志记录机制
 """
+import time
+import traceback
 from functools import wraps
 from typing import Callable, Any
 
-from crawlo.utils.enhanced_error_handler import EnhancedErrorHandler, ErrorContext
 from crawlo.utils.log import get_logger
 
 
 class ErrorHandler:
-    """统一错误处理器（简化版，使用增强版作为后端实现）"""
-    
+    """统一错误处理器（简化版，避免循环依赖）"""
+
     def __init__(self, logger_name: str = __name__, log_level: str = 'ERROR'):
-        self.logger = get_logger(logger_name, log_level)
-        # 使用增强版错误处理器作为后端
-        self._enhanced_handler = EnhancedErrorHandler(logger_name, log_level)
-    
-    def handle_error(self, exception: Exception, context: str = "", 
+        # 延迟初始化logger避免循环依赖
+        self._logger = None
+        self.logger_name = logger_name
+        self.log_level = log_level
+
+    @property
+    def logger(self):
+        if self._logger is None:
+            self._logger = get_logger(self.logger_name)
+        return self._logger
+
+    def handle_error(self, exception: Exception, context: str = "",
                      raise_error: bool = True, log_error: bool = True) -> None:
         """
         统一处理错误
@@ -30,14 +38,16 @@ class ErrorHandler:
             raise_error: 是否重新抛出异常
             log_error: 是否记录错误日志
         """
-        # 转换为增强版错误上下文
-        error_context = ErrorContext(context=context) if context else None
-        self._enhanced_handler.handle_error(
-            exception, context=error_context, 
-            raise_error=raise_error, log_error=log_error
-        )
-    
-    def safe_call(self, func: Callable, *args, default_return=None, 
+        if log_error:
+            error_msg = f"Error in {context}: {str(exception)}" if context else str(exception)
+            self.logger.error(error_msg)
+            # 在DEBUG级别记录详细的堆栈跟踪
+            self.logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
+
+        if raise_error:
+            raise exception
+
+    def safe_call(self, func: Callable, *args, default_return=None,
                   context: str = "", **kwargs) -> Any:
         """
         安全调用函数，捕获并处理异常
@@ -52,13 +62,13 @@ class ErrorHandler:
         Returns:
             函数返回值或默认值
         """
-        error_context = ErrorContext(context=context) if context else None
-        return self._enhanced_handler.safe_call(
-            func, *args, default_return=default_return, 
-            context=error_context, **kwargs
-        )
-    
-    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0, 
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            self.handle_error(e, context=context, raise_error=False)
+            return default_return
+
+    def retry_on_failure(self, max_retries: int = 3, delay: float = 1.0,
                          exceptions: tuple = (Exception,)):
         """
         装饰器：失败时重试
@@ -68,16 +78,46 @@ class ErrorHandler:
             delay: 重试间隔（秒）
             exceptions: 需要重试的异常类型
         """
+
         def decorator(func):
-            # 直接使用增强版处理器的重试装饰器
-            return self._enhanced_handler.retry_on_failure(
-                max_retries=max_retries, delay=delay, exceptions=exceptions
-            )(func)
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                last_exception = None
+                for attempt in range(max_retries + 1):
+                    try:
+                        return func(*args, **kwargs)
+                    except exceptions as e:
+                        last_exception = e
+                        if attempt < max_retries:
+                            self.logger.warning(f"Attempt {attempt + 1} failed, retrying in {delay}s: {e}")
+                            time.sleep(delay)
+                        else:
+                            self.logger.error(f"All {max_retries + 1} attempts failed")
+                            raise e
+                return None
+
+            return wrapper
+
         return decorator
 
 
-# 全局错误处理器实例
-default_error_handler = ErrorHandler()
+# 全局错误处理器实例（延迟初始化）
+_default_error_handler = None
+
+
+def get_default_error_handler():
+    """Get the default error handler with lazy initialization"""
+    global _default_error_handler
+    if _default_error_handler is None:
+        _default_error_handler = ErrorHandler()
+    return _default_error_handler
+
+
+# 为了向后兼容，保留老的接口
+def __getattr__(name):
+    if name == 'default_error_handler':
+        return get_default_error_handler()
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
 
 
 def handle_exception(context: str = "", raise_error: bool = True, log_error: bool = True):
@@ -89,6 +129,7 @@ def handle_exception(context: str = "", raise_error: bool = True, log_error: boo
         raise_error: 是否重新抛出异常
         log_error: 是否记录错误日志
     """
+
     def decorator(func):
         @wraps(func)
         async def async_wrapper(*args, **kwargs):
@@ -96,29 +137,29 @@ def handle_exception(context: str = "", raise_error: bool = True, log_error: boo
                 return await func(*args, **kwargs)
             except Exception as e:
                 default_error_handler.handle_error(
-                    e, context=f"{context} - {func.__name__}", 
+                    e, context=f"{context} - {func.__name__}",
                     raise_error=raise_error, log_error=log_error
                 )
                 if not raise_error:
                     return None
-        
+
         @wraps(func)
         def sync_wrapper(*args, **kwargs):
             try:
                 return func(*args, **kwargs)
             except Exception as e:
                 default_error_handler.handle_error(
-                    e, context=f"{context} - {func.__name__}", 
+                    e, context=f"{context} - {func.__name__}",
                     raise_error=raise_error, log_error=log_error
                 )
                 if not raise_error:
                     return None
-        
+
         # 根据函数是否为异步函数返回相应的包装器
         import inspect
         if inspect.iscoroutinefunction(func):
             return async_wrapper
         else:
             return sync_wrapper
-    
-    return decorator
+
+    return decorator

crawlo/utils/log.py

@@ -1,147 +1,44 @@
-import os
-from logging import (
-    Formatter,
-    StreamHandler,
-    FileHandler,
-    Logger,
-    DEBUG,
-    INFO,
-    getLevelName,
-)
+# ==================== 向后兼容的日志接口 ====================
+# 主要功能已迁移到 crawlo.logging 模块
+# 本文件仅保留最基本的兼容性接口
+
+import logging
+from typing import Optional
+
+# 向后兼容：导入新的日志系统
+try:
+    from crawlo.logging import get_logger as new_get_logger, configure_logging
+    _NEW_LOGGING_AVAILABLE = True
+except ImportError:
+    _NEW_LOGGING_AVAILABLE = False
+    new_get_logger = None
+    configure_logging = None
 
 LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
 
-
-class LoggerManager:
-    """日志管理器，提供统一的日志配置和获取接口"""
-    logger_cache = {}
-    _default_filename = None
-    _default_level = DEBUG  # 设置为最低级别，由handler控制实际输出
-    _default_file_level = DEBUG  # 默认为DEBUG级别，确保所有INFO级别日志都能写入文件
-    _default_console_level = DEBUG  # 默认为DEBUG级别
-    _default_log_format = LOG_FORMAT
-    _default_encoding = 'utf-8'
-    _configured = False  # 标记是否已配置
-
-    @classmethod
-    def _to_level(cls, level):
-        """安全转换为日志级别 int"""
-        if level is None:
-            return INFO
-        if isinstance(level, int):
-            return level
-        if isinstance(level, str):
-            # 使用logging模块内置的级别转换
-            level_value = getLevelName(level.upper())
-            # getLevelName在无效级别时返回字符串，我们需要返回数字
-            if isinstance(level_value, int):
-                return level_value
-            else:
-                return INFO
-        if hasattr(level, 'get'):  # 如 SettingManager 或 dict
-            lv = level.get('LOG_LEVEL')
-            if isinstance(lv, int):
-                return lv
-            if isinstance(lv, str):
-                level_value = getLevelName(lv.upper())
-                if isinstance(level_value, int):
-                    return level_value
-                else:
-                    return INFO
-        return INFO
-
-    @classmethod
-    def configure(cls, settings=None, **kwargs):
-        """
-        使用 settings 对象或关键字参数配置日志
-        """
-        # 优先使用 settings，否则用 kwargs
-        get_val = settings.get if hasattr(settings, 'get') else (lambda k, d=None: kwargs.get(k, d))
-
-        filename = get_val('LOG_FILE')
-        level = get_val('LOG_LEVEL', 'INFO')  # 默认为INFO级别
-        file_level = get_val('LOG_FILE_LEVEL', level)  # 默认继承LOG_LEVEL的值
-        # 根据项目规范，已完全移除LOG_CONSOLE_LEVEL支持，统一使用LOG_LEVEL控制控制台和文件的日志输出级别
-        log_format = get_val('LOG_FORMAT', LOG_FORMAT)
-        encoding = get_val('LOG_ENCODING', 'utf-8')
-
-        cls._default_filename = filename
-        cls._default_level = cls._to_level(level)
-        cls._default_file_level = cls._to_level(file_level)
-        # 控制台日志级别直接使用LOG_LEVEL的值，不再支持LOG_CONSOLE_LEVEL
-        cls._default_console_level = cls._default_level
-        cls._default_log_format = log_format
-        cls._default_encoding = encoding
-
-        cls._configured = True
-
-    @classmethod
-    def get_logger(cls, name='default', level=None, filename=None):
-        """
-        获取logger实例
-        """
-        # 确定最终参数
-        # 如果传入了level参数，则使用它，否则使用默认级别
-        if level is not None:
-            final_level = cls._to_level(level)
-        else:
-            # Logger级别设置为DEBUG（最低级别），由handler控制实际输出
-            final_level = DEBUG
-
-        final_filename = filename if filename is not None else cls._default_filename
-
-        # 安全的字符串化 key，避免任何 unhashable 类型
-        key_parts = [
-            name,
-            str(final_level),
-            final_filename or 'no_file',
-        ]
-        key = '|'.join(key_parts)  # 如 "my_spider|20|logs/app.log"
-
-        if key in cls.logger_cache:
-            # 更新logger级别
-            cls.logger_cache[key].setLevel(final_level)
-            return cls.logger_cache[key]
-
-        # 创建 logger
-        _logger = Logger(name=name)
-        _logger.setLevel(final_level)
-
-        formatter = Formatter(cls._default_log_format)
-
-        # 控制台
-        if cls._default_console_level is not False:
-            ch = StreamHandler()
-            ch.setFormatter(formatter)
-            ch.setLevel(cls._default_console_level)
-            _logger.addHandler(ch)
-
-        # 文件
-        if final_filename:
-            try:
-                log_dir = os.path.dirname(final_filename)
-                if log_dir and not os.path.exists(log_dir):
-                    os.makedirs(log_dir, exist_ok=True)
-
-                # 使用普通文件处理器（移除日志轮转功能）
-                fh = FileHandler(final_filename, mode='a', encoding=cls._default_encoding)
-
-                fh.setFormatter(formatter)
-                fh.setLevel(cls._default_file_level)
-                _logger.addHandler(fh)
-            except (PermissionError, FileNotFoundError) as e:
-                print(f"[Logger] 无法创建日志文件 {final_filename}: {e}")
-            except Exception as e:
-                print(f"[Logger] 创建日志文件时发生未知错误 {final_filename}: {e}")
-
-        cls.logger_cache[key] = _logger
-        return _logger
-
-    @classmethod
-    def is_configured(cls):
-        """检查日志系统是否已配置"""
-        return cls._configured
-
-
-# 全局快捷函数
-get_logger = LoggerManager.get_logger
+# 向后兼容的日志函数
+def get_logger(name: str = 'default', level: Optional[int] = None):
+    """获取Logger实例 - 向后兼容函数"""
+    if _NEW_LOGGING_AVAILABLE and new_get_logger:
+        # 使用新的日志系统
+        return new_get_logger(name)
+    else:
+        # 降级到基本的Python logging
+        logger = logging.getLogger(name)
+        if not logger.handlers:
+            handler = logging.StreamHandler()
+            formatter = logging.Formatter(LOG_FORMAT)
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+            logger.setLevel(level or logging.INFO)
+        return logger
+
+# 兼容性函数
+def get_component_logger(component_class, settings=None, level=None):
+    """获取组件Logger - 向后兼容"""
+    if hasattr(component_class, '__name__'):
+        component_name = component_class.__name__
+    else:
+        component_name = str(component_class)
+    
+    return get_logger(component_name)

crawlo/utils/redis_connection_pool.py

@@ -9,8 +9,9 @@ from typing import Dict, Any, Optional
 
 import redis.asyncio as aioredis
 
-from crawlo.utils.error_handler import ErrorHandler
-from crawlo.utils.log import get_logger
+# 延迟导入避免循环依赖
+# from crawlo.utils.error_handler import ErrorHandler
+# from crawlo.utils.log import get_logger
 
 
 class OptimizedRedisConnectionPool:
@@ -31,8 +32,10 @@ class OptimizedRedisConnectionPool:
     def __init__(self, redis_url: str, **kwargs):
         self.redis_url = redis_url
         self.config = {**self.DEFAULT_CONFIG, **kwargs}
-        self.logger = get_logger(self.__class__.__name__)
-        self.error_handler = ErrorHandler(self.__class__.__name__)
+        
+        # 延迟初始化logger和error_handler
+        self._logger = None
+        self._error_handler = None
         
         # 连接池实例
         self._connection_pool: Optional[aioredis.ConnectionPool] = None
@@ -50,6 +53,22 @@ class OptimizedRedisConnectionPool:
         # 初始化连接池
         self._initialize_pool()
     
+    @property
+    def logger(self):
+        """延迟初始化logger"""
+        if self._logger is None:
+            from crawlo.utils.log import get_logger
+            self._logger = get_logger(self.__class__.__name__)
+        return self._logger
+    
+    @property
+    def error_handler(self):
+        """延迟初始化error_handler"""
+        if self._error_handler is None:
+            from crawlo.utils.error_handler import ErrorHandler
+            self._error_handler = ErrorHandler(self.__class__.__name__)
+        return self._error_handler
+    
     def _initialize_pool(self):
         """初始化连接池"""
         try:
@@ -176,8 +195,26 @@ class RedisBatchOperationHelper:
     def __init__(self, redis_client: aioredis.Redis, batch_size: int = 100):
         self.redis_client = redis_client
         self.batch_size = batch_size
-        self.logger = get_logger(self.__class__.__name__)
-        self.error_handler = ErrorHandler(self.__class__.__name__)
+        
+        # 延迟初始化logger和error_handler
+        self._logger = None
+        self._error_handler = None
+    
+    @property
+    def logger(self):
+        """延迟初始化logger"""
+        if self._logger is None:
+            from crawlo.utils.log import get_logger
+            self._logger = get_logger(self.__class__.__name__)
+        return self._logger
+    
+    @property
+    def error_handler(self):
+        """延迟初始化error_handler"""
+        if self._error_handler is None:
+            from crawlo.utils.error_handler import ErrorHandler
+            self._error_handler = ErrorHandler(self.__class__.__name__)
+        return self._error_handler
     
     async def batch_execute(self, operations: list, batch_size: Optional[int] = None) -> list:
         """

crawlo/utils/request_serializer.py

@@ -15,7 +15,14 @@ class RequestSerializer:
     """Request 序列化工具类"""
     
     def __init__(self):
-        self.logger = get_logger(self.__class__.__name__)
+        # 延迟初始化logger避免循环依赖
+        self._logger = None
+    
+    @property
+    def logger(self):
+        if self._logger is None:
+            self._logger = get_logger(self.__class__.__name__)
+        return self._logger
     
     def prepare_for_serialization(self, request):
         """