crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/items/items.py

@@ -1,105 +1,105 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Item 类定义
-"""
-from copy import deepcopy
-from pprint import pformat
-from typing import Any, Iterator, Dict
-from collections.abc import MutableMapping
-
-from .base import ItemMeta
-from crawlo.exceptions import ItemInitError, ItemAttributeError
-
-
-class Item(MutableMapping, metaclass=ItemMeta):
-    """
-    数据项基类，用于定义结构化数据
-    """
-    FIELDS: Dict[str, Any] = {}
-
-    def __init__(self, *args, **kwargs):
-        if args:
-            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
-
-        self._values: Dict[str, Any] = {}
-
-        # 初始化字段，默认值填充
-        for field_name, field_obj in self.FIELDS.items():
-            if field_obj.default is not None:
-                self._values[field_name] = field_obj.default
-
-        # 覆盖默认值或设置新值
-        for key, value in kwargs.items():
-            self[key] = value
-
-    def __getitem__(self, item: str) -> Any:
-        return self._values[item]
-
-    def __setitem__(self, key: str, value: Any) -> None:
-        if key not in self.FIELDS:
-            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
-
-        field = self.FIELDS[key]
-        try:
-            validated_value = field.validate(value, field_name=key)
-            self._values[key] = validated_value
-        except Exception as e:
-            error_lines = [
-                "",
-                "【字段校验失败】",
-                f"字段名称: {key}",
-                f"数据类型: {type(value)}",
-                f"原始值:   {repr(value)}",
-                f"是否允许空值: {field.nullable}",
-                f"错误原因: {str(e)}",
-                ""
-            ]
-            detailed_error = "\n".join(error_lines)
-            raise type(e)(detailed_error) from e
-
-    def __delitem__(self, key: str) -> None:
-        del self._values[key]
-
-    def __setattr__(self, key: str, value: Any) -> None:
-        if not key.startswith("_"):
-            raise AttributeError(
-                f"设置字段值请使用 item[{key!r}] = {value!r}"
-            )
-        super().__setattr__(key, value)
-
-    def __getattr__(self, item: str) -> Any:
-        raise AttributeError(
-            f"{self.__class__.__name__} 不支持字段：{item}。"
-            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
-        )
-
-    def __getattribute__(self, item: str) -> Any:
-        try:
-            field = super().__getattribute__("FIELDS")
-            if isinstance(field, dict) and item in field:
-                raise ItemAttributeError(
-                    f"获取字段值请使用 item[{item!r}]"
-                )
-        except AttributeError:
-            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
-        return super().__getattribute__(item)
-
-    def __repr__(self) -> str:
-        return pformat(dict(self))
-
-    __str__ = __repr__
-
-    def __iter__(self) -> Iterator[str]:
-        return iter(self._values)
-
-    def __len__(self) -> int:
-        return len(self._values)
-
-    def to_dict(self) -> Dict[str, Any]:
-        """转换为字典"""
-        return dict(self)
-
-    def copy(self) -> "Item":
-        """深拷贝当前 Item"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Item 类定义
+"""
+from copy import deepcopy
+from pprint import pformat
+from typing import Any, Iterator, Dict
+from collections.abc import MutableMapping
+
+from .base import ItemMeta
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+
+class Item(MutableMapping, metaclass=ItemMeta):
+    """
+    数据项基类，用于定义结构化数据
+    """
+    FIELDS: Dict[str, Any] = {}
+
+    def __init__(self, *args, **kwargs):
+        if args:
+            raise ItemInitError(f"{self.__class__.__name__} 不支持位置参数：{args}，请使用关键字参数初始化。")
+
+        self._values: Dict[str, Any] = {}
+
+        # 初始化字段，默认值填充
+        for field_name, field_obj in self.FIELDS.items():
+            if field_obj.default is not None:
+                self._values[field_name] = field_obj.default
+
+        # 覆盖默认值或设置新值
+        for key, value in kwargs.items():
+            self[key] = value
+
+    def __getitem__(self, item: str) -> Any:
+        return self._values[item]
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        if key not in self.FIELDS:
+            raise KeyError(f"{self.__class__.__name__} 不包含字段：{key}")
+
+        field = self.FIELDS[key]
+        try:
+            validated_value = field.validate(value, field_name=key)
+            self._values[key] = validated_value
+        except Exception as e:
+            error_lines = [
+                "",
+                "【字段校验失败】",
+                f"字段名称: {key}",
+                f"数据类型: {type(value)}",
+                f"原始值:   {repr(value)}",
+                f"是否允许空值: {field.nullable}",
+                f"错误原因: {str(e)}",
+                ""
+            ]
+            detailed_error = "\n".join(error_lines)
+            raise type(e)(detailed_error) from e
+
+    def __delitem__(self, key: str) -> None:
+        del self._values[key]
+
+    def __setattr__(self, key: str, value: Any) -> None:
+        if not key.startswith("_"):
+            raise AttributeError(
+                f"设置字段值请使用 item[{key!r}] = {value!r}"
+            )
+        super().__setattr__(key, value)
+
+    def __getattr__(self, item: str) -> Any:
+        raise AttributeError(
+            f"{self.__class__.__name__} 不支持字段：{item}。"
+            f"请先在 `{self.__class__.__name__}` 中声明该字段，再通过 item[{item!r}] 获取。"
+        )
+
+    def __getattribute__(self, item: str) -> Any:
+        try:
+            field = super().__getattribute__("FIELDS")
+            if isinstance(field, dict) and item in field:
+                raise ItemAttributeError(
+                    f"获取字段值请使用 item[{item!r}]"
+                )
+        except AttributeError:
+            pass  # 如果 FIELDS 尚未定义，继续执行后续逻辑
+        return super().__getattribute__(item)
+
+    def __repr__(self) -> str:
+        return pformat(dict(self))
+
+    __str__ = __repr__
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._values)
+
+    def __len__(self) -> int:
+        return len(self._values)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return dict(self)
+
+    def copy(self) -> "Item":
+        """深拷贝当前 Item"""
         return deepcopy(self)

crawlo/logging/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo统一日志系统
+=================
+
+设计原则：
+1. 简单优先 - 避免过度设计
+2. 性能优先 - 减少锁竞争和复杂逻辑  
+3. 一致性 - 统一的日志接口
+4. 可靠性 - 确保日志始终可用
+"""
+
+from .manager import LogManager
+from .factory import LoggerFactory
+from .config import LogConfig
+
+# 统一的公共接口
+def get_logger(name: str = 'default'):
+    """获取logger实例"""
+    return LoggerFactory.get_logger(name)
+
+def configure_logging(settings=None, **kwargs):
+    """配置日志系统"""
+    return LogManager().configure(settings, **kwargs)
+
+def is_configured() -> bool:
+    """检查日志系统是否已配置"""
+    return LogManager().is_configured
+
+__all__ = [
+    'LogManager',
+    'LoggerFactory', 
+    'LogConfig',
+    'get_logger',
+    'configure_logging',
+    'is_configured'
+]

crawlo/logging/config.py

@@ -0,0 +1,97 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志配置管理
+"""
+
+import os
+from dataclasses import dataclass, field
+from typing import Optional, Dict, Any
+
+
+@dataclass
+class LogConfig:
+    """日志配置数据类 - 简单明确的配置结构"""
+    
+    # 基本配置
+    level: str = "INFO"
+    format: str = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+    encoding: str = "utf-8"
+    
+    # 文件配置
+    file_path: Optional[str] = None
+    max_bytes: int = 10 * 1024 * 1024  # 10MB
+    backup_count: int = 5
+    
+    # 控制台配置
+    console_enabled: bool = True
+    file_enabled: bool = True
+    
+    # 模块级别配置
+    module_levels: Dict[str, str] = field(default_factory=dict)
+    
+    @classmethod
+    def from_settings(cls, settings) -> 'LogConfig':
+        """从settings对象创建配置"""
+        if not settings:
+            return cls()
+            
+        # 使用settings的get方法而不是getattr
+        if hasattr(settings, 'get'):
+            get_val = settings.get
+        else:
+            get_val = lambda k, d=None: getattr(settings, k, d)
+        
+        # 获取默认值
+        format_default_value = "%(asctime)s - [%(name)s] - %(levelname)s: %(message)s"
+        
+        return cls(
+            level=get_val('LOG_LEVEL', 'INFO'),
+            format=get_val('LOG_FORMAT', format_default_value),
+            encoding=get_val('LOG_ENCODING', 'utf-8'),
+            file_path=get_val('LOG_FILE'),
+            max_bytes=get_val('LOG_MAX_BYTES', 10 * 1024 * 1024),
+            backup_count=get_val('LOG_BACKUP_COUNT', 5),
+            console_enabled=get_val('LOG_CONSOLE_ENABLED', True),
+            file_enabled=get_val('LOG_FILE_ENABLED', True),
+            module_levels=get_val('LOG_LEVELS', {})
+        )
+    
+    @classmethod
+    def from_dict(cls, config_dict: Dict[str, Any]) -> 'LogConfig':
+        """从字典创建配置"""
+        return cls(**{k: v for k, v in config_dict.items() if k in cls.__annotations__})
+    
+    def get_module_level(self, module_name: str) -> str:
+        """获取模块的日志级别"""
+        # 先查找精确匹配
+        if module_name in self.module_levels:
+            return self.module_levels[module_name]
+        
+        # 查找父模块匹配
+        parts = module_name.split('.')
+        for i in range(len(parts) - 1, 0, -1):
+            parent_module = '.'.join(parts[:i])
+            if parent_module in self.module_levels:
+                return self.module_levels[parent_module]
+        
+        # 返回默认级别
+        return self.level
+    
+    def validate(self) -> bool:
+        """验证配置有效性"""
+        valid_levels = {'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'}
+        
+        if self.level.upper() not in valid_levels:
+            return False
+        
+        # 确保日志目录存在
+        if self.file_path and self.file_enabled:
+            try:
+                log_dir = os.path.dirname(self.file_path)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+            except (OSError, PermissionError):
+                return False
+        
+        return True

crawlo/logging/factory.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志器工厂 - 创建和缓存Logger实例
+"""
+
+import logging
+import os
+import threading
+from logging.handlers import RotatingFileHandler
+from typing import Dict, Optional
+from weakref import WeakValueDictionary
+
+from .manager import get_config, is_configured, configure
+from .config import LogConfig
+
+
+class LoggerFactory:
+    """
+    Logger工厂类 - 负责创建和缓存Logger实例
+    
+    特点：
+    1. 使用WeakValueDictionary避免内存泄漏
+    2. 线程安全的Logger创建
+    3. 自动配置管理
+    4. 简单的缓存策略
+    """
+    
+    # Logger缓存 - 使用弱引用避免内存泄漏
+    _logger_cache: WeakValueDictionary = WeakValueDictionary()
+    _cache_lock = threading.RLock()
+    
+    @classmethod
+    def get_logger(cls, name: str = 'crawlo') -> logging.Logger:
+        """
+        获取Logger实例
+        
+        Args:
+            name: Logger名称
+            
+        Returns:
+            logging.Logger: 配置好的Logger实例
+        """
+        # 确保日志系统已配置
+        if not is_configured():
+            configure()  # 使用默认配置
+        
+        # 检查缓存
+        with cls._cache_lock:
+            if name in cls._logger_cache:
+                return cls._logger_cache[name]
+            
+            # 创建新的Logger
+            logger = cls._create_logger(name)
+            cls._logger_cache[name] = logger
+            return logger
+    
+    @classmethod
+    def _create_logger(cls, name: str) -> logging.Logger:
+        """创建新的Logger实例"""
+        config = get_config()
+        if not config:
+            raise RuntimeError("Log system not configured")
+        
+        # 创建Logger
+        logger = logging.getLogger(name)
+        logger.setLevel(logging.DEBUG)  # Logger本身设为最低级别
+        
+        # 清除现有handlers（避免重复添加）
+        logger.handlers.clear()
+        
+        # 获取模块级别
+        module_level = config.get_module_level(name)
+        level = getattr(logging, module_level.upper(), logging.INFO)
+        
+        # 创建formatter
+        formatter = logging.Formatter(config.format)
+        
+        # 添加控制台Handler
+        if config.console_enabled:
+            console_handler = logging.StreamHandler()
+            console_handler.setFormatter(formatter)
+            console_handler.setLevel(level)
+            logger.addHandler(console_handler)
+        
+        # 添加文件Handler
+        if config.file_enabled and config.file_path:
+            try:
+                # 确保日志目录存在
+                log_dir = os.path.dirname(config.file_path)
+                if log_dir and not os.path.exists(log_dir):
+                    os.makedirs(log_dir, exist_ok=True)
+                
+                file_handler = RotatingFileHandler(
+                    filename=config.file_path,
+                    maxBytes=config.max_bytes,
+                    backupCount=config.backup_count,
+                    encoding=config.encoding
+                )
+                file_handler.setFormatter(formatter)
+                file_handler.setLevel(level)
+                logger.addHandler(file_handler)
+            except Exception as e:
+                # 文件Handler创建失败时，至少保证控制台输出
+                pass
+        
+        # 防止向上传播（避免重复输出）
+        logger.propagate = False
+        
+        return logger
+    
+    @classmethod
+    def clear_cache(cls):
+        """清空Logger缓存"""
+        with cls._cache_lock:
+            cls._logger_cache.clear()
+    
+    @classmethod
+    def refresh_loggers(cls, new_config: LogConfig):
+        """刷新所有缓存的Logger（配置更新时使用）"""
+        with cls._cache_lock:
+            # 清空缓存，强制重新创建
+            cls._logger_cache.clear()
+
+
+# 便捷函数
+def get_logger(name: str = 'crawlo') -> logging.Logger:
+    """获取Logger实例的便捷函数"""
+    return LoggerFactory.get_logger(name)

crawlo/logging/manager.py

@@ -0,0 +1,112 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+日志管理器 - 核心组件
+"""
+
+import threading
+from typing import Optional, Any
+from .config import LogConfig
+
+
+class LogManager:
+    """
+    日志管理器 - 单例模式
+    
+    职责：
+    1. 全局日志配置管理
+    2. 配置状态跟踪
+    3. 线程安全的配置更新
+    """
+    
+    _instance: Optional['LogManager'] = None
+    _lock = threading.Lock()
+    
+    def __new__(cls) -> 'LogManager':
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super(LogManager, cls).__new__(cls)
+                    cls._instance._initialized = False
+        return cls._instance
+    
+    def __init__(self):
+        if hasattr(self, '_initialized') and self._initialized:
+            return
+            
+        self._config: Optional[LogConfig] = None
+        self._configured = False
+        self._config_lock = threading.RLock()
+        self._initialized = True
+    
+    @property
+    def config(self) -> Optional[LogConfig]:
+        """获取当前配置"""
+        with self._config_lock:
+            return self._config
+    
+    @property
+    def is_configured(self) -> bool:
+        """检查是否已配置"""
+        return self._configured
+    
+    def configure(self, settings=None, **kwargs) -> LogConfig:
+        """
+        配置日志系统
+        
+        Args:
+            settings: 配置对象或None
+            **kwargs: 关键字参数配置
+            
+        Returns:
+            LogConfig: 生效的配置对象
+        """
+        with self._config_lock:
+            # 总是重新配置，即使已经配置过
+            # 从不同来源创建配置
+            if settings is not None:
+                # 检查settings是否已经是LogConfig对象
+                if isinstance(settings, LogConfig):
+                    config = settings
+                else:
+                    config = LogConfig.from_settings(settings)
+            elif kwargs:
+                config = LogConfig.from_dict(kwargs)
+            else:
+                config = LogConfig()  # 使用默认配置
+            
+            # 验证配置
+            if not config.validate():
+                raise ValueError("Invalid log configuration")
+            
+            self._config = config
+            self._configured = True
+            
+            return config
+    
+    def reset(self):
+        """重置配置（主要用于测试）"""
+        with self._config_lock:
+            self._config = None
+            self._configured = False
+
+
+# 全局实例
+_log_manager = LogManager()
+
+# 模块级便捷函数
+def configure(settings=None, **kwargs) -> LogConfig:
+    """配置日志系统"""
+    return _log_manager.configure(settings, **kwargs)
+
+def is_configured() -> bool:
+    """检查是否已配置"""
+    return _log_manager.is_configured
+
+def get_config() -> Optional[LogConfig]:
+    """获取当前配置"""
+    return _log_manager.config
+
+def reset():
+    """重置配置"""
+    _log_manager.reset()

crawlo/middleware/__init__.py

@@ -1,21 +1,21 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import Request, Response
-
-
-class BaseMiddleware(object):
-    def process_request(self, request, spider) -> None | Request | Response:
-        # 请求预处理
-        pass
-
-    def process_response(self, request, response, spider) -> Request | Response:
-        # 响应预处理
-        pass
-
-    def process_exception(self, request, exp, spider) -> None | Request | Response:
-        # 异常预处理
-        pass
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from crawlo import Request, Response
+
+
+class BaseMiddleware(object):
+    def process_request(self, request, spider) -> None | Request | Response:
+        # 请求预处理
+        pass
+
+    def process_response(self, request, response, spider) -> Request | Response:
+        # 响应预处理
+        pass
+
+    def process_exception(self, request, exp, spider) -> None | Request | Response:
+        # 异常预处理
+        pass
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls()