crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/utils/resource_manager.py

@@ -0,0 +1,337 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+资源管理器 - 统一管理所有可清理资源
+========================================
+
+功能特性：
+- 统一注册和清理资源
+- 支持异步资源清理
+- 资源泄露检测
+- 清理顺序保证（LIFO）
+"""
+import asyncio
+import time
+import traceback
+from typing import Any, Callable, List, Tuple, Optional, Dict
+from enum import Enum
+
+from crawlo.logging import get_logger
+
+
+class ResourceType(Enum):
+    """资源类型枚举"""
+    DOWNLOADER = "downloader"
+    REDIS_POOL = "redis_pool"
+    QUEUE = "queue"
+    FILTER = "filter"
+    PIPELINE = "pipeline"
+    MIDDLEWARE = "middleware"
+    EXTENSION = "extension"
+    SESSION = "session"
+    BROWSER = "browser"
+    OTHER = "other"
+
+
+class ResourceStatus(Enum):
+    """资源状态"""
+    ACTIVE = "active"
+    CLOSING = "closing"
+    CLOSED = "closed"
+    ERROR = "error"
+
+
+class ManagedResource:
+    """托管资源"""
+    
+    def __init__(self, 
+                 resource: Any,
+                 cleanup_func: Callable,
+                 resource_type: ResourceType = ResourceType.OTHER,
+                 name: Optional[str] = None):
+        self.resource = resource
+        self.cleanup_func = cleanup_func
+        self.resource_type = resource_type
+        self.name = name or f"{resource_type.value}_{id(resource)}"
+        self.status = ResourceStatus.ACTIVE
+        self.created_at = time.time()
+        self.closed_at: Optional[float] = None
+    
+    async def cleanup(self) -> bool:
+        """清理资源"""
+        if self.status == ResourceStatus.CLOSED:
+            return True
+        
+        self.status = ResourceStatus.CLOSING
+        try:
+            # 检查cleanup_func是否为异步函数
+            if asyncio.iscoroutinefunction(self.cleanup_func):
+                await self.cleanup_func(self.resource)
+            else:
+                # 同步函数，直接调用
+                result = self.cleanup_func(self.resource)
+                # 如果返回的是协程，则await
+                if asyncio.iscoroutine(result):
+                    await result
+            
+            self.status = ResourceStatus.CLOSED
+            self.closed_at = time.time()
+            return True
+        except Exception as e:
+            self.status = ResourceStatus.ERROR
+            raise e
+    
+    def get_lifetime(self) -> float:
+        """获取资源生命周期（秒）"""
+        end_time = self.closed_at or time.time()
+        return end_time - self.created_at
+
+
+class ResourceManager:
+    """
+    资源管理器 - 统一管理所有可清理资源
+    
+    特性：
+    1. 自动跟踪注册的资源
+    2. 保证清理顺序（LIFO - 后进先出）
+    3. 容错清理（一个失败不影响其他）
+    4. 资源泄露检测
+    5. 统计和监控
+    """
+    
+    def __init__(self, name: str = "default"):
+        self.name = name
+        self._resources: List[ManagedResource] = []
+        self._lock = asyncio.Lock()
+        self._cleanup_errors: List[Tuple[str, Exception]] = []
+        self._logger = get_logger(f"ResourceManager.{name}")
+        
+        # 统计信息
+        self._stats = {
+            'total_registered': 0,
+            'total_cleaned': 0,
+            'total_errors': 0,
+            'active_resources': 0,
+        }
+    
+    def register(self,
+                 resource: Any,
+                 cleanup_func: Callable,
+                 resource_type: ResourceType = ResourceType.OTHER,
+                 name: Optional[str] = None) -> ManagedResource:
+        """
+        注册需要清理的资源
+        
+        Args:
+            resource: 资源对象
+            cleanup_func: 清理函数（同步或异步）
+            resource_type: 资源类型
+            name: 资源名称（用于日志）
+        
+        Returns:
+            托管资源对象
+        """
+        managed = ManagedResource(resource, cleanup_func, resource_type, name)
+        self._resources.append(managed)
+        self._stats['total_registered'] += 1
+        self._stats['active_resources'] += 1
+        
+        self._logger.debug(f"Resource registered: {managed.name} ({resource_type.value})")
+        return managed
+    
+    async def cleanup_all(self, reverse: bool = True) -> Dict[str, Any]:
+        """
+        清理所有注册的资源
+        
+        Args:
+            reverse: 是否反向清理（LIFO，推荐）
+        
+        Returns:
+            清理结果统计
+        """
+        async with self._lock:
+            if not self._resources:
+                self._logger.debug("No resources to cleanup")
+                return self._get_cleanup_stats()
+            
+            self._logger.info(f"Starting cleanup of {len(self._resources)} resources...")
+            
+            # 反向清理（后创建的先清理）
+            resources = reversed(self._resources) if reverse else self._resources
+            
+            cleanup_start = time.time()
+            success_count = 0
+            error_count = 0
+            
+            for managed in resources:
+                try:
+                    self._logger.debug(f"Cleaning up: {managed.name}")
+                    await managed.cleanup()
+                    success_count += 1
+                    self._stats['total_cleaned'] += 1
+                    self._stats['active_resources'] -= 1
+                except Exception as e:
+                    error_count += 1
+                    self._stats['total_errors'] += 1
+                    self._cleanup_errors.append((managed.name, e))
+                    self._logger.error(
+                        f"Failed to cleanup {managed.name}: {e}",
+                        exc_info=True
+                    )
+                    # 继续清理其他资源，不中断
+            
+            cleanup_duration = time.time() - cleanup_start
+            
+            # 清空资源列表
+            self._resources.clear()
+            
+            result = {
+                'success': success_count,
+                'errors': error_count,
+                'duration': cleanup_duration,
+                'total_resources': success_count + error_count,
+            }
+            
+            if error_count > 0:
+                self._logger.warning(
+                    f"Cleanup completed with errors: {success_count} success, "
+                    f"{error_count} errors in {cleanup_duration:.2f}s"
+                )
+            else:
+                self._logger.info(
+                    f"Cleanup completed successfully: {success_count} resources "
+                    f"in {cleanup_duration:.2f}s"
+                )
+            
+            return result
+    
+    async def cleanup_by_type(self, resource_type: ResourceType) -> int:
+        """
+        按类型清理资源
+        
+        Args:
+            resource_type: 资源类型
+        
+        Returns:
+            清理的资源数量
+        """
+        async with self._lock:
+            to_cleanup = [r for r in self._resources if r.resource_type == resource_type]
+            
+            if not to_cleanup:
+                return 0
+            
+            cleaned = 0
+            for managed in reversed(to_cleanup):
+                try:
+                    await managed.cleanup()
+                    self._resources.remove(managed)
+                    cleaned += 1
+                    self._stats['total_cleaned'] += 1
+                    self._stats['active_resources'] -= 1
+                except Exception as e:
+                    self._logger.error(f"Failed to cleanup {managed.name}: {e}")
+                    self._stats['total_errors'] += 1
+            
+            return cleaned
+    
+    def get_active_resources(self) -> List[ManagedResource]:
+        """获取所有活跃资源"""
+        return [r for r in self._resources if r.status == ResourceStatus.ACTIVE]
+    
+    def get_resources_by_type(self, resource_type: ResourceType) -> List[ManagedResource]:
+        """按类型获取资源"""
+        return [r for r in self._resources if r.resource_type == resource_type]
+    
+    def detect_leaks(self, max_lifetime: float = 3600) -> List[ManagedResource]:
+        """
+        检测可能的资源泄露
+        
+        Args:
+            max_lifetime: 最大生命周期（秒），超过此时间未清理视为泄露
+        
+        Returns:
+            可能泄露的资源列表
+        """
+        current_time = time.time()
+        leaks = []
+        
+        for managed in self._resources:
+            if managed.status == ResourceStatus.ACTIVE:
+                lifetime = current_time - managed.created_at
+                if lifetime > max_lifetime:
+                    leaks.append(managed)
+                    self._logger.warning(
+                        f"Potential leak detected: {managed.name} "
+                        f"(lifetime: {lifetime:.2f}s)"
+                    )
+        
+        return leaks
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """获取统计信息"""
+        return {
+            **self._stats,
+            'cleanup_errors': len(self._cleanup_errors),
+            'active_by_type': self._get_active_by_type(),
+        }
+    
+    def _get_active_by_type(self) -> Dict[str, int]:
+        """按类型统计活跃资源"""
+        result = {}
+        for managed in self._resources:
+            if managed.status == ResourceStatus.ACTIVE:
+                type_name = managed.resource_type.value
+                result[type_name] = result.get(type_name, 0) + 1
+        return result
+    
+    def _get_cleanup_stats(self) -> Dict[str, Any]:
+        """获取清理统计"""
+        return {
+            'success': 0,
+            'errors': 0,
+            'duration': 0.0,
+            'total_resources': 0,
+        }
+    
+    async def __aenter__(self):
+        """上下文管理器入口"""
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器退出，自动清理"""
+        await self.cleanup_all()
+        return False
+
+
+# 全局资源管理器注册表
+_global_managers: Dict[str, ResourceManager] = {}
+
+
+def get_resource_manager(name: str = "default") -> ResourceManager:
+    """
+    获取资源管理器实例（单例）
+    
+    Args:
+        name: 管理器名称
+    
+    Returns:
+        资源管理器实例
+    """
+    if name not in _global_managers:
+        _global_managers[name] = ResourceManager(name)
+    return _global_managers[name]
+
+
+async def cleanup_all_managers():
+    """清理所有资源管理器"""
+    logger = get_logger("ResourceManager")
+    
+    for name, manager in _global_managers.items():
+        try:
+            logger.info(f"Cleaning up resource manager: {name}")
+            await manager.cleanup_all()
+        except Exception as e:
+            logger.error(f"Failed to cleanup manager {name}: {e}")
+    
+    _global_managers.clear()

crawlo/utils/response_helper.py

@@ -0,0 +1,113 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+响应处理工具模块
+==================
+提供用于处理HTTP响应的辅助函数，包括Cookie处理和正则表达式操作。
+
+该模块包含以下主要函数：
+- parse_cookies: 从响应头中解析Cookies
+- regex_search: 在文本上执行正则表达式搜索
+- regex_findall: 在文本上执行正则表达式查找
+- get_header_value: 从响应头中获取值，处理大小写不敏感的情况
+"""
+
+import re
+from typing import Dict, List, Any, Optional, Union
+from http.cookies import SimpleCookie
+
+
+def parse_cookies(cookie_header: str) -> Dict[str, str]:
+    """
+    从响应头中解析并返回Cookies
+    
+    :param cookie_header: Set-Cookie头部的值
+    :return: 解析后的Cookies字典
+    """
+    if isinstance(cookie_header, list):
+        cookie_header = ", ".join(cookie_header)
+    
+    if not cookie_header:
+        return {}
+        
+    cookies = SimpleCookie()
+    try:
+        cookies.load(cookie_header)
+        return {key: morsel.value for key, morsel in cookies.items()}
+    except Exception:
+        # 如果解析失败，返回空字典
+        return {}
+
+
+def regex_search(pattern: str, text: str, flags: int = re.DOTALL) -> Optional[re.Match]:
+    """
+    在文本上执行正则表达式搜索
+    
+    :param pattern: 正则表达式模式
+    :param text: 要搜索的文本
+    :param flags: 正则表达式标志
+    :return: 匹配对象或None
+    """
+    if not isinstance(pattern, str):
+        raise TypeError("Pattern must be a string")
+    if not isinstance(text, str):
+        raise TypeError("Text must be a string")
+    return re.search(pattern, text, flags=flags)
+
+
+def regex_findall(pattern: str, text: str, flags: int = re.DOTALL) -> List[Any]:
+    """
+    在文本上执行正则表达式查找
+    
+    :param pattern: 正则表达式模式
+    :param text: 要搜索的文本
+    :param flags: 正则表达式标志
+    :return: 匹配结果列表
+    """
+    if not isinstance(pattern, str):
+        raise TypeError("Pattern must be a string")
+    if not isinstance(text, str):
+        raise TypeError("Text must be a string")
+    return re.findall(pattern, text, flags=flags)
+
+
+def get_header_value(headers: Dict[str, Any], header_name: str, default: Any = None) -> Any:
+    """
+    从响应头中获取值，处理大小写不敏感的情况
+    
+    :param headers: 响应头字典
+    :param header_name: 头部名称
+    :param default: 默认值
+    :return: 头部值或默认值
+    """
+    if not headers or not header_name:
+        return default
+        
+    # 首先尝试直接匹配
+    if header_name in headers:
+        return headers[header_name]
+        
+    # 尝试小写匹配
+    lower_header = header_name.lower()
+    if lower_header in headers:
+        return headers[lower_header]
+        
+    # 尝试首字母大写匹配
+    capitalized_header = header_name.capitalize()
+    if capitalized_header in headers:
+        return headers[capitalized_header]
+        
+    # 尝试标题格式匹配
+    title_header = header_name.title()
+    if title_header in headers:
+        return headers[title_header]
+        
+    return default
+
+
+__all__ = [
+    "parse_cookies",
+    "regex_search",
+    "regex_findall",
+    "get_header_value"
+]

crawlo/utils/selector_helper.py

@@ -15,8 +15,9 @@
 所有方法都采用了简洁直观的命名风格，便于记忆和使用。
 """
 
-from typing import List, Any, Optional
-from parsel import Selector, SelectorList
+from typing import List, Any
+
+from parsel import SelectorList
 
 
 def extract_text(elements: SelectorList, join_str: str = " ") -> str:

crawlo/utils/singleton.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+单例模式工具模块
+================
+
+提供同步和异步两种单例实现方式，适用于不同的使用场景。
+
+使用场景：
+1. 同步单例：用于框架初始化、配置管理等同步代码
+2. 异步单例：用于数据库连接池、网络资源等异步代码
+
+示例：
+    # 同步单例
+    @singleton
+    class CoreInitializer:
+        pass
+    
+    # 异步单例（在连接池管理器中使用）
+    class MySQLConnectionPoolManager:
+        _instances: Dict[str, 'MySQLConnectionPoolManager'] = {}
+        _lock = asyncio.Lock()
+        
+        @classmethod
+        async def get_pool(cls, ...):
+            async with cls._lock:
+                if pool_key not in cls._instances:
+                    cls._instances[pool_key] = cls(pool_key)
+            return cls._instances[pool_key].pool
+"""
+
+import threading
+from typing import Any, Dict, Type
+
+
+class SingletonMeta(type):
+    """单例元类"""
+    _instances: Dict[Type, Any] = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            with cls._lock:
+                if cls not in cls._instances:
+                    instance = super().__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+def singleton(cls):
+    """
+    单例装饰器
+    
+    Args:
+        cls: 要装饰的类
+        
+    Returns:
+        装饰后的类，确保只有一个实例
+    """
+    instances = {}
+    lock = threading.Lock()
+
+    def get_instance(*args, **kwargs):
+        if cls not in instances:
+            with lock:
+                if cls not in instances:
+                    instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return get_instance

crawlo/utils/spider_loader.py

@@ -9,7 +9,7 @@ from crawlo.interfaces import ISpiderLoader
 from crawlo.settings.setting_manager import SettingManager
 from crawlo.spider import Spider
 from crawlo.network.request import Request
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 logger = get_logger(__name__)
 

crawlo/utils/text_helper.py

@@ -3,7 +3,7 @@ import json
 import re
 from typing import Any, Union, List, Dict, Tuple, Optional
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 logger = get_logger(__name__)