crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/utils/resource_manager.py

@@ -0,0 +1,337 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+资源管理器 - 统一管理所有可清理资源
+========================================
+
+功能特性：
+- 统一注册和清理资源
+- 支持异步资源清理
+- 资源泄露检测
+- 清理顺序保证（LIFO）
+"""
+import asyncio
+import time
+import traceback
+from typing import Any, Callable, List, Tuple, Optional, Dict
+from enum import Enum
+
+from crawlo.logging import get_logger
+
+
+class ResourceType(Enum):
+    """资源类型枚举"""
+    DOWNLOADER = "downloader"
+    REDIS_POOL = "redis_pool"
+    QUEUE = "queue"
+    FILTER = "filter"
+    PIPELINE = "pipeline"
+    MIDDLEWARE = "middleware"
+    EXTENSION = "extension"
+    SESSION = "session"
+    BROWSER = "browser"
+    OTHER = "other"
+
+
+class ResourceStatus(Enum):
+    """资源状态"""
+    ACTIVE = "active"
+    CLOSING = "closing"
+    CLOSED = "closed"
+    ERROR = "error"
+
+
+class ManagedResource:
+    """托管资源"""
+    
+    def __init__(self, 
+                 resource: Any,
+                 cleanup_func: Callable,
+                 resource_type: ResourceType = ResourceType.OTHER,
+                 name: Optional[str] = None):
+        self.resource = resource
+        self.cleanup_func = cleanup_func
+        self.resource_type = resource_type
+        self.name = name or f"{resource_type.value}_{id(resource)}"
+        self.status = ResourceStatus.ACTIVE
+        self.created_at = time.time()
+        self.closed_at: Optional[float] = None
+    
+    async def cleanup(self) -> bool:
+        """清理资源"""
+        if self.status == ResourceStatus.CLOSED:
+            return True
+        
+        self.status = ResourceStatus.CLOSING
+        try:
+            # 检查cleanup_func是否为异步函数
+            if asyncio.iscoroutinefunction(self.cleanup_func):
+                await self.cleanup_func(self.resource)
+            else:
+                # 同步函数，直接调用
+                result = self.cleanup_func(self.resource)
+                # 如果返回的是协程，则await
+                if asyncio.iscoroutine(result):
+                    await result
+            
+            self.status = ResourceStatus.CLOSED
+            self.closed_at = time.time()
+            return True
+        except Exception as e:
+            self.status = ResourceStatus.ERROR
+            raise e
+    
+    def get_lifetime(self) -> float:
+        """获取资源生命周期（秒）"""
+        end_time = self.closed_at or time.time()
+        return end_time - self.created_at
+
+
+class ResourceManager:
+    """
+    资源管理器 - 统一管理所有可清理资源
+    
+    特性：
+    1. 自动跟踪注册的资源
+    2. 保证清理顺序（LIFO - 后进先出）
+    3. 容错清理（一个失败不影响其他）
+    4. 资源泄露检测
+    5. 统计和监控
+    """
+    
+    def __init__(self, name: str = "default"):
+        self.name = name
+        self._resources: List[ManagedResource] = []
+        self._lock = asyncio.Lock()
+        self._cleanup_errors: List[Tuple[str, Exception]] = []
+        self._logger = get_logger(f"ResourceManager.{name}")
+        
+        # 统计信息
+        self._stats = {
+            'total_registered': 0,
+            'total_cleaned': 0,
+            'total_errors': 0,
+            'active_resources': 0,
+        }
+    
+    def register(self,
+                 resource: Any,
+                 cleanup_func: Callable,
+                 resource_type: ResourceType = ResourceType.OTHER,
+                 name: Optional[str] = None) -> ManagedResource:
+        """
+        注册需要清理的资源
+        
+        Args:
+            resource: 资源对象
+            cleanup_func: 清理函数（同步或异步）
+            resource_type: 资源类型
+            name: 资源名称（用于日志）
+        
+        Returns:
+            托管资源对象
+        """
+        managed = ManagedResource(resource, cleanup_func, resource_type, name)
+        self._resources.append(managed)
+        self._stats['total_registered'] += 1
+        self._stats['active_resources'] += 1
+        
+        self._logger.debug(f"Resource registered: {managed.name} ({resource_type.value})")
+        return managed
+    
+    async def cleanup_all(self, reverse: bool = True) -> Dict[str, Any]:
+        """
+        清理所有注册的资源
+        
+        Args:
+            reverse: 是否反向清理（LIFO，推荐）
+        
+        Returns:
+            清理结果统计
+        """
+        async with self._lock:
+            if not self._resources:
+                self._logger.debug("No resources to cleanup")
+                return self._get_cleanup_stats()
+            
+            self._logger.info(f"Starting cleanup of {len(self._resources)} resources...")
+            
+            # 反向清理（后创建的先清理）
+            resources = reversed(self._resources) if reverse else self._resources
+            
+            cleanup_start = time.time()
+            success_count = 0
+            error_count = 0
+            
+            for managed in resources:
+                try:
+                    self._logger.debug(f"Cleaning up: {managed.name}")
+                    await managed.cleanup()
+                    success_count += 1
+                    self._stats['total_cleaned'] += 1
+                    self._stats['active_resources'] -= 1
+                except Exception as e:
+                    error_count += 1
+                    self._stats['total_errors'] += 1
+                    self._cleanup_errors.append((managed.name, e))
+                    self._logger.error(
+                        f"Failed to cleanup {managed.name}: {e}",
+                        exc_info=True
+                    )
+                    # 继续清理其他资源，不中断
+            
+            cleanup_duration = time.time() - cleanup_start
+            
+            # 清空资源列表
+            self._resources.clear()
+            
+            result = {
+                'success': success_count,
+                'errors': error_count,
+                'duration': cleanup_duration,
+                'total_resources': success_count + error_count,
+            }
+            
+            if error_count > 0:
+                self._logger.warning(
+                    f"Cleanup completed with errors: {success_count} success, "
+                    f"{error_count} errors in {cleanup_duration:.2f}s"
+                )
+            else:
+                self._logger.info(
+                    f"Cleanup completed successfully: {success_count} resources "
+                    f"in {cleanup_duration:.2f}s"
+                )
+            
+            return result
+    
+    async def cleanup_by_type(self, resource_type: ResourceType) -> int:
+        """
+        按类型清理资源
+        
+        Args:
+            resource_type: 资源类型
+        
+        Returns:
+            清理的资源数量
+        """
+        async with self._lock:
+            to_cleanup = [r for r in self._resources if r.resource_type == resource_type]
+            
+            if not to_cleanup:
+                return 0
+            
+            cleaned = 0
+            for managed in reversed(to_cleanup):
+                try:
+                    await managed.cleanup()
+                    self._resources.remove(managed)
+                    cleaned += 1
+                    self._stats['total_cleaned'] += 1
+                    self._stats['active_resources'] -= 1
+                except Exception as e:
+                    self._logger.error(f"Failed to cleanup {managed.name}: {e}")
+                    self._stats['total_errors'] += 1
+            
+            return cleaned
+    
+    def get_active_resources(self) -> List[ManagedResource]:
+        """获取所有活跃资源"""
+        return [r for r in self._resources if r.status == ResourceStatus.ACTIVE]
+    
+    def get_resources_by_type(self, resource_type: ResourceType) -> List[ManagedResource]:
+        """按类型获取资源"""
+        return [r for r in self._resources if r.resource_type == resource_type]
+    
+    def detect_leaks(self, max_lifetime: float = 3600) -> List[ManagedResource]:
+        """
+        检测可能的资源泄露
+        
+        Args:
+            max_lifetime: 最大生命周期（秒），超过此时间未清理视为泄露
+        
+        Returns:
+            可能泄露的资源列表
+        """
+        current_time = time.time()
+        leaks = []
+        
+        for managed in self._resources:
+            if managed.status == ResourceStatus.ACTIVE:
+                lifetime = current_time - managed.created_at
+                if lifetime > max_lifetime:
+                    leaks.append(managed)
+                    self._logger.warning(
+                        f"Potential leak detected: {managed.name} "
+                        f"(lifetime: {lifetime:.2f}s)"
+                    )
+        
+        return leaks
+    
+    def get_stats(self) -> Dict[str, Any]:
+        """获取统计信息"""
+        return {
+            **self._stats,
+            'cleanup_errors': len(self._cleanup_errors),
+            'active_by_type': self._get_active_by_type(),
+        }
+    
+    def _get_active_by_type(self) -> Dict[str, int]:
+        """按类型统计活跃资源"""
+        result = {}
+        for managed in self._resources:
+            if managed.status == ResourceStatus.ACTIVE:
+                type_name = managed.resource_type.value
+                result[type_name] = result.get(type_name, 0) + 1
+        return result
+    
+    def _get_cleanup_stats(self) -> Dict[str, Any]:
+        """获取清理统计"""
+        return {
+            'success': 0,
+            'errors': 0,
+            'duration': 0.0,
+            'total_resources': 0,
+        }
+    
+    async def __aenter__(self):
+        """上下文管理器入口"""
+        return self
+    
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器退出，自动清理"""
+        await self.cleanup_all()
+        return False
+
+
+# 全局资源管理器注册表
+_global_managers: Dict[str, ResourceManager] = {}
+
+
+def get_resource_manager(name: str = "default") -> ResourceManager:
+    """
+    获取资源管理器实例（单例）
+    
+    Args:
+        name: 管理器名称
+    
+    Returns:
+        资源管理器实例
+    """
+    if name not in _global_managers:
+        _global_managers[name] = ResourceManager(name)
+    return _global_managers[name]
+
+
+async def cleanup_all_managers():
+    """清理所有资源管理器"""
+    logger = get_logger("ResourceManager")
+    
+    for name, manager in _global_managers.items():
+        try:
+            logger.info(f"Cleaning up resource manager: {name}")
+            await manager.cleanup_all()
+        except Exception as e:
+            logger.error(f"Failed to cleanup manager {name}: {e}")
+    
+    _global_managers.clear()

crawlo/utils/selector_helper.py

@@ -1,138 +1,138 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-选择器辅助工具模块
-==================
-提供用于处理parsel选择器的辅助函数，用于提取文本和属性等操作。
-
-该模块包含以下主要函数：
-- extract_text: 从元素列表中提取文本并拼接
-- extract_texts: 从元素列表中提取多个文本列表
-- extract_attr: 从元素列表中提取单个元素的属性值
-- extract_attrs: 从元素列表中提取多个元素的属性值列表
-- is_xpath: 判断查询语句是否为XPath
-
-所有方法都采用了简洁直观的命名风格，便于记忆和使用。
-"""
-
-from typing import List, Any, Optional
-from parsel import Selector, SelectorList
-
-
-def extract_text(elements: SelectorList, join_str: str = " ") -> str:
-    """
-    从元素列表中提取文本并拼接
-    
-    :param elements: SelectorList元素列表
-    :param join_str: 文本拼接分隔符
-    :return: 拼接后的文本
-    
-    示例:
-        title_elements = selector.css('title')
-        title_text = extract_text(title_elements)
-    """
-    texts = []
-    for element in elements:
-        # 获取元素的所有文本节点
-        if hasattr(element, 'xpath'):
-            element_texts = element.xpath('.//text()').getall()
-        else:
-            element_texts = [str(element)]
-        # 清理并添加非空文本
-        for text in element_texts:
-            cleaned = text.strip()
-            if cleaned:
-                texts.append(cleaned)
-    return join_str.join(texts)
-
-
-def extract_texts(elements: SelectorList, join_str: str = " ") -> List[str]:
-    """
-    从元素列表中提取多个文本列表
-    
-    :param elements: SelectorList元素列表
-    :param join_str: 单个节点内文本拼接分隔符
-    :return: 纯文本列表(每个元素对应一个节点的文本)
-    
-    示例:
-        li_elements = selector.css('.list li')
-        li_texts = extract_texts(li_elements)
-    """
-    result = []
-    for element in elements:
-        # 对每个元素提取文本
-        if hasattr(element, 'xpath'):
-            texts = element.xpath('.//text()').getall()
-        else:
-            texts = [str(element)]
-            
-        # 清理文本并拼接
-        clean_texts = [text.strip() for text in texts if text.strip()]
-        if clean_texts:
-            result.append(join_str.join(clean_texts))
-            
-    return result
-
-
-def extract_attr(elements: SelectorList, attr_name: str, default: Any = None) -> Any:
-    """
-    从元素列表中提取单个元素的属性值
-    
-    :param elements: SelectorList元素列表
-    :param attr_name: 属性名称
-    :param default: 默认返回值
-    :return: 属性值或默认值
-    
-    示例:
-        link_elements = selector.css('.link')
-        link_href = extract_attr(link_elements, 'href')
-    """
-    # 使用parsel的attrib属性获取第一个匹配元素的属性值
-    if hasattr(elements, 'attrib'):
-        return elements.attrib.get(attr_name, default)
-    # 如果elements是SelectorList，获取第一个元素的属性
-    elif len(elements) > 0 and hasattr(elements[0], 'attrib'):
-        return elements[0].attrib.get(attr_name, default)
-    return default
-
-
-def extract_attrs(elements: SelectorList, attr_name: str) -> List[Any]:
-    """
-    从元素列表中提取多个元素的属性值列表
-    
-    :param elements: SelectorList元素列表
-    :param attr_name: 属性名称
-    :return: 属性值列表
-    
-    示例:
-        all_links = selector.css('a')
-        all_hrefs = extract_attrs(all_links, 'href')
-    """
-    result = []
-    for element in elements:
-        # 使用parsel的attrib属性获取元素的属性值
-        if hasattr(element, 'attrib'):
-            attr_value = element.attrib.get(attr_name)
-            if attr_value is not None:
-                result.append(attr_value)
-                
-    return result
-
-
-def is_xpath(query: str) -> bool:
-    """
-    判断查询语句是否为XPath
-    
-    :param query: 查询语句
-    :return: 是否为XPath
-    """
-    return query.startswith(('/', '//', './'))
-
-
-__all__ = [
-    "extract_text",
-    "extract_texts",
-    "extract_attr",
-    "extract_attrs",
-    "is_xpath"
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+选择器辅助工具模块
+==================
+提供用于处理parsel选择器的辅助函数，用于提取文本和属性等操作。
+
+该模块包含以下主要函数：
+- extract_text: 从元素列表中提取文本并拼接
+- extract_texts: 从元素列表中提取多个文本列表
+- extract_attr: 从元素列表中提取单个元素的属性值
+- extract_attrs: 从元素列表中提取多个元素的属性值列表
+- is_xpath: 判断查询语句是否为XPath
+
+所有方法都采用了简洁直观的命名风格，便于记忆和使用。
+"""
+
+from typing import List, Any, Optional
+from parsel import Selector, SelectorList
+
+
+def extract_text(elements: SelectorList, join_str: str = " ") -> str:
+    """
+    从元素列表中提取文本并拼接
+    
+    :param elements: SelectorList元素列表
+    :param join_str: 文本拼接分隔符
+    :return: 拼接后的文本
+    
+    示例:
+        title_elements = selector.css('title')
+        title_text = extract_text(title_elements)
+    """
+    texts = []
+    for element in elements:
+        # 获取元素的所有文本节点
+        if hasattr(element, 'xpath'):
+            element_texts = element.xpath('.//text()').getall()
+        else:
+            element_texts = [str(element)]
+        # 清理并添加非空文本
+        for text in element_texts:
+            cleaned = text.strip()
+            if cleaned:
+                texts.append(cleaned)
+    return join_str.join(texts)
+
+
+def extract_texts(elements: SelectorList, join_str: str = " ") -> List[str]:
+    """
+    从元素列表中提取多个文本列表
+    
+    :param elements: SelectorList元素列表
+    :param join_str: 单个节点内文本拼接分隔符
+    :return: 纯文本列表(每个元素对应一个节点的文本)
+    
+    示例:
+        li_elements = selector.css('.list li')
+        li_texts = extract_texts(li_elements)
+    """
+    result = []
+    for element in elements:
+        # 对每个元素提取文本
+        if hasattr(element, 'xpath'):
+            texts = element.xpath('.//text()').getall()
+        else:
+            texts = [str(element)]
+            
+        # 清理文本并拼接
+        clean_texts = [text.strip() for text in texts if text.strip()]
+        if clean_texts:
+            result.append(join_str.join(clean_texts))
+            
+    return result
+
+
+def extract_attr(elements: SelectorList, attr_name: str, default: Any = None) -> Any:
+    """
+    从元素列表中提取单个元素的属性值
+    
+    :param elements: SelectorList元素列表
+    :param attr_name: 属性名称
+    :param default: 默认返回值
+    :return: 属性值或默认值
+    
+    示例:
+        link_elements = selector.css('.link')
+        link_href = extract_attr(link_elements, 'href')
+    """
+    # 使用parsel的attrib属性获取第一个匹配元素的属性值
+    if hasattr(elements, 'attrib'):
+        return elements.attrib.get(attr_name, default)
+    # 如果elements是SelectorList，获取第一个元素的属性
+    elif len(elements) > 0 and hasattr(elements[0], 'attrib'):
+        return elements[0].attrib.get(attr_name, default)
+    return default
+
+
+def extract_attrs(elements: SelectorList, attr_name: str) -> List[Any]:
+    """
+    从元素列表中提取多个元素的属性值列表
+    
+    :param elements: SelectorList元素列表
+    :param attr_name: 属性名称
+    :return: 属性值列表
+    
+    示例:
+        all_links = selector.css('a')
+        all_hrefs = extract_attrs(all_links, 'href')
+    """
+    result = []
+    for element in elements:
+        # 使用parsel的attrib属性获取元素的属性值
+        if hasattr(element, 'attrib'):
+            attr_value = element.attrib.get(attr_name)
+            if attr_value is not None:
+                result.append(attr_value)
+                
+    return result
+
+
+def is_xpath(query: str) -> bool:
+    """
+    判断查询语句是否为XPath
+    
+    :param query: 查询语句
+    :return: 是否为XPath
+    """
+    return query.startswith(('/', '//', './'))
+
+
+__all__ = [
+    "extract_text",
+    "extract_texts",
+    "extract_attr",
+    "extract_attrs",
+    "is_xpath"
 ]

crawlo/utils/singleton.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+单例模式工具模块
+================
+
+提供同步和异步两种单例实现方式，适用于不同的使用场景。
+
+使用场景：
+1. 同步单例：用于框架初始化、配置管理等同步代码
+2. 异步单例：用于数据库连接池、网络资源等异步代码
+
+示例：
+    # 同步单例
+    @singleton
+    class CoreInitializer:
+        pass
+    
+    # 异步单例（在连接池管理器中使用）
+    class MySQLConnectionPoolManager:
+        _instances: Dict[str, 'MySQLConnectionPoolManager'] = {}
+        _lock = asyncio.Lock()
+        
+        @classmethod
+        async def get_pool(cls, ...):
+            async with cls._lock:
+                if pool_key not in cls._instances:
+                    cls._instances[pool_key] = cls(pool_key)
+            return cls._instances[pool_key].pool
+"""
+
+import threading
+from typing import Any, Dict, Type
+
+
+class SingletonMeta(type):
+    """单例元类"""
+    _instances: Dict[Type, Any] = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            with cls._lock:
+                if cls not in cls._instances:
+                    instance = super().__call__(*args, **kwargs)
+                    cls._instances[cls] = instance
+        return cls._instances[cls]
+
+
+def singleton(cls):
+    """
+    单例装饰器
+    
+    Args:
+        cls: 要装饰的类
+        
+    Returns:
+        装饰后的类，确保只有一个实例
+    """
+    instances = {}
+    lock = threading.Lock()
+
+    def get_instance(*args, **kwargs):
+        if cls not in instances:
+            with lock:
+                if cls not in instances:
+                    instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return get_instance