crawlo 1.4.5__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/factories/registry.py

@@ -1,85 +1,85 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-组件注册表 - 管理所有组件的注册和创建
-"""
-
-import threading
-from typing import Dict, List, Type, Any, Optional
-
-from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
-
-
-class ComponentRegistry:
-    """
-    组件注册表
-    
-    职责：
-    1. 管理组件规范的注册
-    2. 根据类型查找合适的工厂
-    3. 处理依赖关系
-    4. 创建组件实例
-    """
-    
-    def __init__(self):
-        self._specs: Dict[str, ComponentSpec] = {}
-        self._factories: List[ComponentFactory] = []
-        self._default_factory = DefaultComponentFactory()
-        self._lock = threading.RLock()
-    
-    def register(self, spec: ComponentSpec):
-        """注册组件规范"""
-        with self._lock:
-            self._specs[spec.name] = spec
-    
-    def register_factory(self, factory: ComponentFactory):
-        """注册组件工厂"""
-        with self._lock:
-            self._factories.append(factory)
-    
-    def get_spec(self, name: str) -> Optional[ComponentSpec]:
-        """获取组件规范"""
-        with self._lock:
-            return self._specs.get(name)
-    
-    def get_factory(self, component_type: Type) -> ComponentFactory:
-        """获取支持指定类型的工厂"""
-        with self._lock:
-            for factory in self._factories:
-                if factory.supports(component_type):
-                    return factory
-            return self._default_factory
-    
-    def create(self, name: str, **kwargs) -> Any:
-        """创建组件实例"""
-        spec = self.get_spec(name)
-        if not spec:
-            raise ValueError(f"Component spec '{name}' not found")
-        
-        factory = self.get_factory(spec.component_type)
-        return factory.create(spec, **kwargs)
-    
-    def get(self, name: str, **kwargs) -> Any:
-        """获取组件实例（create的别名）"""
-        return self.create(name, **kwargs)
-    
-    def list_components(self) -> List[str]:
-        """列出所有已注册的组件"""
-        with self._lock:
-            return list(self._specs.keys())
-    
-    def clear(self):
-        """清空注册表"""
-        with self._lock:
-            self._specs.clear()
-            self._factories.clear()
-            self._default_factory.clear_singletons()
-
-
-# 全局组件注册表
-_global_registry = ComponentRegistry()
-
-
-def get_component_registry() -> ComponentRegistry:
-    """获取全局组件注册表"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件注册表 - 管理所有组件的注册和创建
+"""
+
+import threading
+from typing import Dict, List, Type, Any, Optional
+
+from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
+
+
+class ComponentRegistry:
+    """
+    组件注册表
+    
+    职责：
+    1. 管理组件规范的注册
+    2. 根据类型查找合适的工厂
+    3. 处理依赖关系
+    4. 创建组件实例
+    """
+    
+    def __init__(self):
+        self._specs: Dict[str, ComponentSpec] = {}
+        self._factories: List[ComponentFactory] = []
+        self._default_factory = DefaultComponentFactory()
+        self._lock = threading.RLock()
+    
+    def register(self, spec: ComponentSpec):
+        """注册组件规范"""
+        with self._lock:
+            self._specs[spec.name] = spec
+    
+    def register_factory(self, factory: ComponentFactory):
+        """注册组件工厂"""
+        with self._lock:
+            self._factories.append(factory)
+    
+    def get_spec(self, name: str) -> Optional[ComponentSpec]:
+        """获取组件规范"""
+        with self._lock:
+            return self._specs.get(name)
+    
+    def get_factory(self, component_type: Type) -> ComponentFactory:
+        """获取支持指定类型的工厂"""
+        with self._lock:
+            for factory in self._factories:
+                if factory.supports(component_type):
+                    return factory
+            return self._default_factory
+    
+    def create(self, name: str, **kwargs) -> Any:
+        """创建组件实例"""
+        spec = self.get_spec(name)
+        if not spec:
+            raise ValueError(f"Component spec '{name}' not found")
+        
+        factory = self.get_factory(spec.component_type)
+        return factory.create(spec, **kwargs)
+    
+    def get(self, name: str, **kwargs) -> Any:
+        """获取组件实例（create的别名）"""
+        return self.create(name, **kwargs)
+    
+    def list_components(self) -> List[str]:
+        """列出所有已注册的组件"""
+        with self._lock:
+            return list(self._specs.keys())
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._specs.clear()
+            self._factories.clear()
+            self._default_factory.clear_singletons()
+
+
+# 全局组件注册表
+_global_registry = ComponentRegistry()
+
+
+def get_component_registry() -> ComponentRegistry:
+    """获取全局组件注册表"""
     return _global_registry

crawlo/factories/utils.py

@@ -0,0 +1,135 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+工厂工具模块 - 提供通用的组件注册和创建工具
+"""
+
+from typing import Any, Callable, List, Optional, Type, Union
+from .base import ComponentSpec
+from .registry import get_component_registry
+
+
+def register_component(
+    name: str,
+    component_type: Union[Type, str],
+    factory_func: Callable[..., Any],
+    dependencies: Optional[List[str]] = None,
+    singleton: bool = False,
+    config_key: Optional[str] = None
+) -> None:
+    """
+    注册组件的便捷函数
+    
+    Args:
+        name: 组件名称
+        component_type: 组件类型
+        factory_func: 工厂函数
+        dependencies: 依赖列表
+        singleton: 是否单例
+        config_key: 配置键名
+    """
+    registry = get_component_registry()
+    
+    # 如果component_type是字符串，创建一个动态类型
+    if isinstance(component_type, str):
+        component_type = type(component_type, (), {})
+    
+    spec_kwargs = {
+        'name': name,
+        'component_type': component_type,
+        'factory_func': factory_func,
+        'dependencies': dependencies or [],
+        'singleton': singleton
+    }
+    
+    # 只有当config_key不为None时才添加
+    if config_key is not None:
+        spec_kwargs['config_key'] = config_key
+    
+    spec = ComponentSpec(**spec_kwargs)
+    
+    registry.register(spec)
+
+
+def register_components(component_list: List[dict]) -> None:
+    """
+    批量注册组件
+    
+    Args:
+        component_list: 组件定义列表，每个元素是一个包含组件信息的字典
+    """
+    for component_info in component_list:
+        register_component(**component_info)
+
+
+def create_component_factory(
+    component_name: str,
+    module_path: str,
+    class_name: str,
+    dependencies: Optional[List[str]] = None,
+    singleton: bool = False
+) -> Callable[..., Any]:
+    """
+    创建组件工厂函数的便捷函数
+    
+    Args:
+        component_name: 组件名称（用于错误信息）
+        module_path: 模块路径
+        class_name: 类名
+        dependencies: 依赖列表
+        singleton: 是否单例
+        
+    Returns:
+        工厂函数
+    """
+    def factory_func(*args, **kwargs):
+        try:
+            # 动态导入模块
+            module = __import__(module_path, fromlist=[class_name])
+            component_class = getattr(module, class_name)
+            
+            # 检查是否需要调用create_instance方法
+            if hasattr(component_class, 'create_instance'):
+                return component_class.create_instance(*args, **kwargs)
+            else:
+                return component_class(*args, **kwargs)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create {component_name}: {e}")
+    
+    return factory_func
+
+
+def create_crawler_component_factory(
+    component_name: str,
+    module_path: str,
+    class_name: str
+) -> Callable[..., Any]:
+    """
+    创建需要crawler依赖的组件工厂函数
+    
+    Args:
+        component_name: 组件名称
+        module_path: 模块路径
+        class_name: 类名
+        
+    Returns:
+        工厂函数
+    """
+    def factory_func(crawler=None, **kwargs):
+        if crawler is None:
+            raise ValueError(f"Crawler instance required for component {component_name}")
+        
+        try:
+            # 动态导入模块
+            module = __import__(module_path, fromlist=[class_name])
+            component_class = getattr(module, class_name)
+            
+            # 检查是否需要调用create_instance方法
+            if hasattr(component_class, 'create_instance'):
+                return component_class.create_instance(crawler, **kwargs)
+            else:
+                return component_class(crawler, **kwargs)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create {component_name}: {e}")
+    
+    return factory_func

crawlo/filters/__init__.py

@@ -1,154 +1,171 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Crawlo过滤器模块
-================
-提供多种请求去重过滤器实现。
-
-过滤器类型:
-- MemoryFilter: 基于内存的高效去重，适合单机模式
-- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
-- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
-
-核心接口:
-- BaseFilter: 所有过滤器的基类
-- requested(): 检查请求是否重复的主要方法
-"""
-from abc import ABC, abstractmethod
-from typing import Optional
-
-from crawlo.utils.request import request_fingerprint
-
-
-class BaseFilter(ABC):
-    """
-    请求去重过滤器基类
-    
-    提供统一的去重接口和统计功能。
-    所有过滤器实现都应该继承此类。
-    """
-
-    def __init__(self, logger, stats, debug: bool = False):
-        """
-        初始化过滤器
-        
-        :param logger: 日志器实例
-        :param stats: 统计信息存储
-        :param debug: 是否启用调试日志
-        """
-        self.logger = logger
-        self.stats = stats
-        self.debug = debug
-        self._request_count = 0
-        self._duplicate_count = 0
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
-        return cls(*args, **kwargs)
-
-    def requested(self, request) -> bool:
-        """
-        检查请求是否重复（主要接口）
-        
-        :param request: 请求对象
-        :return: True 表示重复，False 表示新请求
-        """
-        self._request_count += 1
-        fp = request_fingerprint(request)
-        
-        if fp in self:
-            self._duplicate_count += 1
-            self.log_stats(request)
-            return True
-            
-        self.add_fingerprint(fp)
-        return False
-
-    @abstractmethod
-    def add_fingerprint(self, fp: str) -> None:
-        """
-        添加请求指纹（子类必须实现）
-        
-        :param fp: 请求指纹字符串
-        """
-        pass
-    
-    @abstractmethod
-    def __contains__(self, item: str) -> bool:
-        """
-        检查指纹是否存在（支持 in 操作符）
-        
-        :param item: 要检查的指纹
-        :return: 是否已存在
-        """
-        pass
-
-    def log_stats(self, request) -> None:
-        """
-        记录统计信息
-        
-        :param request: 重复的请求对象
-        """
-        if self.debug:
-            self.logger.debug(f'过滤重复请求: {request}')
-        self.stats.inc_value(f'{self}/filtered_count')
-    
-    def get_stats(self) -> dict:
-        """
-        获取过滤器统计信息
-        
-        :return: 统计信息字典
-        """
-        return {
-            'total_requests': self._request_count,
-            'duplicate_requests': self._duplicate_count,
-            'unique_requests': self._request_count - self._duplicate_count,
-            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
-        }
-    
-    def reset_stats(self) -> None:
-        """重置统计信息"""
-        self._request_count = 0
-        self._duplicate_count = 0
-    
-    def close(self) -> None:
-        """关闭过滤器并清理资源"""
-        pass
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__}'
-
-
-# 导出所有可用的过滤器
-__all__ = ['BaseFilter']
-
-# 动态导入具体实现
-try:
-    from .memory_filter import MemoryFilter, MemoryFileFilter
-    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
-except ImportError:
-    MemoryFilter = None
-    MemoryFileFilter = None
-
-try:
-    from .aioredis_filter import AioRedisFilter
-    __all__.append('AioRedisFilter')
-except ImportError:
-    AioRedisFilter = None
-
-# 提供便捷的过滤器映射
-FILTER_MAP = {
-    'memory': MemoryFilter,
-    'memory_file': MemoryFileFilter,
-    'redis': AioRedisFilter,
-    'aioredis': AioRedisFilter,  # 别名
-}
-
-# 过滤掉不可用的过滤器
-FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
-
-def get_filter_class(name: str):
-    """根据名称获取过滤器类"""
-    if name in FILTER_MAP:
-        return FILTER_MAP[name]
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo过滤器模块
+================
+提供多种请求去重过滤器实现。
+
+过滤器类型:
+- MemoryFilter: 基于内存的高效去重，适合单机模式
+- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
+- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
+
+核心接口:
+- BaseFilter: 所有过滤器的基类
+- requested(): 检查请求是否重复的主要方法
+"""
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from crawlo.utils.fingerprint import FingerprintGenerator
+
+
+class BaseFilter(ABC):
+    """
+    请求去重过滤器基类
+    
+    提供统一的去重接口和统计功能。
+    所有过滤器实现都应该继承此类。
+    """
+
+    def __init__(self, logger, stats, debug: bool = False):
+        """
+        初始化过滤器
+        
+        :param logger: 日志器实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试日志
+        """
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+        self._request_count = 0
+        self._duplicate_count = 0
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def _get_fingerprint(self, request) -> str:
+        """
+        获取请求指纹（内部辅助方法）
+        
+        使用统一的 FingerprintGenerator 生成请求指纹。
+        子类可以直接调用此方法，避免重复实现。
+        
+        :param request: 请求对象
+        :return: 请求指纹字符串
+        """
+        return FingerprintGenerator.request_fingerprint(
+            request.method,
+            request.url,
+            request.body or b'',
+            dict(request.headers) if hasattr(request, 'headers') else {}
+        )
+
+    def requested(self, request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        self._request_count += 1
+        fp = self._get_fingerprint(request)
+        
+        if fp in self:
+            self._duplicate_count += 1
+            self.log_stats(request)
+            return True
+            
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹（子类必须实现）
+        
+        :param fp: 请求指纹字符串
+        """
+        pass
+    
+    @abstractmethod
+    def __contains__(self, item: str) -> bool:
+        """
+        检查指纹是否存在（支持 in 操作符）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        pass
+
+    def log_stats(self, request) -> None:
+        """
+        记录统计信息
+        
+        :param request: 重复的请求对象
+        """
+        if self.debug:
+            self.logger.debug(f'过滤重复请求: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+    
+    def get_stats(self) -> dict:
+        """
+        获取过滤器统计信息
+        
+        :return: 统计信息字典
+        """
+        return {
+            'total_requests': self._request_count,
+            'duplicate_requests': self._duplicate_count,
+            'unique_requests': self._request_count - self._duplicate_count,
+            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
+        }
+    
+    def reset_stats(self) -> None:
+        """重置统计信息"""
+        self._request_count = 0
+        self._duplicate_count = 0
+    
+    def close(self) -> None:
+        """关闭过滤器并清理资源"""
+        pass
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'
+
+
+# 导出所有可用的过滤器
+__all__ = ['BaseFilter']
+
+# 动态导入具体实现
+try:
+    from .memory_filter import MemoryFilter, MemoryFileFilter
+    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
+except ImportError:
+    MemoryFilter = None
+    MemoryFileFilter = None
+
+try:
+    from .aioredis_filter import AioRedisFilter
+    __all__.append('AioRedisFilter')
+except ImportError:
+    AioRedisFilter = None
+
+# 提供便捷的过滤器映射
+FILTER_MAP = {
+    'memory': MemoryFilter,
+    'memory_file': MemoryFileFilter,
+    'redis': AioRedisFilter,
+    'aioredis': AioRedisFilter,  # 别名
+}
+
+# 过滤掉不可用的过滤器
+FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
+
+def get_filter_class(name: str):
+    """根据名称获取过滤器类"""
+    if name in FILTER_MAP:
+        return FILTER_MAP[name]
     raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")