crawlo 1.3.3__py3-none-any.whl → 1.3.5__py3-none-any.whl

crawlo/factories/crawler.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawler组件工厂 - 专门用于创建Crawler相关组件
+"""
+
+from typing import Any, Type
+
+from .base import ComponentFactory, ComponentSpec
+from .registry import get_component_registry
+
+
+class CrawlerComponentFactory(ComponentFactory):
+    """Crawler组件工厂"""
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建Crawler相关组件"""
+        # 检查是否需要crawler依赖
+        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
+            raise ValueError(f"Crawler instance required for component {spec.name}")
+        
+        return spec.factory_func(**kwargs)
+    
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型"""
+        # 这里可以根据需要定义支持的组件类型
+        supported_types = [
+            'Engine', 'Scheduler', 'StatsCollector', 
+            'Subscriber', 'ExtensionManager'
+        ]
+        return component_type.__name__ in supported_types
+
+
+def register_crawler_components():
+    """注册Crawler相关组件"""
+    registry = get_component_registry()
+    
+    # 注册工厂
+    registry.register_factory(CrawlerComponentFactory())
+    
+    # 注册组件规范
+    
+    # Engine组件
+    def create_engine(crawler, **kwargs):
+        from crawlo.core.engine import Engine
+        return Engine(crawler)
+    
+    registry.register(ComponentSpec(
+        name='engine',
+        component_type=type('Engine', (), {}),
+        factory_func=create_engine,
+        dependencies=['crawler']
+    ))
+    
+    # Scheduler组件
+    def create_scheduler(crawler, **kwargs):
+        from crawlo.core.scheduler import Scheduler
+        return Scheduler.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='scheduler',
+        component_type=type('Scheduler', (), {}),
+        factory_func=create_scheduler,
+        dependencies=['crawler']
+    ))
+    
+    # StatsCollector组件
+    def create_stats(crawler, **kwargs):
+        from crawlo.stats_collector import StatsCollector
+        return StatsCollector(crawler)
+    
+    registry.register(ComponentSpec(
+        name='stats',
+        component_type=type('StatsCollector', (), {}),
+        factory_func=create_stats,
+        dependencies=['crawler']
+    ))
+    
+    # Subscriber组件
+    def create_subscriber(**kwargs):
+        from crawlo.subscriber import Subscriber
+        return Subscriber()
+    
+    registry.register(ComponentSpec(
+        name='subscriber',
+        component_type=type('Subscriber', (), {}),
+        factory_func=create_subscriber
+    ))
+    
+    # ExtensionManager组件
+    def create_extension_manager(crawler, **kwargs):
+        from crawlo.extension import ExtensionManager
+        return ExtensionManager.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='extension_manager',
+        component_type=type('ExtensionManager', (), {}),
+        factory_func=create_extension_manager,
+        dependencies=['crawler']
+    ))
+
+
+# 自动注册
+register_crawler_components()

crawlo/factories/registry.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件注册表 - 管理所有组件的注册和创建
+"""
+
+import threading
+from typing import Dict, List, Type, Any, Optional
+
+from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
+
+
+class ComponentRegistry:
+    """
+    组件注册表
+    
+    职责：
+    1. 管理组件规范的注册
+    2. 根据类型查找合适的工厂
+    3. 处理依赖关系
+    4. 创建组件实例
+    """
+    
+    def __init__(self):
+        self._specs: Dict[str, ComponentSpec] = {}
+        self._factories: List[ComponentFactory] = []
+        self._default_factory = DefaultComponentFactory()
+        self._lock = threading.RLock()
+    
+    def register(self, spec: ComponentSpec):
+        """注册组件规范"""
+        with self._lock:
+            self._specs[spec.name] = spec
+    
+    def register_factory(self, factory: ComponentFactory):
+        """注册组件工厂"""
+        with self._lock:
+            self._factories.append(factory)
+    
+    def get_spec(self, name: str) -> Optional[ComponentSpec]:
+        """获取组件规范"""
+        with self._lock:
+            return self._specs.get(name)
+    
+    def get_factory(self, component_type: Type) -> ComponentFactory:
+        """获取支持指定类型的工厂"""
+        with self._lock:
+            for factory in self._factories:
+                if factory.supports(component_type):
+                    return factory
+            return self._default_factory
+    
+    def create(self, name: str, **kwargs) -> Any:
+        """创建组件实例"""
+        spec = self.get_spec(name)
+        if not spec:
+            raise ValueError(f"Component spec '{name}' not found")
+        
+        factory = self.get_factory(spec.component_type)
+        return factory.create(spec, **kwargs)
+    
+    def get(self, name: str, **kwargs) -> Any:
+        """获取组件实例（create的别名）"""
+        return self.create(name, **kwargs)
+    
+    def list_components(self) -> List[str]:
+        """列出所有已注册的组件"""
+        with self._lock:
+            return list(self._specs.keys())
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._specs.clear()
+            self._factories.clear()
+            self._default_factory.clear_singletons()
+
+
+# 全局组件注册表
+_global_registry = ComponentRegistry()
+
+
+def get_component_registry() -> ComponentRegistry:
+    """获取全局组件注册表"""
+    return _global_registry

crawlo/filters/__init__.py

@@ -1,154 +1,154 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-Crawlo Filters Module
-====================
-提供多种请求去重过滤器实现。
-
-过滤器类型:
-- MemoryFilter: 基于内存的高效去重，适合单机模式
-- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
-- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
-
-核心接口:
-- BaseFilter: 所有过滤器的基类
-- requested(): 检查请求是否重复的主要方法
-"""
-from abc import ABC, abstractmethod
-from typing import Optional
-
-from crawlo.utils.request import request_fingerprint
-
-
-class BaseFilter(ABC):
-    """
-    请求去重过滤器基类
-    
-    提供统一的去重接口和统计功能。
-    所有过滤器实现都应该继承此类。
-    """
-
-    def __init__(self, logger, stats, debug: bool = False):
-        """
-        初始化过滤器
-        
-        :param logger: 日志器实例
-        :param stats: 统计信息存储
-        :param debug: 是否启用调试日志
-        """
-        self.logger = logger
-        self.stats = stats
-        self.debug = debug
-        self._request_count = 0
-        self._duplicate_count = 0
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
-        return cls(*args, **kwargs)
-
-    def requested(self, request) -> bool:
-        """
-        检查请求是否重复（主要接口）
-        
-        :param request: 请求对象
-        :return: True 表示重复，False 表示新请求
-        """
-        self._request_count += 1
-        fp = request_fingerprint(request)
-        
-        if fp in self:
-            self._duplicate_count += 1
-            self.log_stats(request)
-            return True
-            
-        self.add_fingerprint(fp)
-        return False
-
-    @abstractmethod
-    def add_fingerprint(self, fp: str) -> None:
-        """
-        添加请求指纹（子类必须实现）
-        
-        :param fp: 请求指纹字符串
-        """
-        pass
-    
-    @abstractmethod
-    def __contains__(self, item: str) -> bool:
-        """
-        检查指纹是否存在（支持 in 操作符）
-        
-        :param item: 要检查的指纹
-        :return: 是否已存在
-        """
-        pass
-
-    def log_stats(self, request) -> None:
-        """
-        记录统计信息
-        
-        :param request: 重复的请求对象
-        """
-        if self.debug:
-            self.logger.debug(f'Filtered duplicate request: {request}')
-        self.stats.inc_value(f'{self}/filtered_count')
-    
-    def get_stats(self) -> dict:
-        """
-        获取过滤器统计信息
-        
-        :return: 统计信息字典
-        """
-        return {
-            'total_requests': self._request_count,
-            'duplicate_requests': self._duplicate_count,
-            'unique_requests': self._request_count - self._duplicate_count,
-            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
-        }
-    
-    def reset_stats(self) -> None:
-        """重置统计信息"""
-        self._request_count = 0
-        self._duplicate_count = 0
-    
-    def close(self) -> None:
-        """关闭过滤器并清理资源"""
-        pass
-
-    def __str__(self) -> str:
-        return f'{self.__class__.__name__}'
-
-
-# 导出所有可用的过滤器
-__all__ = ['BaseFilter']
-
-# 动态导入具体实现
-try:
-    from .memory_filter import MemoryFilter, MemoryFileFilter
-    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
-except ImportError:
-    MemoryFilter = None
-    MemoryFileFilter = None
-
-try:
-    from .aioredis_filter import AioRedisFilter
-    __all__.append('AioRedisFilter')
-except ImportError:
-    AioRedisFilter = None
-
-# 提供便捷的过滤器映射
-FILTER_MAP = {
-    'memory': MemoryFilter,
-    'memory_file': MemoryFileFilter,
-    'redis': AioRedisFilter,
-    'aioredis': AioRedisFilter,  # 别名
-}
-
-# 过滤掉不可用的过滤器
-FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
-
-def get_filter_class(name: str):
-    """根据名称获取过滤器类"""
-    if name in FILTER_MAP:
-        return FILTER_MAP[name]
-    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Crawlo Filters Module
+====================
+提供多种请求去重过滤器实现。
+
+过滤器类型:
+- MemoryFilter: 基于内存的高效去重，适合单机模式
+- AioRedisFilter: 基于Redis的分布式去重，适合分布式模式
+- MemoryFileFilter: 内存+文件持久化，适合需要重启恢复的场景
+
+核心接口:
+- BaseFilter: 所有过滤器的基类
+- requested(): 检查请求是否重复的主要方法
+"""
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from crawlo.utils.request import request_fingerprint
+
+
+class BaseFilter(ABC):
+    """
+    请求去重过滤器基类
+    
+    提供统一的去重接口和统计功能。
+    所有过滤器实现都应该继承此类。
+    """
+
+    def __init__(self, logger, stats, debug: bool = False):
+        """
+        初始化过滤器
+        
+        :param logger: 日志器实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试日志
+        """
+        self.logger = logger
+        self.stats = stats
+        self.debug = debug
+        self._request_count = 0
+        self._duplicate_count = 0
+
+    @classmethod
+    def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
+        return cls(*args, **kwargs)
+
+    def requested(self, request) -> bool:
+        """
+        检查请求是否重复（主要接口）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        self._request_count += 1
+        fp = request_fingerprint(request)
+        
+        if fp in self:
+            self._duplicate_count += 1
+            self.log_stats(request)
+            return True
+            
+        self.add_fingerprint(fp)
+        return False
+
+    @abstractmethod
+    def add_fingerprint(self, fp: str) -> None:
+        """
+        添加请求指纹（子类必须实现）
+        
+        :param fp: 请求指纹字符串
+        """
+        pass
+    
+    @abstractmethod
+    def __contains__(self, item: str) -> bool:
+        """
+        检查指纹是否存在（支持 in 操作符）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        pass
+
+    def log_stats(self, request) -> None:
+        """
+        记录统计信息
+        
+        :param request: 重复的请求对象
+        """
+        if self.debug:
+            self.logger.debug(f'Filtered duplicate request: {request}')
+        self.stats.inc_value(f'{self}/filtered_count')
+    
+    def get_stats(self) -> dict:
+        """
+        获取过滤器统计信息
+        
+        :return: 统计信息字典
+        """
+        return {
+            'total_requests': self._request_count,
+            'duplicate_requests': self._duplicate_count,
+            'unique_requests': self._request_count - self._duplicate_count,
+            'duplicate_rate': f"{self._duplicate_count / max(1, self._request_count) * 100:.2f}%"
+        }
+    
+    def reset_stats(self) -> None:
+        """重置统计信息"""
+        self._request_count = 0
+        self._duplicate_count = 0
+    
+    def close(self) -> None:
+        """关闭过滤器并清理资源"""
+        pass
+
+    def __str__(self) -> str:
+        return f'{self.__class__.__name__}'
+
+
+# 导出所有可用的过滤器
+__all__ = ['BaseFilter']
+
+# 动态导入具体实现
+try:
+    from .memory_filter import MemoryFilter, MemoryFileFilter
+    __all__.extend(['MemoryFilter', 'MemoryFileFilter'])
+except ImportError:
+    MemoryFilter = None
+    MemoryFileFilter = None
+
+try:
+    from .aioredis_filter import AioRedisFilter
+    __all__.append('AioRedisFilter')
+except ImportError:
+    AioRedisFilter = None
+
+# 提供便捷的过滤器映射
+FILTER_MAP = {
+    'memory': MemoryFilter,
+    'memory_file': MemoryFileFilter,
+    'redis': AioRedisFilter,
+    'aioredis': AioRedisFilter,  # 别名
+}
+
+# 过滤掉不可用的过滤器
+FILTER_MAP = {k: v for k, v in FILTER_MAP.items() if v is not None}
+
+def get_filter_class(name: str):
+    """根据名称获取过滤器类"""
+    if name in FILTER_MAP:
+        return FILTER_MAP[name]
+    raise ValueError(f"未知的过滤器类型: {name}。可用类型: {list(FILTER_MAP.keys())}")