crawlo 1.3.4__py3-none-any.whl → 1.3.6__py3-none-any.whl

crawlo/factories/base.py

@@ -1,69 +1,69 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-组件工厂基类和规范
-"""
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Type, Any, Dict, Optional, Callable
-
-
-@dataclass
-class ComponentSpec:
-    """组件规范 - 定义如何创建组件"""
-    
-    name: str
-    component_type: Type
-    factory_func: Callable[..., Any]
-    dependencies: list = None
-    singleton: bool = False
-    config_key: str = None
-    
-    def __post_init__(self):
-        if self.dependencies is None:
-            self.dependencies = []
-
-
-class ComponentFactory(ABC):
-    """组件工厂基类"""
-    
-    @abstractmethod
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建组件实例"""
-        pass
-    
-    @abstractmethod
-    def supports(self, component_type: Type) -> bool:
-        """检查是否支持指定类型的组件"""
-        pass
-
-
-class DefaultComponentFactory(ComponentFactory):
-    """默认组件工厂实现"""
-    
-    def __init__(self):
-        self._instances: Dict[str, Any] = {}
-    
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建组件实例"""
-        # 单例模式检查
-        if spec.singleton and spec.name in self._instances:
-            return self._instances[spec.name]
-        
-        # 调用工厂函数创建实例
-        instance = spec.factory_func(**kwargs)
-        
-        # 保存单例实例
-        if spec.singleton:
-            self._instances[spec.name] = instance
-        
-        return instance
-    
-    def supports(self, component_type: Type) -> bool:
-        """支持所有类型"""
-        return True
-    
-    def clear_singletons(self):
-        """清除单例实例（测试用）"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件工厂基类和规范
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Type, Any, Dict, Optional, Callable
+
+
+@dataclass
+class ComponentSpec:
+    """组件规范 - 定义如何创建组件"""
+    
+    name: str
+    component_type: Type
+    factory_func: Callable[..., Any]
+    dependencies: list = None
+    singleton: bool = False
+    config_key: str = None
+    
+    def __post_init__(self):
+        if self.dependencies is None:
+            self.dependencies = []
+
+
+class ComponentFactory(ABC):
+    """组件工厂基类"""
+    
+    @abstractmethod
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        pass
+    
+    @abstractmethod
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型的组件"""
+        pass
+
+
+class DefaultComponentFactory(ComponentFactory):
+    """默认组件工厂实现"""
+    
+    def __init__(self):
+        self._instances: Dict[str, Any] = {}
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        # 单例模式检查
+        if spec.singleton and spec.name in self._instances:
+            return self._instances[spec.name]
+        
+        # 调用工厂函数创建实例
+        instance = spec.factory_func(**kwargs)
+        
+        # 保存单例实例
+        if spec.singleton:
+            self._instances[spec.name] = instance
+        
+        return instance
+    
+    def supports(self, component_type: Type) -> bool:
+        """支持所有类型"""
+        return True
+    
+    def clear_singletons(self):
+        """清除单例实例（测试用）"""
         self._instances.clear()

crawlo/factories/crawler.py

@@ -1,104 +1,104 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawler组件工厂 - 专门用于创建Crawler相关组件
-"""
-
-from typing import Any, Type
-
-from .base import ComponentFactory, ComponentSpec
-from .registry import get_component_registry
-
-
-class CrawlerComponentFactory(ComponentFactory):
-    """Crawler组件工厂"""
-    
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建Crawler相关组件"""
-        # 检查是否需要crawler依赖
-        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
-            raise ValueError(f"Crawler instance required for component {spec.name}")
-        
-        return spec.factory_func(**kwargs)
-    
-    def supports(self, component_type: Type) -> bool:
-        """检查是否支持指定类型"""
-        # 这里可以根据需要定义支持的组件类型
-        supported_types = [
-            'Engine', 'Scheduler', 'StatsCollector', 
-            'Subscriber', 'ExtensionManager'
-        ]
-        return component_type.__name__ in supported_types
-
-
-def register_crawler_components():
-    """注册Crawler相关组件"""
-    registry = get_component_registry()
-    
-    # 注册工厂
-    registry.register_factory(CrawlerComponentFactory())
-    
-    # 注册组件规范
-    
-    # Engine组件
-    def create_engine(crawler, **kwargs):
-        from crawlo.core.engine import Engine
-        return Engine(crawler)
-    
-    registry.register(ComponentSpec(
-        name='engine',
-        component_type=type('Engine', (), {}),
-        factory_func=create_engine,
-        dependencies=['crawler']
-    ))
-    
-    # Scheduler组件
-    def create_scheduler(crawler, **kwargs):
-        from crawlo.core.scheduler import Scheduler
-        return Scheduler.create_instance(crawler)
-    
-    registry.register(ComponentSpec(
-        name='scheduler',
-        component_type=type('Scheduler', (), {}),
-        factory_func=create_scheduler,
-        dependencies=['crawler']
-    ))
-    
-    # StatsCollector组件
-    def create_stats(crawler, **kwargs):
-        from crawlo.stats_collector import StatsCollector
-        return StatsCollector(crawler)
-    
-    registry.register(ComponentSpec(
-        name='stats',
-        component_type=type('StatsCollector', (), {}),
-        factory_func=create_stats,
-        dependencies=['crawler']
-    ))
-    
-    # Subscriber组件
-    def create_subscriber(**kwargs):
-        from crawlo.subscriber import Subscriber
-        return Subscriber()
-    
-    registry.register(ComponentSpec(
-        name='subscriber',
-        component_type=type('Subscriber', (), {}),
-        factory_func=create_subscriber
-    ))
-    
-    # ExtensionManager组件
-    def create_extension_manager(crawler, **kwargs):
-        from crawlo.extension import ExtensionManager
-        return ExtensionManager.create_instance(crawler)
-    
-    registry.register(ComponentSpec(
-        name='extension_manager',
-        component_type=type('ExtensionManager', (), {}),
-        factory_func=create_extension_manager,
-        dependencies=['crawler']
-    ))
-
-
-# 自动注册
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawler组件工厂 - 专门用于创建Crawler相关组件
+"""
+
+from typing import Any, Type
+
+from .base import ComponentFactory, ComponentSpec
+from .registry import get_component_registry
+
+
+class CrawlerComponentFactory(ComponentFactory):
+    """Crawler组件工厂"""
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建Crawler相关组件"""
+        # 检查是否需要crawler依赖
+        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
+            raise ValueError(f"Crawler instance required for component {spec.name}")
+        
+        return spec.factory_func(**kwargs)
+    
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型"""
+        # 这里可以根据需要定义支持的组件类型
+        supported_types = [
+            'Engine', 'Scheduler', 'StatsCollector', 
+            'Subscriber', 'ExtensionManager'
+        ]
+        return component_type.__name__ in supported_types
+
+
+def register_crawler_components():
+    """注册Crawler相关组件"""
+    registry = get_component_registry()
+    
+    # 注册工厂
+    registry.register_factory(CrawlerComponentFactory())
+    
+    # 注册组件规范
+    
+    # Engine组件
+    def create_engine(crawler, **kwargs):
+        from crawlo.core.engine import Engine
+        return Engine(crawler)
+    
+    registry.register(ComponentSpec(
+        name='engine',
+        component_type=type('Engine', (), {}),
+        factory_func=create_engine,
+        dependencies=['crawler']
+    ))
+    
+    # Scheduler组件
+    def create_scheduler(crawler, **kwargs):
+        from crawlo.core.scheduler import Scheduler
+        return Scheduler.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='scheduler',
+        component_type=type('Scheduler', (), {}),
+        factory_func=create_scheduler,
+        dependencies=['crawler']
+    ))
+    
+    # StatsCollector组件
+    def create_stats(crawler, **kwargs):
+        from crawlo.stats_collector import StatsCollector
+        return StatsCollector(crawler)
+    
+    registry.register(ComponentSpec(
+        name='stats',
+        component_type=type('StatsCollector', (), {}),
+        factory_func=create_stats,
+        dependencies=['crawler']
+    ))
+    
+    # Subscriber组件
+    def create_subscriber(**kwargs):
+        from crawlo.subscriber import Subscriber
+        return Subscriber()
+    
+    registry.register(ComponentSpec(
+        name='subscriber',
+        component_type=type('Subscriber', (), {}),
+        factory_func=create_subscriber
+    ))
+    
+    # ExtensionManager组件
+    def create_extension_manager(crawler, **kwargs):
+        from crawlo.extension import ExtensionManager
+        return ExtensionManager.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='extension_manager',
+        component_type=type('ExtensionManager', (), {}),
+        factory_func=create_extension_manager,
+        dependencies=['crawler']
+    ))
+
+
+# 自动注册
 register_crawler_components()

crawlo/factories/registry.py

@@ -1,85 +1,85 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-组件注册表 - 管理所有组件的注册和创建
-"""
-
-import threading
-from typing import Dict, List, Type, Any, Optional
-
-from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
-
-
-class ComponentRegistry:
-    """
-    组件注册表
-    
-    职责：
-    1. 管理组件规范的注册
-    2. 根据类型查找合适的工厂
-    3. 处理依赖关系
-    4. 创建组件实例
-    """
-    
-    def __init__(self):
-        self._specs: Dict[str, ComponentSpec] = {}
-        self._factories: List[ComponentFactory] = []
-        self._default_factory = DefaultComponentFactory()
-        self._lock = threading.RLock()
-    
-    def register(self, spec: ComponentSpec):
-        """注册组件规范"""
-        with self._lock:
-            self._specs[spec.name] = spec
-    
-    def register_factory(self, factory: ComponentFactory):
-        """注册组件工厂"""
-        with self._lock:
-            self._factories.append(factory)
-    
-    def get_spec(self, name: str) -> Optional[ComponentSpec]:
-        """获取组件规范"""
-        with self._lock:
-            return self._specs.get(name)
-    
-    def get_factory(self, component_type: Type) -> ComponentFactory:
-        """获取支持指定类型的工厂"""
-        with self._lock:
-            for factory in self._factories:
-                if factory.supports(component_type):
-                    return factory
-            return self._default_factory
-    
-    def create(self, name: str, **kwargs) -> Any:
-        """创建组件实例"""
-        spec = self.get_spec(name)
-        if not spec:
-            raise ValueError(f"Component spec '{name}' not found")
-        
-        factory = self.get_factory(spec.component_type)
-        return factory.create(spec, **kwargs)
-    
-    def get(self, name: str, **kwargs) -> Any:
-        """获取组件实例（create的别名）"""
-        return self.create(name, **kwargs)
-    
-    def list_components(self) -> List[str]:
-        """列出所有已注册的组件"""
-        with self._lock:
-            return list(self._specs.keys())
-    
-    def clear(self):
-        """清空注册表"""
-        with self._lock:
-            self._specs.clear()
-            self._factories.clear()
-            self._default_factory.clear_singletons()
-
-
-# 全局组件注册表
-_global_registry = ComponentRegistry()
-
-
-def get_component_registry() -> ComponentRegistry:
-    """获取全局组件注册表"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件注册表 - 管理所有组件的注册和创建
+"""
+
+import threading
+from typing import Dict, List, Type, Any, Optional
+
+from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
+
+
+class ComponentRegistry:
+    """
+    组件注册表
+    
+    职责：
+    1. 管理组件规范的注册
+    2. 根据类型查找合适的工厂
+    3. 处理依赖关系
+    4. 创建组件实例
+    """
+    
+    def __init__(self):
+        self._specs: Dict[str, ComponentSpec] = {}
+        self._factories: List[ComponentFactory] = []
+        self._default_factory = DefaultComponentFactory()
+        self._lock = threading.RLock()
+    
+    def register(self, spec: ComponentSpec):
+        """注册组件规范"""
+        with self._lock:
+            self._specs[spec.name] = spec
+    
+    def register_factory(self, factory: ComponentFactory):
+        """注册组件工厂"""
+        with self._lock:
+            self._factories.append(factory)
+    
+    def get_spec(self, name: str) -> Optional[ComponentSpec]:
+        """获取组件规范"""
+        with self._lock:
+            return self._specs.get(name)
+    
+    def get_factory(self, component_type: Type) -> ComponentFactory:
+        """获取支持指定类型的工厂"""
+        with self._lock:
+            for factory in self._factories:
+                if factory.supports(component_type):
+                    return factory
+            return self._default_factory
+    
+    def create(self, name: str, **kwargs) -> Any:
+        """创建组件实例"""
+        spec = self.get_spec(name)
+        if not spec:
+            raise ValueError(f"Component spec '{name}' not found")
+        
+        factory = self.get_factory(spec.component_type)
+        return factory.create(spec, **kwargs)
+    
+    def get(self, name: str, **kwargs) -> Any:
+        """获取组件实例（create的别名）"""
+        return self.create(name, **kwargs)
+    
+    def list_components(self) -> List[str]:
+        """列出所有已注册的组件"""
+        with self._lock:
+            return list(self._specs.keys())
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._specs.clear()
+            self._factories.clear()
+            self._default_factory.clear_singletons()
+
+
+# 全局组件注册表
+_global_registry = ComponentRegistry()
+
+
+def get_component_registry() -> ComponentRegistry:
+    """获取全局组件注册表"""
     return _global_registry