crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/initialization/registry.py

@@ -1,144 +1,144 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-初始化器注册表 - 管理所有初始化器的注册和执行
-"""
-
-import threading
-from typing import Dict, Optional, Callable, List
-from .context import InitializationContext
-from .phases import InitializationPhase, PhaseResult
-
-
-class Initializer:
-    """初始化器基类"""
-    
-    def __init__(self, phase: InitializationPhase):
-        self._phase = phase
-    
-    @property
-    def phase(self) -> InitializationPhase:
-        """获取初始化阶段"""
-        return self._phase
-    
-    def initialize(self, context: InitializationContext) -> PhaseResult:
-        """执行初始化 - 子类必须实现"""
-        raise NotImplementedError("Subclasses must implement initialize method")
-
-
-class BaseInitializer(Initializer):
-    """基础初始化器类 - 为向后兼容保留"""
-    
-    def __init__(self, phase: InitializationPhase):
-        super().__init__(phase)
-    
-    def _create_result(self, success: bool, duration: float = 0.0, 
-                      artifacts: Optional[Dict] = None, error: Optional[Exception] = None) -> PhaseResult:
-        """创建初始化结果"""
-        from .utils import create_initialization_result
-        return create_initialization_result(
-            phase=self.phase,
-            success=success,
-            duration=duration,
-            artifacts=artifacts,
-            error=error
-        )
-
-
-class InitializerRegistry:
-    """
-    初始化器注册表 - 管理所有初始化器的注册和执行
-    
-    特点：
-    1. 线程安全的注册和执行
-    2. 支持函数式和类式初始化器
-    3. 统一的结果处理
-    """
-    
-    def __init__(self):
-        self._initializers: Dict[InitializationPhase, Initializer] = {}
-        self._lock = threading.RLock()
-    
-    def register(self, initializer: Initializer):
-        """注册初始化器"""
-        with self._lock:
-            phase = initializer.phase
-            if phase in self._initializers:
-                raise ValueError(f"Initializer for phase {phase} already registered")
-            self._initializers[phase] = initializer
-    
-    def register_function(self, phase: InitializationPhase, 
-                         init_func: Callable[[InitializationContext], PhaseResult]):
-        """注册函数式初始化器"""
-        
-        class FunctionInitializer(Initializer):
-            def __init__(self, phase: InitializationPhase, func: Callable):
-                super().__init__(phase)
-                self._phase = phase
-                self._func = func
-            
-            def initialize(self, context: InitializationContext) -> PhaseResult:
-                return self._func(context)
-        
-        self.register(FunctionInitializer(phase, init_func))
-    
-    def get_initializer(self, phase: InitializationPhase) -> Optional[Initializer]:
-        """获取指定阶段的初始化器"""
-        with self._lock:
-            return self._initializers.get(phase)
-    
-    def get_all_phases(self) -> List[InitializationPhase]:
-        """获取所有已注册的阶段"""
-        with self._lock:
-            return list(self._initializers.keys())
-    
-    def has_initializer(self, phase: InitializationPhase) -> bool:
-        """检查是否有指定阶段的初始化器"""
-        with self._lock:
-            return phase in self._initializers
-    
-    def clear(self):
-        """清空注册表"""
-        with self._lock:
-            self._initializers.clear()
-    
-    def execute_phase(self, phase: InitializationPhase, 
-                     context: InitializationContext) -> PhaseResult:
-        """执行指定阶段的初始化"""
-        initializer = self.get_initializer(phase)
-        if not initializer:
-            error = ValueError(f"No initializer registered for phase {phase}")
-            return PhaseResult(
-                phase=phase,
-                success=False,
-                error=error
-            )
-        
-        try:
-            return initializer.initialize(context)
-        except Exception as e:
-            return PhaseResult(
-                phase=phase,
-                success=False,
-                error=e
-            )
-
-
-# 全局注册表实例
-_global_registry = InitializerRegistry()
-
-
-def get_global_registry() -> InitializerRegistry:
-    """获取全局注册表"""
-    return _global_registry
-
-
-def register_initializer(initializer: Initializer):
-    """注册初始化器到全局注册表"""
-    _global_registry.register(initializer)
-
-
-def register_phase_function(phase: InitializationPhase, 
-                           init_func: Callable[[InitializationContext], PhaseResult]):
-    """注册函数式初始化器到全局注册表"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+初始化器注册表 - 管理所有初始化器的注册和执行
+"""
+
+import threading
+from typing import Dict, Optional, Callable, List
+from .context import InitializationContext
+from .phases import InitializationPhase, PhaseResult
+
+
+class Initializer:
+    """初始化器基类"""
+    
+    def __init__(self, phase: InitializationPhase):
+        self._phase = phase
+    
+    @property
+    def phase(self) -> InitializationPhase:
+        """获取初始化阶段"""
+        return self._phase
+    
+    def initialize(self, context: InitializationContext) -> PhaseResult:
+        """执行初始化 - 子类必须实现"""
+        raise NotImplementedError("Subclasses must implement initialize method")
+
+
+class BaseInitializer(Initializer):
+    """基础初始化器类 - 为向后兼容保留"""
+    
+    def __init__(self, phase: InitializationPhase):
+        super().__init__(phase)
+    
+    def _create_result(self, success: bool, duration: float = 0.0, 
+                      artifacts: Optional[Dict] = None, error: Optional[Exception] = None) -> PhaseResult:
+        """创建初始化结果"""
+        from .utils import create_initialization_result
+        return create_initialization_result(
+            phase=self.phase,
+            success=success,
+            duration=duration,
+            artifacts=artifacts,
+            error=error
+        )
+
+
+class InitializerRegistry:
+    """
+    初始化器注册表 - 管理所有初始化器的注册和执行
+    
+    特点：
+    1. 线程安全的注册和执行
+    2. 支持函数式和类式初始化器
+    3. 统一的结果处理
+    """
+    
+    def __init__(self):
+        self._initializers: Dict[InitializationPhase, Initializer] = {}
+        self._lock = threading.RLock()
+    
+    def register(self, initializer: Initializer):
+        """注册初始化器"""
+        with self._lock:
+            phase = initializer.phase
+            if phase in self._initializers:
+                raise ValueError(f"Initializer for phase {phase} already registered")
+            self._initializers[phase] = initializer
+    
+    def register_function(self, phase: InitializationPhase, 
+                         init_func: Callable[[InitializationContext], PhaseResult]):
+        """注册函数式初始化器"""
+        
+        class FunctionInitializer(Initializer):
+            def __init__(self, phase: InitializationPhase, func: Callable):
+                super().__init__(phase)
+                self._phase = phase
+                self._func = func
+            
+            def initialize(self, context: InitializationContext) -> PhaseResult:
+                return self._func(context)
+        
+        self.register(FunctionInitializer(phase, init_func))
+    
+    def get_initializer(self, phase: InitializationPhase) -> Optional[Initializer]:
+        """获取指定阶段的初始化器"""
+        with self._lock:
+            return self._initializers.get(phase)
+    
+    def get_all_phases(self) -> List[InitializationPhase]:
+        """获取所有已注册的阶段"""
+        with self._lock:
+            return list(self._initializers.keys())
+    
+    def has_initializer(self, phase: InitializationPhase) -> bool:
+        """检查是否有指定阶段的初始化器"""
+        with self._lock:
+            return phase in self._initializers
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._initializers.clear()
+    
+    def execute_phase(self, phase: InitializationPhase, 
+                     context: InitializationContext) -> PhaseResult:
+        """执行指定阶段的初始化"""
+        initializer = self.get_initializer(phase)
+        if not initializer:
+            error = ValueError(f"No initializer registered for phase {phase}")
+            return PhaseResult(
+                phase=phase,
+                success=False,
+                error=error
+            )
+        
+        try:
+            return initializer.initialize(context)
+        except Exception as e:
+            return PhaseResult(
+                phase=phase,
+                success=False,
+                error=e
+            )
+
+
+# 全局注册表实例
+_global_registry = InitializerRegistry()
+
+
+def get_global_registry() -> InitializerRegistry:
+    """获取全局注册表"""
+    return _global_registry
+
+
+def register_initializer(initializer: Initializer):
+    """注册初始化器到全局注册表"""
+    _global_registry.register(initializer)
+
+
+def register_phase_function(phase: InitializationPhase, 
+                           init_func: Callable[[InitializationContext], PhaseResult]):
+    """注册函数式初始化器到全局注册表"""
     _global_registry.register_function(phase, init_func)

crawlo/initialization/utils.py

@@ -1,49 +1,49 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-初始化工具模块 - 提供通用的初始化工具函数
-"""
-
-import time
-from typing import Optional, Dict, Any
-from .phases import PhaseResult, InitializationPhase
-
-
-def create_initialization_result(
-    phase: 'InitializationPhase',
-    success: bool, 
-    duration: float = 0.0,
-    artifacts: Optional[Dict[str, Any]] = None, 
-    error: Optional[Exception] = None
-) -> PhaseResult:
-    """
-    创建标准化的初始化结果
-    
-    Args:
-        phase: 初始化阶段
-        success: 是否成功
-        duration: 执行时长
-        artifacts: 产生的工件数据
-        error: 异常对象
-        
-    Returns:
-        PhaseResult: 标准化的初始化结果
-    """
-    return PhaseResult(
-        phase=phase,
-        success=success,
-        duration=duration,
-        artifacts=artifacts or {},
-        error=error
-    )
-
-
-class InitializationTimer:
-    """初始化计时器"""
-    
-    def __init__(self):
-        self.start_time = time.time()
-    
-    def get_duration(self) -> float:
-        """获取经过的时间"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+初始化工具模块 - 提供通用的初始化工具函数
+"""
+
+import time
+from typing import Optional, Dict, Any
+from .phases import PhaseResult, InitializationPhase
+
+
+def create_initialization_result(
+    phase: 'InitializationPhase',
+    success: bool, 
+    duration: float = 0.0,
+    artifacts: Optional[Dict[str, Any]] = None, 
+    error: Optional[Exception] = None
+) -> PhaseResult:
+    """
+    创建标准化的初始化结果
+    
+    Args:
+        phase: 初始化阶段
+        success: 是否成功
+        duration: 执行时长
+        artifacts: 产生的工件数据
+        error: 异常对象
+        
+    Returns:
+        PhaseResult: 标准化的初始化结果
+    """
+    return PhaseResult(
+        phase=phase,
+        success=success,
+        duration=duration,
+        artifacts=artifacts or {},
+        error=error
+    )
+
+
+class InitializationTimer:
+    """初始化计时器"""
+    
+    def __init__(self):
+        self.start_time = time.time()
+    
+    def get_duration(self) -> float:
+        """获取经过的时间"""
         return time.time() - self.start_time

crawlo/interfaces.py

@@ -1,24 +1,24 @@
-from abc import ABC, abstractmethod
-from typing import List, Type, Protocol
-
-from crawlo.spider import Spider
-from crawlo.network.request import Request
-
-
-class ISpiderLoader(Protocol):
-    """Spider loader interface"""
-    
-    @abstractmethod
-    def load(self, spider_name: str) -> Type[Spider]:
-        """Load a spider by name"""
-        pass
-    
-    @abstractmethod
-    def list(self) -> List[str]:
-        """List all available spider names"""
-        pass
-    
-    @abstractmethod
-    def find_by_request(self, request: Request) -> List[str]:
-        """Find spider names that can handle the given request"""
+from abc import ABC, abstractmethod
+from typing import List, Type, Protocol
+
+from crawlo.spider import Spider
+from crawlo.network.request import Request
+
+
+class ISpiderLoader(Protocol):
+    """Spider loader interface"""
+    
+    @abstractmethod
+    def load(self, spider_name: str) -> Type[Spider]:
+        """Load a spider by name"""
+        pass
+    
+    @abstractmethod
+    def list(self) -> List[str]:
+        """List all available spider names"""
+        pass
+    
+    @abstractmethod
+    def find_by_request(self, request: Request) -> List[str]:
+        """Find spider names that can handle the given request"""
         pass

crawlo/items/__init__.py

@@ -1,23 +1,23 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-crawlo.items 包
-===============
-提供 Item 和 Field 类用于数据定义和验证。
-"""
-from .items import Item
-from .fields import Field
-from .base import ItemMeta
-
-from crawlo.exceptions import ItemInitError, ItemAttributeError
-
-__all__ = [
-    'Item',
-    'Field',
-    'ItemMeta',
-    'ItemInitError',
-    'ItemAttributeError'
-]
-
-
-
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+crawlo.items 包
+===============
+提供 Item 和 Field 类用于数据定义和验证。
+"""
+from .items import Item
+from .fields import Field
+from .base import ItemMeta
+
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+__all__ = [
+    'Item',
+    'Field',
+    'ItemMeta',
+    'ItemInitError',
+    'ItemAttributeError'
+]
+
+
+

crawlo/items/base.py

@@ -1,23 +1,23 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-基础元类定义
-"""
-from abc import ABCMeta
-from .fields import Field
-
-
-class ItemMeta(ABCMeta):
-    def __new__(mcs, name, bases, attrs):
-        fields = {}
-        cls_attrs = {}
-
-        for attr_name, attr_value in attrs.items():
-            if isinstance(attr_value, Field):
-                fields[attr_name] = attr_value
-            else:
-                cls_attrs[attr_name] = attr_value
-
-        cls_instance = super().__new__(mcs, name, bases, cls_attrs)
-        cls_instance.FIELDS = fields
-        return cls_instance
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+基础元类定义
+"""
+from abc import ABCMeta
+from .fields import Field
+
+
+class ItemMeta(ABCMeta):
+    def __new__(mcs, name, bases, attrs):
+        fields = {}
+        cls_attrs = {}
+
+        for attr_name, attr_value in attrs.items():
+            if isinstance(attr_value, Field):
+                fields[attr_name] = attr_value
+            else:
+                cls_attrs[attr_name] = attr_value
+
+        cls_instance = super().__new__(mcs, name, bases, cls_attrs)
+        cls_instance.FIELDS = fields
+        return cls_instance

crawlo/items/fields.py

@@ -1,53 +1,53 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Field 类定义
-"""
-from typing import Any, Optional, Type
-
-
-class Field:
-    """
-    字段定义类，用于定义 Item 的字段属性和验证规则
-    """
-    def __init__(
-        self,
-        nullable: bool = True,
-        *,
-        default: Any = None,
-        field_type: Optional[Type] = None,
-        max_length: Optional[int] = None,
-        description: str = ""
-    ):
-        self.nullable = nullable
-        self.default = default
-        self.field_type = field_type
-        self.max_length = max_length
-        self.description = description
-
-    def validate(self, value: Any, field_name: str = "") -> Any:
-        """
-        验证字段值是否符合规则
-        """
-        if value is None or (isinstance(value, str) and value.strip() == ""):
-            if self.default is not None:
-                return self.default
-            elif not self.nullable:
-                raise ValueError(
-                    f"字段 '{field_name}' 不允许为空。"
-                )
-
-        if value is not None and not (isinstance(value, str) and value.strip() == ""):
-            if self.field_type and not isinstance(value, self.field_type):
-                raise TypeError(
-                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
-                )
-            if self.max_length and len(str(value)) > self.max_length:
-                raise ValueError(
-                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
-                )
-
-        return value
-
-    def __repr__(self):
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Field 类定义
+"""
+from typing import Any, Optional, Type
+
+
+class Field:
+    """
+    字段定义类，用于定义 Item 的字段属性和验证规则
+    """
+    def __init__(
+        self,
+        nullable: bool = True,
+        *,
+        default: Any = None,
+        field_type: Optional[Type] = None,
+        max_length: Optional[int] = None,
+        description: str = ""
+    ):
+        self.nullable = nullable
+        self.default = default
+        self.field_type = field_type
+        self.max_length = max_length
+        self.description = description
+
+    def validate(self, value: Any, field_name: str = "") -> Any:
+        """
+        验证字段值是否符合规则
+        """
+        if value is None or (isinstance(value, str) and value.strip() == ""):
+            if self.default is not None:
+                return self.default
+            elif not self.nullable:
+                raise ValueError(
+                    f"字段 '{field_name}' 不允许为空。"
+                )
+
+        if value is not None and not (isinstance(value, str) and value.strip() == ""):
+            if self.field_type and not isinstance(value, self.field_type):
+                raise TypeError(
+                    f"字段 '{field_name}' 类型错误：期望类型 {self.field_type}, 得到 {type(value)}，值：{value!r}"
+                )
+            if self.max_length and len(str(value)) > self.max_length:
+                raise ValueError(
+                    f"字段 '{field_name}' 长度超限：最大长度 {self.max_length}，当前长度 {len(str(value))}，值：{value!r}"
+                )
+
+        return value
+
+    def __repr__(self):
         return f"<Field nullable={self.nullable} type={self.field_type} default={self.default}>"