crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/extension/memory_monitor.py

@@ -4,9 +4,9 @@ import asyncio
 import psutil
 from typing import Any, Optional
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils.error_handler import ErrorHandler
-from crawlo.event import spider_opened, spider_closed
+from crawlo.event import CrawlerEvent
 
 
 class MemoryMonitorExtension:
@@ -19,7 +19,7 @@ class MemoryMonitorExtension:
         self.task: Optional[asyncio.Task] = None
         self.process = psutil.Process()
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
         
         # 获取配置参数
@@ -35,8 +35,8 @@ class MemoryMonitorExtension:
             raise NotConfigured("MemoryMonitorExtension: MEMORY_MONITOR_ENABLED is False")
         
         o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+        crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
         return o
 
     async def spider_opened(self) -> None:

crawlo/extension/performance_profiler.py

@@ -7,9 +7,9 @@ import asyncio
 import cProfile
 from typing import Any, Optional
 
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils.error_handler import ErrorHandler
-from crawlo.event import spider_opened, spider_closed
+from crawlo.event import CrawlerEvent
 
 
 class PerformanceProfilerExtension:
@@ -20,7 +20,7 @@ class PerformanceProfilerExtension:
 
     def __init__(self, crawler: Any):
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
         
         # 获取配置参数
@@ -44,8 +44,8 @@ class PerformanceProfilerExtension:
         
         o = cls(crawler)
         if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
         return o
 
     async def spider_opened(self) -> None:

crawlo/extension/request_recorder.py

@@ -5,8 +5,8 @@ import json
 from typing import Any
 from datetime import datetime
 
-from crawlo import event
-from crawlo.utils.log import get_logger
+from crawlo.event import CrawlerEvent
+from crawlo.logging import get_logger
 
 
 class RequestRecorderExtension:
@@ -17,7 +17,7 @@ class RequestRecorderExtension:
 
     def __init__(self, crawler: Any):
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         
         # 获取配置参数
         self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
@@ -40,9 +40,9 @@ class RequestRecorderExtension:
         
         o = cls(crawler)
         if o.enabled:
-            crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
-            crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-            crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
+            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
+            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
         return o
 
     async def request_scheduled(self, request: Any, spider: Any) -> None:

crawlo/factories/base.py

@@ -6,7 +6,7 @@
 
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import Type, Any, Dict, Optional, Callable
+from typing import Type, Any, Dict, Callable
 
 
 @dataclass

crawlo/factories/crawler.py

@@ -31,73 +31,74 @@ class CrawlerComponentFactory(ComponentFactory):
         return component_type.__name__ in supported_types
 
 
+# Engine组件
+def create_engine(crawler, **kwargs):
+    from crawlo.core.engine import Engine
+    return Engine(crawler)
+
+# Scheduler组件
+def create_scheduler(crawler, **kwargs):
+    from crawlo.core.scheduler import Scheduler
+    return Scheduler.create_instance(crawler)
+
+# StatsCollector组件
+def create_stats(crawler, **kwargs):
+    from crawlo.stats_collector import StatsCollector
+    return StatsCollector(crawler)
+
+# Subscriber组件
+def create_subscriber(**kwargs):
+    from crawlo.subscriber import Subscriber
+    return Subscriber()
+
+# ExtensionManager组件
+def create_extension_manager(crawler, **kwargs):
+    from crawlo.extension import ExtensionManager
+    return ExtensionManager.create_instance(crawler)
+
 def register_crawler_components():
     """注册Crawler相关组件"""
-    registry = get_component_registry()
+    from .utils import register_components
     
     # 注册工厂
+    registry = get_component_registry()
     registry.register_factory(CrawlerComponentFactory())
     
-    # 注册组件规范
-    
-    # Engine组件
-    def create_engine(crawler, **kwargs):
-        from crawlo.core.engine import Engine
-        return Engine(crawler)
-    
-    registry.register(ComponentSpec(
-        name='engine',
-        component_type=type('Engine', (), {}),
-        factory_func=create_engine,
-        dependencies=['crawler']
-    ))
-    
-    # Scheduler组件
-    def create_scheduler(crawler, **kwargs):
-        from crawlo.core.scheduler import Scheduler
-        return Scheduler.create_instance(crawler)
-    
-    registry.register(ComponentSpec(
-        name='scheduler',
-        component_type=type('Scheduler', (), {}),
-        factory_func=create_scheduler,
-        dependencies=['crawler']
-    ))
-    
-    # StatsCollector组件
-    def create_stats(crawler, **kwargs):
-        from crawlo.stats_collector import StatsCollector
-        return StatsCollector(crawler)
-    
-    registry.register(ComponentSpec(
-        name='stats',
-        component_type=type('StatsCollector', (), {}),
-        factory_func=create_stats,
-        dependencies=['crawler']
-    ))
-    
-    # Subscriber组件
-    def create_subscriber(**kwargs):
-        from crawlo.subscriber import Subscriber
-        return Subscriber()
-    
-    registry.register(ComponentSpec(
-        name='subscriber',
-        component_type=type('Subscriber', (), {}),
-        factory_func=create_subscriber
-    ))
-    
-    # ExtensionManager组件
-    def create_extension_manager(crawler, **kwargs):
-        from crawlo.extension import ExtensionManager
-        return ExtensionManager.create_instance(crawler)
+    # 批量注册组件
+    component_list = [
+        {
+            'name': 'engine',
+            'component_type': 'Engine',
+            'factory_func': create_engine,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'scheduler',
+            'component_type': 'Scheduler',
+            'factory_func': create_scheduler,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'stats',
+            'component_type': 'StatsCollector',
+            'factory_func': create_stats,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'subscriber',
+            'component_type': 'Subscriber',
+            'factory_func': create_subscriber,
+            'dependencies': []
+        },
+        {
+            'name': 'extension_manager',
+            'component_type': 'ExtensionManager',
+            'factory_func': create_extension_manager,
+            'dependencies': ['crawler']
+        }
+    ]
     
-    registry.register(ComponentSpec(
-        name='extension_manager',
-        component_type=type('ExtensionManager', (), {}),
-        factory_func=create_extension_manager,
-        dependencies=['crawler']
-    ))
+    register_components(component_list)
 
 
 # 自动注册

crawlo/factories/utils.py

@@ -0,0 +1,135 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+工厂工具模块 - 提供通用的组件注册和创建工具
+"""
+
+from typing import Any, Callable, List, Optional, Type, Union
+from .base import ComponentSpec
+from .registry import get_component_registry
+
+
+def register_component(
+    name: str,
+    component_type: Union[Type, str],
+    factory_func: Callable[..., Any],
+    dependencies: Optional[List[str]] = None,
+    singleton: bool = False,
+    config_key: Optional[str] = None
+) -> None:
+    """
+    注册组件的便捷函数
+    
+    Args:
+        name: 组件名称
+        component_type: 组件类型
+        factory_func: 工厂函数
+        dependencies: 依赖列表
+        singleton: 是否单例
+        config_key: 配置键名
+    """
+    registry = get_component_registry()
+    
+    # 如果component_type是字符串，创建一个动态类型
+    if isinstance(component_type, str):
+        component_type = type(component_type, (), {})
+    
+    spec_kwargs = {
+        'name': name,
+        'component_type': component_type,
+        'factory_func': factory_func,
+        'dependencies': dependencies or [],
+        'singleton': singleton
+    }
+    
+    # 只有当config_key不为None时才添加
+    if config_key is not None:
+        spec_kwargs['config_key'] = config_key
+    
+    spec = ComponentSpec(**spec_kwargs)
+    
+    registry.register(spec)
+
+
+def register_components(component_list: List[dict]) -> None:
+    """
+    批量注册组件
+    
+    Args:
+        component_list: 组件定义列表，每个元素是一个包含组件信息的字典
+    """
+    for component_info in component_list:
+        register_component(**component_info)
+
+
+def create_component_factory(
+    component_name: str,
+    module_path: str,
+    class_name: str,
+    dependencies: Optional[List[str]] = None,
+    singleton: bool = False
+) -> Callable[..., Any]:
+    """
+    创建组件工厂函数的便捷函数
+    
+    Args:
+        component_name: 组件名称（用于错误信息）
+        module_path: 模块路径
+        class_name: 类名
+        dependencies: 依赖列表
+        singleton: 是否单例
+        
+    Returns:
+        工厂函数
+    """
+    def factory_func(*args, **kwargs):
+        try:
+            # 动态导入模块
+            module = __import__(module_path, fromlist=[class_name])
+            component_class = getattr(module, class_name)
+            
+            # 检查是否需要调用create_instance方法
+            if hasattr(component_class, 'create_instance'):
+                return component_class.create_instance(*args, **kwargs)
+            else:
+                return component_class(*args, **kwargs)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create {component_name}: {e}")
+    
+    return factory_func
+
+
+def create_crawler_component_factory(
+    component_name: str,
+    module_path: str,
+    class_name: str
+) -> Callable[..., Any]:
+    """
+    创建需要crawler依赖的组件工厂函数
+    
+    Args:
+        component_name: 组件名称
+        module_path: 模块路径
+        class_name: 类名
+        
+    Returns:
+        工厂函数
+    """
+    def factory_func(crawler=None, **kwargs):
+        if crawler is None:
+            raise ValueError(f"Crawler instance required for component {component_name}")
+        
+        try:
+            # 动态导入模块
+            module = __import__(module_path, fromlist=[class_name])
+            component_class = getattr(module, class_name)
+            
+            # 检查是否需要调用create_instance方法
+            if hasattr(component_class, 'create_instance'):
+                return component_class.create_instance(crawler, **kwargs)
+            else:
+                return component_class(crawler, **kwargs)
+        except Exception as e:
+            raise RuntimeError(f"Failed to create {component_name}: {e}")
+    
+    return factory_func

crawlo/filters/__init__.py

@@ -17,7 +17,7 @@ Crawlo过滤器模块
 from abc import ABC, abstractmethod
 from typing import Optional
 
-from crawlo.utils.request import request_fingerprint
+from crawlo.utils.fingerprint import FingerprintGenerator
 
 
 class BaseFilter(ABC):
@@ -46,6 +46,23 @@ class BaseFilter(ABC):
     def create_instance(cls, *args, **kwargs) -> 'BaseFilter':
         return cls(*args, **kwargs)
 
+    def _get_fingerprint(self, request) -> str:
+        """
+        获取请求指纹（内部辅助方法）
+        
+        使用统一的 FingerprintGenerator 生成请求指纹。
+        子类可以直接调用此方法，避免重复实现。
+        
+        :param request: 请求对象
+        :return: 请求指纹字符串
+        """
+        return FingerprintGenerator.request_fingerprint(
+            request.method,
+            request.url,
+            request.body or b'',
+            dict(request.headers) if hasattr(request, 'headers') else {}
+        )
+
     def requested(self, request) -> bool:
         """
         检查请求是否重复（主要接口）
@@ -54,7 +71,7 @@ class BaseFilter(ABC):
         :return: True 表示重复，False 表示新请求
         """
         self._request_count += 1
-        fp = request_fingerprint(request)
+        fp = self._get_fingerprint(request)
         
         if fp in self:
             self._duplicate_count += 1