crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/extension/request_recorder.py

@@ -1,108 +1,108 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import os
-import json
-from typing import Any
-from datetime import datetime
-
-from crawlo.event import CrawlerEvent
-from crawlo.logging import get_logger
-
-
-class RequestRecorderExtension:
-    """
-    请求记录扩展
-    记录所有发送的请求信息到文件，便于调试和分析
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__)
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
-        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
-        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
-        
-        # 创建输出目录
-        if self.enabled:
-            os.makedirs(self.output_dir, exist_ok=True)
-            
-        self.current_file = None
-        self.current_file_size = 0
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
-            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
-            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
-        return o
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-            
-        try:
-            request_info = {
-                'timestamp': datetime.now().isoformat(),
-                'type': 'request',
-                'url': request.url,
-                'method': request.method,
-                'headers': dict(request.headers),
-                'meta': getattr(request, 'meta', {}),
-            }
-            
-            await self._write_record(request_info)
-        except Exception as e:
-            self.logger.error(f"Error recording request: {e}")
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        try:
-            response_info = {
-                'timestamp': datetime.now().isoformat(),
-                'type': 'response',
-                'url': response.url,
-                'status_code': response.status_code,
-                'headers': dict(response.headers),
-            }
-            
-            await self._write_record(response_info)
-        except Exception as e:
-            self.logger.error(f"Error recording response: {e}")
-
-    async def spider_closed(self, spider: Any) -> None:
-        """爬虫关闭时清理资源"""
-        if self.current_file:
-            self.current_file.close()
-            self.current_file = None
-        self.logger.info("Request recorder closed.")
-
-    async def _write_record(self, record: dict) -> None:
-        """写入记录到文件"""
-        # 检查是否需要创建新文件
-        if not self.current_file or self.current_file_size > self.max_file_size:
-            if self.current_file:
-                self.current_file.close()
-            
-            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
-            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
-            self.current_file = open(filename, 'a', encoding='utf-8')
-            self.current_file_size = 0
-        
-        # 写入记录
-        line = json.dumps(record, ensure_ascii=False) + '\n'
-        self.current_file.write(line)
-        self.current_file.flush()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import json
+from typing import Any
+from datetime import datetime
+
+from crawlo.event import CrawlerEvent
+from crawlo.logging import get_logger
+
+
+class RequestRecorderExtension:
+    """
+    请求记录扩展
+    记录所有发送的请求信息到文件，便于调试和分析
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__)
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
+        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
+        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+            
+        self.current_file = None
+        self.current_file_size = 0
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
+            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
+            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
+        return o
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+            
+        try:
+            request_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'request',
+                'url': request.url,
+                'method': request.method,
+                'headers': dict(request.headers),
+                'meta': getattr(request, 'meta', {}),
+            }
+            
+            await self._write_record(request_info)
+        except Exception as e:
+            self.logger.error(f"Error recording request: {e}")
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        try:
+            response_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'response',
+                'url': response.url,
+                'status_code': response.status_code,
+                'headers': dict(response.headers),
+            }
+            
+            await self._write_record(response_info)
+        except Exception as e:
+            self.logger.error(f"Error recording response: {e}")
+
+    async def spider_closed(self, spider: Any) -> None:
+        """爬虫关闭时清理资源"""
+        if self.current_file:
+            self.current_file.close()
+            self.current_file = None
+        self.logger.info("Request recorder closed.")
+
+    async def _write_record(self, record: dict) -> None:
+        """写入记录到文件"""
+        # 检查是否需要创建新文件
+        if not self.current_file or self.current_file_size > self.max_file_size:
+            if self.current_file:
+                self.current_file.close()
+            
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
+            self.current_file = open(filename, 'a', encoding='utf-8')
+            self.current_file_size = 0
+        
+        # 写入记录
+        line = json.dumps(record, ensure_ascii=False) + '\n'
+        self.current_file.write(line)
+        self.current_file.flush()
         self.current_file_size += len(line.encode('utf-8'))

crawlo/factories/__init__.py

@@ -1,28 +1,28 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo组件工厂系统
-==================
-
-提供统一的组件创建和依赖注入机制
-"""
-
-from .registry import ComponentRegistry, get_component_registry
-from .base import ComponentFactory, ComponentSpec
-from .crawler import CrawlerComponentFactory
-
-# 公共接口
-register_component = get_component_registry().register
-get_component = get_component_registry().get
-create_component = get_component_registry().create
-
-__all__ = [
-    'ComponentRegistry',
-    'ComponentFactory', 
-    'ComponentSpec',
-    'CrawlerComponentFactory',
-    'get_component_registry',
-    'register_component',
-    'get_component',
-    'create_component'
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo组件工厂系统
+==================
+
+提供统一的组件创建和依赖注入机制
+"""
+
+from .registry import ComponentRegistry, get_component_registry
+from .base import ComponentFactory, ComponentSpec
+from .crawler import CrawlerComponentFactory
+
+# 公共接口
+register_component = get_component_registry().register
+get_component = get_component_registry().get
+create_component = get_component_registry().create
+
+__all__ = [
+    'ComponentRegistry',
+    'ComponentFactory', 
+    'ComponentSpec',
+    'CrawlerComponentFactory',
+    'get_component_registry',
+    'register_component',
+    'get_component',
+    'create_component'
 ]

crawlo/factories/base.py

@@ -1,69 +1,69 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-组件工厂基类和规范
-"""
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Type, Any, Dict, Callable
-
-
-@dataclass
-class ComponentSpec:
-    """组件规范 - 定义如何创建组件"""
-    
-    name: str
-    component_type: Type
-    factory_func: Callable[..., Any]
-    dependencies: list = None
-    singleton: bool = False
-    config_key: str = None
-    
-    def __post_init__(self):
-        if self.dependencies is None:
-            self.dependencies = []
-
-
-class ComponentFactory(ABC):
-    """组件工厂基类"""
-    
-    @abstractmethod
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建组件实例"""
-        pass
-    
-    @abstractmethod
-    def supports(self, component_type: Type) -> bool:
-        """检查是否支持指定类型的组件"""
-        pass
-
-
-class DefaultComponentFactory(ComponentFactory):
-    """默认组件工厂实现"""
-    
-    def __init__(self):
-        self._instances: Dict[str, Any] = {}
-    
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建组件实例"""
-        # 单例模式检查
-        if spec.singleton and spec.name in self._instances:
-            return self._instances[spec.name]
-        
-        # 调用工厂函数创建实例
-        instance = spec.factory_func(**kwargs)
-        
-        # 保存单例实例
-        if spec.singleton:
-            self._instances[spec.name] = instance
-        
-        return instance
-    
-    def supports(self, component_type: Type) -> bool:
-        """支持所有类型"""
-        return True
-    
-    def clear_singletons(self):
-        """清除单例实例（测试用）"""
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件工厂基类和规范
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Type, Any, Dict, Callable
+
+
+@dataclass
+class ComponentSpec:
+    """组件规范 - 定义如何创建组件"""
+    
+    name: str
+    component_type: Type
+    factory_func: Callable[..., Any]
+    dependencies: list = None
+    singleton: bool = False
+    config_key: str = None
+    
+    def __post_init__(self):
+        if self.dependencies is None:
+            self.dependencies = []
+
+
+class ComponentFactory(ABC):
+    """组件工厂基类"""
+    
+    @abstractmethod
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        pass
+    
+    @abstractmethod
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型的组件"""
+        pass
+
+
+class DefaultComponentFactory(ComponentFactory):
+    """默认组件工厂实现"""
+    
+    def __init__(self):
+        self._instances: Dict[str, Any] = {}
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        # 单例模式检查
+        if spec.singleton and spec.name in self._instances:
+            return self._instances[spec.name]
+        
+        # 调用工厂函数创建实例
+        instance = spec.factory_func(**kwargs)
+        
+        # 保存单例实例
+        if spec.singleton:
+            self._instances[spec.name] = instance
+        
+        return instance
+    
+    def supports(self, component_type: Type) -> bool:
+        """支持所有类型"""
+        return True
+    
+    def clear_singletons(self):
+        """清除单例实例（测试用）"""
         self._instances.clear()

crawlo/factories/crawler.py

@@ -1,105 +1,105 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawler组件工厂 - 专门用于创建Crawler相关组件
-"""
-
-from typing import Any, Type
-
-from .base import ComponentFactory, ComponentSpec
-from .registry import get_component_registry
-
-
-class CrawlerComponentFactory(ComponentFactory):
-    """Crawler组件工厂"""
-    
-    def create(self, spec: ComponentSpec, **kwargs) -> Any:
-        """创建Crawler相关组件"""
-        # 检查是否需要crawler依赖
-        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
-            raise ValueError(f"Crawler instance required for component {spec.name}")
-        
-        return spec.factory_func(**kwargs)
-    
-    def supports(self, component_type: Type) -> bool:
-        """检查是否支持指定类型"""
-        # 这里可以根据需要定义支持的组件类型
-        supported_types = [
-            'Engine', 'Scheduler', 'StatsCollector', 
-            'Subscriber', 'ExtensionManager'
-        ]
-        return component_type.__name__ in supported_types
-
-
-# Engine组件
-def create_engine(crawler, **kwargs):
-    from crawlo.core.engine import Engine
-    return Engine(crawler)
-
-# Scheduler组件
-def create_scheduler(crawler, **kwargs):
-    from crawlo.core.scheduler import Scheduler
-    return Scheduler.create_instance(crawler)
-
-# StatsCollector组件
-def create_stats(crawler, **kwargs):
-    from crawlo.stats_collector import StatsCollector
-    return StatsCollector(crawler)
-
-# Subscriber组件
-def create_subscriber(**kwargs):
-    from crawlo.subscriber import Subscriber
-    return Subscriber()
-
-# ExtensionManager组件
-def create_extension_manager(crawler, **kwargs):
-    from crawlo.extension import ExtensionManager
-    return ExtensionManager.create_instance(crawler)
-
-def register_crawler_components():
-    """注册Crawler相关组件"""
-    from .utils import register_components
-    
-    # 注册工厂
-    registry = get_component_registry()
-    registry.register_factory(CrawlerComponentFactory())
-    
-    # 批量注册组件
-    component_list = [
-        {
-            'name': 'engine',
-            'component_type': 'Engine',
-            'factory_func': create_engine,
-            'dependencies': ['crawler']
-        },
-        {
-            'name': 'scheduler',
-            'component_type': 'Scheduler',
-            'factory_func': create_scheduler,
-            'dependencies': ['crawler']
-        },
-        {
-            'name': 'stats',
-            'component_type': 'StatsCollector',
-            'factory_func': create_stats,
-            'dependencies': ['crawler']
-        },
-        {
-            'name': 'subscriber',
-            'component_type': 'Subscriber',
-            'factory_func': create_subscriber,
-            'dependencies': []
-        },
-        {
-            'name': 'extension_manager',
-            'component_type': 'ExtensionManager',
-            'factory_func': create_extension_manager,
-            'dependencies': ['crawler']
-        }
-    ]
-    
-    register_components(component_list)
-
-
-# 自动注册
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawler组件工厂 - 专门用于创建Crawler相关组件
+"""
+
+from typing import Any, Type
+
+from .base import ComponentFactory, ComponentSpec
+from .registry import get_component_registry
+
+
+class CrawlerComponentFactory(ComponentFactory):
+    """Crawler组件工厂"""
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建Crawler相关组件"""
+        # 检查是否需要crawler依赖
+        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
+            raise ValueError(f"Crawler instance required for component {spec.name}")
+        
+        return spec.factory_func(**kwargs)
+    
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型"""
+        # 这里可以根据需要定义支持的组件类型
+        supported_types = [
+            'Engine', 'Scheduler', 'StatsCollector', 
+            'Subscriber', 'ExtensionManager'
+        ]
+        return component_type.__name__ in supported_types
+
+
+# Engine组件
+def create_engine(crawler, **kwargs):
+    from crawlo.core.engine import Engine
+    return Engine(crawler)
+
+# Scheduler组件
+def create_scheduler(crawler, **kwargs):
+    from crawlo.core.scheduler import Scheduler
+    return Scheduler.create_instance(crawler)
+
+# StatsCollector组件
+def create_stats(crawler, **kwargs):
+    from crawlo.stats_collector import StatsCollector
+    return StatsCollector(crawler)
+
+# Subscriber组件
+def create_subscriber(**kwargs):
+    from crawlo.subscriber import Subscriber
+    return Subscriber()
+
+# ExtensionManager组件
+def create_extension_manager(crawler, **kwargs):
+    from crawlo.extension import ExtensionManager
+    return ExtensionManager.create_instance(crawler)
+
+def register_crawler_components():
+    """注册Crawler相关组件"""
+    from .utils import register_components
+    
+    # 注册工厂
+    registry = get_component_registry()
+    registry.register_factory(CrawlerComponentFactory())
+    
+    # 批量注册组件
+    component_list = [
+        {
+            'name': 'engine',
+            'component_type': 'Engine',
+            'factory_func': create_engine,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'scheduler',
+            'component_type': 'Scheduler',
+            'factory_func': create_scheduler,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'stats',
+            'component_type': 'StatsCollector',
+            'factory_func': create_stats,
+            'dependencies': ['crawler']
+        },
+        {
+            'name': 'subscriber',
+            'component_type': 'Subscriber',
+            'factory_func': create_subscriber,
+            'dependencies': []
+        },
+        {
+            'name': 'extension_manager',
+            'component_type': 'ExtensionManager',
+            'factory_func': create_extension_manager,
+            'dependencies': ['crawler']
+        }
+    ]
+    
+    register_components(component_list)
+
+
+# 自动注册
 register_crawler_components()