crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/extension/log_interval.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+from typing import Any, Optional
 
 from crawlo.utils.log import get_logger
 from crawlo.event import spider_opened, spider_closed
@@ -8,12 +9,12 @@ from crawlo.event import spider_opened, spider_closed
 
 class LogIntervalExtension(object):
 
-    def __init__(self, crawler):
-        self.task = None
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
         self.stats = crawler.stats
         self.item_count = 0
         self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL')
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
         self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
         self.interval = "" if self.interval == 1 else self.interval
         self.unit = 'min' if self.seconds % 60 == 0 else 's'
@@ -21,29 +22,37 @@ class LogIntervalExtension(object):
         self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
 
     @classmethod
-    def create_instance(cls, crawler):
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
         o = cls(crawler)
         crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
         crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
         return o
 
-    async def spider_opened(self):
+    async def spider_opened(self) -> None:
         self.task = asyncio.create_task(self.interval_log())
-        await self.task
 
-    async def spider_closed(self):
+    async def spider_closed(self) -> None:
         if self.task:
             self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
 
-    async def interval_log(self):
+    async def interval_log(self) -> None:
         while True:
-            last_item_count = self.stats.get_value('item_successful_count', default=0)
-            last_response_count = self.stats.get_value('response_received_count', default=0)
-            item_rate = last_item_count - self.item_count
-            response_rate = last_response_count - self.response_count
-            self.item_count, self.response_count = last_item_count, last_response_count
-            self.logger.info(
-                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-            )
-            await asyncio.sleep(self.seconds)
+            try:
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                self.item_count, self.response_count = last_item_count, last_response_count
+                self.logger.info(
+                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+                )
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
+                await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,44 +1,82 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+from typing import Any
+
 from crawlo import event
 from crawlo.utils.date_tools import now, time_diff
 
 
 class LogStats(object):
 
-    def __init__(self, stats):
+    def __init__(self, stats: Any):
         self._stats = stats
 
     @classmethod
-    def create_instance(cls, crawler):
+    def create_instance(cls, crawler: Any) -> 'LogStats':
         o = cls(crawler.stats)
-        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
-        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
-        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+        # 订阅所有需要的事件
+        event_subscriptions = [
+            (o.spider_opened, event.spider_opened),
+            (o.spider_closed, event.spider_closed),
+            (o.item_successful, event.item_successful),
+            (o.item_discard, event.item_discard),
+            (o.response_received, event.response_received),
+            (o.request_scheduled, event.request_scheduled),
+        ]
+        
+        for handler, evt in event_subscriptions:
+            try:
+                crawler.subscriber.subscribe(handler, event=evt)
+            except Exception as e:
+                # 获取日志记录器并记录错误
+                from crawlo.utils.log import get_logger
+                logger = get_logger(cls.__name__)
+                logger.error(f"Failed to subscribe to event {evt}: {e}")
 
         return o
 
-    async def spider_opened(self):
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+    async def spider_opened(self) -> None:
+        try:
+            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def spider_closed(self):
-        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+    async def spider_closed(self) -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def item_successful(self, _item, _spider):
-        self._stats.inc_value('item_successful_count')
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def item_discard(self, _item, exc, _spider):
-        self._stats.inc_value('item_discard_count')
-        reason = exc.msg
-        if reason:
-            self._stats.inc_value(f"item_discard/{reason}")
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_discard_count')
+            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
+            if reason:
+                self._stats.inc_value(f"item_discard/{reason}")
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def response_received(self, _response, _spider):
-        self._stats.inc_value('response_received_count')
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def request_scheduled(self, _request, _spider):
-        self._stats.inc_value('request_scheduler_count')
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass

crawlo/extension/logging_extension.py

@@ -1,3 +1,4 @@
+from typing import Any
 from crawlo.exceptions import NotConfigured
 from crawlo.utils.log import get_logger
 from crawlo.utils.log import LoggerManager
@@ -9,27 +10,35 @@ class CustomLoggerExtension:
     遵循与 ExtensionManager 一致的接口规范：使用 create_instance
     """
 
-    def __init__(self, settings):
+    def __init__(self, settings: Any):
         self.settings = settings
         # 初始化全局日志配置
         LoggerManager.configure(settings)
 
     @classmethod
-    def create_instance(cls, crawler, *args, **kwargs):
+    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
         """
         工厂方法：兼容 ExtensionManager 的创建方式
         被 ExtensionManager 调用
         """
         # 可以通过 settings 控制是否启用
-        if not crawler.settings.get('LOG_FILE') and not crawler.settings.get('LOG_ENABLE_CUSTOM'):
+        log_file = crawler.settings.get('LOG_FILE')
+        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
+        
+        # 只有当没有配置日志文件且未启用自定义日志时才禁用
+        if not log_file and not log_enable_custom:
             raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
 
         return cls(crawler.settings)
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Any) -> None:
         logger = get_logger(__name__)
-        logger.info(
-            f"CustomLoggerExtension: Logging initialized. "
-            f"LOG_FILE={self.settings.get('LOG_FILE')}, "
-            f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
-        )
+        try:
+            logger.info(
+                f"CustomLoggerExtension: Logging initialized. "
+                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+            )
+        except Exception as e:
+            # 即使日志初始化信息无法打印，也不应该影响程序运行
+            pass

crawlo/extension/memory_monitor.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+import psutil
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+from crawlo.event import spider_opened, spider_closed
+
+
+class MemoryMonitorExtension:
+    """
+    内存监控扩展
+    定期监控爬虫进程的内存使用情况，并在超出阈值时发出警告
+    """
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.process = psutil.Process()
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.interval = self.settings.get_int('MEMORY_MONITOR_INTERVAL', 60)  # 默认60秒检查一次
+        self.warning_threshold = self.settings.get_float('MEMORY_WARNING_THRESHOLD', 80.0)  # 默认80%警告阈值
+        self.critical_threshold = self.settings.get_float('MEMORY_CRITICAL_THRESHOLD', 90.0)  # 默认90%严重阈值
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'MemoryMonitorExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('MEMORY_MONITOR_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("MemoryMonitorExtension: MEMORY_MONITOR_ENABLED is False")
+        
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时开始监控"""
+        try:
+            self.task = asyncio.create_task(self._monitor_loop())
+            self.logger.info(
+                f"Memory monitor started. Interval: {self.interval}s, "
+                f"Warning threshold: {self.warning_threshold}%, Critical threshold: {self.critical_threshold}%"
+            )
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="启动内存监控失败", 
+                raise_error=False
+            )
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止监控"""
+        try:
+            if self.task:
+                self.task.cancel()
+                try:
+                    await self.task
+                except asyncio.CancelledError:
+                    pass
+                self.task = None
+                self.logger.info("Memory monitor stopped.")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="停止内存监控失败", 
+                raise_error=False
+            )
+
+    async def _monitor_loop(self) -> None:
+        """内存监控循环"""
+        while True:
+            try:
+                # 获取内存使用信息
+                memory_info = self.process.memory_info()
+                memory_percent = self.process.memory_percent()
+                
+                # 记录内存使用情况
+                self.logger.debug(
+                    f"Memory usage: {memory_percent:.2f}% "
+                    f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB, "
+                    f"VMS: {memory_info.vms / 1024 / 1024:.2f} MB)"
+                )
+                
+                # 检查是否超过阈值
+                if memory_percent >= self.critical_threshold:
+                    self.logger.critical(
+                        f"Memory usage critical: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                elif memory_percent >= self.warning_threshold:
+                    self.logger.warning(
+                        f"Memory usage high: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                
+                await asyncio.sleep(self.interval)
+            except Exception as e:
+                self.logger.error(f"Error in memory monitoring: {e}")
+                await asyncio.sleep(self.interval)

crawlo/extension/performance_profiler.py

@@ -0,0 +1,134 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import io
+import os
+import pstats
+import asyncio
+import cProfile
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+from crawlo.event import spider_opened, spider_closed
+
+
+class PerformanceProfilerExtension:
+    """
+    性能分析扩展
+    在爬虫运行期间进行性能分析，帮助优化爬虫性能
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('PERFORMANCE_PROFILER_ENABLED', False)
+        self.output_dir = self.settings.get('PERFORMANCE_PROFILER_OUTPUT_DIR', 'profiling')
+        self.interval = self.settings.get_int('PERFORMANCE_PROFILER_INTERVAL', 300)  # 默认5分钟
+        
+        self.profiler: Optional[cProfile.Profile] = None
+        self.task: Optional[asyncio.Task] = None
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'PerformanceProfilerExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('PERFORMANCE_PROFILER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("PerformanceProfilerExtension: PERFORMANCE_PROFILER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时开始性能分析"""
+        if not self.enabled:
+            return
+            
+        try:
+            self.profiler = cProfile.Profile()
+            self.profiler.enable()
+            
+            # 启动定期保存分析结果的任务
+            self.task = asyncio.create_task(self._periodic_save())
+            
+            self.logger.info("Performance profiler started.")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="启动性能分析器失败", 
+                raise_error=False
+            )
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止性能分析并保存结果"""
+        if not self.enabled or not self.profiler:
+            return
+            
+        try:
+            # 停止定期保存任务
+            if self.task:
+                self.task.cancel()
+                try:
+                    await self.task
+                except asyncio.CancelledError:
+                    pass
+            
+            # 停止分析器并保存最终结果
+            self.profiler.disable()
+            
+            # 保存分析结果
+            await self._save_profile("final")
+            self.logger.info("Performance profiler stopped and results saved.")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="停止性能分析器失败", 
+                raise_error=False
+            )
+
+    async def _periodic_save(self) -> None:
+        """定期保存分析结果"""
+        counter = 1
+        while True:
+            try:
+                await asyncio.sleep(self.interval)
+                if self.profiler:
+                    # 临时禁用分析器以保存结果
+                    self.profiler.disable()
+                    await self._save_profile(f"periodic_{counter}")
+                    counter += 1
+                    # 重新启用分析器
+                    self.profiler.enable()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in periodic profiling save: {e}")
+
+    async def _save_profile(self, name: str) -> None:
+        """保存分析结果到文件"""
+        try:
+            # 创建内存中的字符串流
+            s = io.StringIO()
+            ps = pstats.Stats(self.profiler, stream=s)
+            
+            # 排序并打印统计信息
+            ps.sort_stats('cumulative')
+            ps.print_stats()
+            
+            # 保存到文件
+            filename = os.path.join(self.output_dir, f'profile_{name}.txt')
+            with open(filename, 'w', encoding='utf-8') as f:
+                f.write(s.getvalue())
+            
+            self.logger.info(f"Performance profile saved to {filename}")
+        except Exception as e:
+            self.logger.error(f"Error saving performance profile: {e}")

crawlo/extension/request_recorder.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import os
+import json
+from typing import Any
+from datetime import datetime
+
+from crawlo import event
+from crawlo.utils.log import get_logger
+
+
+class RequestRecorderExtension:
+    """
+    请求记录扩展
+    记录所有发送的请求信息到文件，便于调试和分析
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('REQUEST_RECORDER_ENABLED', False)
+        self.output_dir = self.settings.get('REQUEST_RECORDER_OUTPUT_DIR', 'requests_log')
+        self.max_file_size = self.settings.get_int('REQUEST_RECORDER_MAX_FILE_SIZE', 10 * 1024 * 1024)  # 默认10MB
+        
+        # 创建输出目录
+        if self.enabled:
+            os.makedirs(self.output_dir, exist_ok=True)
+            
+        self.current_file = None
+        self.current_file_size = 0
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'RequestRecorderExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('REQUEST_RECORDER_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("RequestRecorderExtension: REQUEST_RECORDER_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+            crawler.subscriber.subscribe(o.response_received, event=event.response_received)
+            crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
+        return o
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+            
+        try:
+            request_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'request',
+                'url': request.url,
+                'method': request.method,
+                'headers': dict(request.headers),
+                'meta': getattr(request, 'meta', {}),
+            }
+            
+            await self._write_record(request_info)
+        except Exception as e:
+            self.logger.error(f"Error recording request: {e}")
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        try:
+            response_info = {
+                'timestamp': datetime.now().isoformat(),
+                'type': 'response',
+                'url': response.url,
+                'status_code': response.status_code,
+                'headers': dict(response.headers),
+            }
+            
+            await self._write_record(response_info)
+        except Exception as e:
+            self.logger.error(f"Error recording response: {e}")
+
+    async def spider_closed(self, spider: Any) -> None:
+        """爬虫关闭时清理资源"""
+        if self.current_file:
+            self.current_file.close()
+            self.current_file = None
+        self.logger.info("Request recorder closed.")
+
+    async def _write_record(self, record: dict) -> None:
+        """写入记录到文件"""
+        # 检查是否需要创建新文件
+        if not self.current_file or self.current_file_size > self.max_file_size:
+            if self.current_file:
+                self.current_file.close()
+            
+            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+            filename = os.path.join(self.output_dir, f'requests_{timestamp}.jsonl')
+            self.current_file = open(filename, 'a', encoding='utf-8')
+            self.current_file_size = 0
+        
+        # 写入记录
+        line = json.dumps(record, ensure_ascii=False) + '\n'
+        self.current_file.write(line)
+        self.current_file.flush()
+        self.current_file_size += len(line.encode('utf-8'))