crawlo 1.3.6__py3-none-any.whl → 1.3.7__py3-none-any.whl

crawlo/extension/log_stats.py

@@ -1,82 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import Any
-
-from crawlo import event
-from crawlo.tools.date_tools import now, time_diff
-
-
-class LogStats(object):
-
-    def __init__(self, stats: Any):
-        self._stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogStats':
-        o = cls(crawler.stats)
-        # 订阅所有需要的事件
-        event_subscriptions = [
-            (o.spider_opened, event.spider_opened),
-            (o.spider_closed, event.spider_closed),
-            (o.item_successful, event.item_successful),
-            (o.item_discard, event.item_discard),
-            (o.response_received, event.response_received),
-            (o.request_scheduled, event.request_scheduled),
-        ]
-        
-        for handler, evt in event_subscriptions:
-            try:
-                crawler.subscriber.subscribe(handler, event=evt)
-            except Exception as e:
-                # 获取日志记录器并记录错误
-                from crawlo.utils.log import get_logger
-                logger = get_logger(cls.__name__)
-                logger.error(f"Failed to subscribe to event {evt}: {e}")
-
-        return o
-
-    async def spider_opened(self) -> None:
-        try:
-            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def spider_closed(self) -> None:
-        try:
-            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_successful(self, _item: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_successful_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_discard_count')
-            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
-            if reason:
-                self._stats.inc_value(f"item_discard/{reason}")
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def response_received(self, _response: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('response_received_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('request_scheduler_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Any
+
+from crawlo import event
+from crawlo.tools.date_tools import now, time_diff
+
+
+class LogStats(object):
+
+    def __init__(self, stats: Any):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogStats':
+        o = cls(crawler.stats)
+        # 订阅所有需要的事件
+        event_subscriptions = [
+            (o.spider_opened, event.spider_opened),
+            (o.spider_closed, event.spider_closed),
+            (o.item_successful, event.item_successful),
+            (o.item_discard, event.item_discard),
+            (o.response_received, event.response_received),
+            (o.request_scheduled, event.request_scheduled),
+        ]
+        
+        for handler, evt in event_subscriptions:
+            try:
+                crawler.subscriber.subscribe(handler, event=evt)
+            except Exception as e:
+                # 获取日志记录器并记录错误
+                from crawlo.utils.log import get_logger
+                logger = get_logger(cls.__name__)
+                logger.error(f"Failed to subscribe to event {evt}: {e}")
+
+        return o
+
+    async def spider_opened(self) -> None:
+        try:
+            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def spider_closed(self) -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_discard_count')
+            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
+            if reason:
+                self._stats.inc_value(f"item_discard/{reason}")
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
             pass

crawlo/extension/logging_extension.py

@@ -1,62 +1,62 @@
-from typing import Any
-from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
-
-# 延迟获取logger，确保在日志系统配置之后获取
-_logger = None
-
-def logger():
-    """延迟获取logger实例，确保在日志系统配置之后获取"""
-    global _logger
-    if _logger is None:
-        _logger = get_logger(__name__)
-    return _logger
-
-
-class CustomLoggerExtension:
-    """
-    日志系统初始化扩展
-    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
-    """
-
-    def __init__(self, settings: Any):
-        self.settings = settings
-        # 使用新的日志系统，但要简化配置传递
-        try:
-            from crawlo.logging import configure_logging
-            # 直接传递settings对象，让日志系统内部处理
-            configure_logging(settings)
-        except Exception as e:
-            # 如果日志系统配置失败，不应该阻止扩展加载
-            # 使用基本日志输出错误信息
-            import logging
-            logging.getLogger(__name__).warning(f"Failed to configure logging system: {e}")
-            # 不抛出异常，让扩展继续加载
-
-    @classmethod
-    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
-        """
-        工厂方法：兼容 ExtensionManager 的创建方式
-        被 ExtensionManager 调用
-        """
-        # 可以通过 settings 控制是否启用
-        log_file = crawler.settings.get('LOG_FILE')
-        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
-        
-        # 只有当没有配置日志文件且未启用自定义日志时才禁用
-        if not log_file and not log_enable_custom:
-            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
-
-        return cls(crawler.settings)
-
-    def spider_opened(self, spider: Any) -> None:
-        logger_instance = logger()
-        try:
-            logger_instance.info(
-                f"CustomLoggerExtension: Logging initialized. "
-                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
-                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
-            )
-        except Exception as e:
-            # 即使日志初始化信息无法打印，也不应该影响程序运行
+from typing import Any
+from crawlo.exceptions import NotConfigured
+from crawlo.utils.log import get_logger
+
+# 延迟获取logger，确保在日志系统配置之后获取
+_logger = None
+
+def logger():
+    """延迟获取logger实例，确保在日志系统配置之后获取"""
+    global _logger
+    if _logger is None:
+        _logger = get_logger(__name__)
+    return _logger
+
+
+class CustomLoggerExtension:
+    """
+    日志系统初始化扩展
+    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
+    """
+
+    def __init__(self, settings: Any):
+        self.settings = settings
+        # 使用新的日志系统，但要简化配置传递
+        try:
+            from crawlo.logging import configure_logging
+            # 直接传递settings对象，让日志系统内部处理
+            configure_logging(settings)
+        except Exception as e:
+            # 如果日志系统配置失败，不应该阻止扩展加载
+            # 使用基本日志输出错误信息
+            import logging
+            logging.getLogger(__name__).warning(f"Failed to configure logging system: {e}")
+            # 不抛出异常，让扩展继续加载
+
+    @classmethod
+    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
+        """
+        工厂方法：兼容 ExtensionManager 的创建方式
+        被 ExtensionManager 调用
+        """
+        # 可以通过 settings 控制是否启用
+        log_file = crawler.settings.get('LOG_FILE')
+        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
+        
+        # 只有当没有配置日志文件且未启用自定义日志时才禁用
+        if not log_file and not log_enable_custom:
+            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
+
+        return cls(crawler.settings)
+
+    def spider_opened(self, spider: Any) -> None:
+        logger_instance = logger()
+        try:
+            logger_instance.info(
+                f"CustomLoggerExtension: Logging initialized. "
+                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+            )
+        except Exception as e:
+            # 即使日志初始化信息无法打印，也不应该影响程序运行
             pass

crawlo/extension/memory_monitor.py

@@ -1,105 +1,105 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-import psutil
-from typing import Any, Optional
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.error_handler import ErrorHandler
-from crawlo.event import spider_opened, spider_closed
-
-
-class MemoryMonitorExtension:
-    """
-    内存监控扩展
-    定期监控爬虫进程的内存使用情况，并在超出阈值时发出警告
-    """
-
-    def __init__(self, crawler: Any):
-        self.task: Optional[asyncio.Task] = None
-        self.process = psutil.Process()
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        
-        # 获取配置参数
-        self.interval = self.settings.get_int('MEMORY_MONITOR_INTERVAL', 60)  # 默认60秒检查一次
-        self.warning_threshold = self.settings.get_float('MEMORY_WARNING_THRESHOLD', 80.0)  # 默认80%警告阈值
-        self.critical_threshold = self.settings.get_float('MEMORY_CRITICAL_THRESHOLD', 90.0)  # 默认90%严重阈值
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'MemoryMonitorExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('MEMORY_MONITOR_ENABLED', False):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("MemoryMonitorExtension: MEMORY_MONITOR_ENABLED is False")
-        
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self) -> None:
-        """爬虫启动时开始监控"""
-        try:
-            self.task = asyncio.create_task(self._monitor_loop())
-            self.logger.info(
-                f"Memory monitor started. Interval: {self.interval}s, "
-                f"Warning threshold: {self.warning_threshold}%, Critical threshold: {self.critical_threshold}%"
-            )
-        except Exception as e:
-            self.error_handler.handle_error(
-                e, 
-                context="启动内存监控失败", 
-                raise_error=False
-            )
-
-    async def spider_closed(self) -> None:
-        """爬虫关闭时停止监控"""
-        try:
-            if self.task:
-                self.task.cancel()
-                try:
-                    await self.task
-                except asyncio.CancelledError:
-                    pass
-                self.task = None
-                self.logger.info("Memory monitor stopped.")
-        except Exception as e:
-            self.error_handler.handle_error(
-                e, 
-                context="停止内存监控失败", 
-                raise_error=False
-            )
-
-    async def _monitor_loop(self) -> None:
-        """内存监控循环"""
-        while True:
-            try:
-                # 获取内存使用信息
-                memory_info = self.process.memory_info()
-                memory_percent = self.process.memory_percent()
-                
-                # 记录内存使用情况
-                self.logger.debug(
-                    f"Memory usage: {memory_percent:.2f}% "
-                    f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB, "
-                    f"VMS: {memory_info.vms / 1024 / 1024:.2f} MB)"
-                )
-                
-                # 检查是否超过阈值
-                if memory_percent >= self.critical_threshold:
-                    self.logger.critical(
-                        f"Memory usage critical: {memory_percent:.2f}% "
-                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
-                    )
-                elif memory_percent >= self.warning_threshold:
-                    self.logger.warning(
-                        f"Memory usage high: {memory_percent:.2f}% "
-                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
-                    )
-                
-                await asyncio.sleep(self.interval)
-            except Exception as e:
-                self.logger.error(f"Error in memory monitoring: {e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+import psutil
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.error_handler import ErrorHandler
+from crawlo.event import spider_opened, spider_closed
+
+
+class MemoryMonitorExtension:
+    """
+    内存监控扩展
+    定期监控爬虫进程的内存使用情况，并在超出阈值时发出警告
+    """
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.process = psutil.Process()
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.error_handler = ErrorHandler(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.interval = self.settings.get_int('MEMORY_MONITOR_INTERVAL', 60)  # 默认60秒检查一次
+        self.warning_threshold = self.settings.get_float('MEMORY_WARNING_THRESHOLD', 80.0)  # 默认80%警告阈值
+        self.critical_threshold = self.settings.get_float('MEMORY_CRITICAL_THRESHOLD', 90.0)  # 默认90%严重阈值
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'MemoryMonitorExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('MEMORY_MONITOR_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("MemoryMonitorExtension: MEMORY_MONITOR_ENABLED is False")
+        
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时开始监控"""
+        try:
+            self.task = asyncio.create_task(self._monitor_loop())
+            self.logger.info(
+                f"Memory monitor started. Interval: {self.interval}s, "
+                f"Warning threshold: {self.warning_threshold}%, Critical threshold: {self.critical_threshold}%"
+            )
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="启动内存监控失败", 
+                raise_error=False
+            )
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止监控"""
+        try:
+            if self.task:
+                self.task.cancel()
+                try:
+                    await self.task
+                except asyncio.CancelledError:
+                    pass
+                self.task = None
+                self.logger.info("Memory monitor stopped.")
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="停止内存监控失败", 
+                raise_error=False
+            )
+
+    async def _monitor_loop(self) -> None:
+        """内存监控循环"""
+        while True:
+            try:
+                # 获取内存使用信息
+                memory_info = self.process.memory_info()
+                memory_percent = self.process.memory_percent()
+                
+                # 记录内存使用情况
+                self.logger.debug(
+                    f"Memory usage: {memory_percent:.2f}% "
+                    f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB, "
+                    f"VMS: {memory_info.vms / 1024 / 1024:.2f} MB)"
+                )
+                
+                # 检查是否超过阈值
+                if memory_percent >= self.critical_threshold:
+                    self.logger.critical(
+                        f"Memory usage critical: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                elif memory_percent >= self.warning_threshold:
+                    self.logger.warning(
+                        f"Memory usage high: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                
+                await asyncio.sleep(self.interval)
+            except Exception as e:
+                self.logger.error(f"Error in memory monitoring: {e}")
                 await asyncio.sleep(self.interval)