crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/extension/log_interval.py

@@ -1,95 +1,95 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import Any, Optional
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler: Any):
-        self.task: Optional[asyncio.Task] = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
-        
-        # 修复时间单位计算逻辑
-        if self.seconds % 60 == 0:
-            self.interval = int(self.seconds / 60)
-            self.unit = 'min'
-        else:
-            self.interval = self.seconds
-            self.unit = 's'
-        
-        # 处理单数情况
-        if self.interval == 1 and self.unit == 'min':
-            self.interval_display = ""
-        else:
-            self.interval_display = str(self.interval)
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self) -> None:
-        self.logger.info("Spider opened, starting interval logging task")
-        self.task = asyncio.create_task(self.interval_log())
-        self.logger.info("Interval logging task started")
-
-    async def spider_closed(self) -> None:
-        self.logger.info("Spider closed, stopping interval logging task")
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-            self.task = None
-
-    async def interval_log(self) -> None:
-        iteration = 0
-        while True:
-            try:
-                iteration += 1
-                self.logger.debug(f"Interval log iteration {iteration} starting")
-                last_item_count = self.stats.get_value('item_successful_count', default=0)
-                last_response_count = self.stats.get_value('response_received_count', default=0)
-                item_rate = last_item_count - self.item_count
-                response_rate = last_response_count - self.response_count
-                
-                # 添加调试信息
-                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
-                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
-                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
-                
-                self.item_count, self.response_count = last_item_count, last_response_count
-                
-                # 修复效率计算，确保使用正确的单位
-                if self.unit == 'min' and self.seconds > 0:
-                    # 转换为每分钟速率
-                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
-                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
-                    self.logger.info(
-                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
-                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
-                    )
-                else:
-                    # 使用原始单位
-                    self.logger.info(
-                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
-                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
-                    )
-                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
-                await asyncio.sleep(self.seconds)
-            except Exception as e:
-                self.logger.error(f"Error in interval logging: {e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        
+        # 修复时间单位计算逻辑
+        if self.seconds % 60 == 0:
+            self.interval = int(self.seconds / 60)
+            self.unit = 'min'
+        else:
+            self.interval = self.seconds
+            self.unit = 's'
+        
+        # 处理单数情况
+        if self.interval == 1 and self.unit == 'min':
+            self.interval_display = ""
+        else:
+            self.interval_display = str(self.interval)
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.logger.info("Spider opened, starting interval logging task")
+        self.task = asyncio.create_task(self.interval_log())
+        self.logger.info("Interval logging task started")
+
+    async def spider_closed(self) -> None:
+        self.logger.info("Spider closed, stopping interval logging task")
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        iteration = 0
+        while True:
+            try:
+                iteration += 1
+                self.logger.debug(f"Interval log iteration {iteration} starting")
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                
+                # 添加调试信息
+                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
+                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
+                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
+                
+                self.item_count, self.response_count = last_item_count, last_response_count
+                
+                # 修复效率计算，确保使用正确的单位
+                if self.unit == 'min' and self.seconds > 0:
+                    # 转换为每分钟速率
+                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
+                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
+                    )
+                else:
+                    # 使用原始单位
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
+                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
+                    )
+                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
                 await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,71 +1,71 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-日志统计扩展
-提供详细的爬虫运行统计信息
-"""
-import asyncio
-from typing import Any
-
-from crawlo.utils.log import get_logger
-from crawlo.utils import now, time_diff
-
-
-class LogStats:
-    """
-    日志统计扩展，记录和输出爬虫运行过程中的各种统计信息
-    """
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._stats = crawler.stats
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls.from_crawler(crawler)
-
-    async def spider_closed(self, reason: str = 'finished') -> None:
-        try:
-            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-            self._stats['reason'] = reason
-        except Exception as e:
-            # 添加日志以便调试
-            self.logger.error(f"Error in spider_closed: {e}")
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_successful(self, _item: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_successful_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
-        try:
-            # 只增加总的丢弃计数，不记录每个丢弃项目的原因详情
-            self._stats.inc_value('item_discard_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def response_received(self, _response: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('response_received_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('request_scheduler_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+日志统计扩展
+提供详细的爬虫运行统计信息
+"""
+import asyncio
+from typing import Any
+
+from crawlo.utils.log import get_logger
+from crawlo.utils import now, time_diff
+
+
+class LogStats:
+    """
+    日志统计扩展，记录和输出爬虫运行过程中的各种统计信息
+    """
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._stats = crawler.stats
+        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls.from_crawler(crawler)
+
+    async def spider_closed(self, reason: str = 'finished') -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+            self._stats['reason'] = reason
+        except Exception as e:
+            # 添加日志以便调试
+            self.logger.error(f"Error in spider_closed: {e}")
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            # 只增加总的丢弃计数，不记录每个丢弃项目的原因详情
+            self._stats.inc_value('item_discard_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
             pass

crawlo/extension/logging_extension.py

@@ -1,62 +1,62 @@
-from typing import Any
-from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
-
-# 延迟获取logger，确保在日志系统配置之后获取
-_logger = None
-
-def logger():
-    """延迟获取logger实例，确保在日志系统配置之后获取"""
-    global _logger
-    if _logger is None:
-        _logger = get_logger(__name__)
-    return _logger
-
-
-class CustomLoggerExtension:
-    """
-    日志系统初始化扩展
-    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
-    """
-
-    def __init__(self, settings: Any):
-        self.settings = settings
-        # 使用新的日志系统，但要简化配置传递
-        try:
-            from crawlo.logging import configure_logging
-            # 直接传递settings对象，让日志系统内部处理
-            configure_logging(settings)
-        except Exception as e:
-            # 如果日志系统配置失败，不应该阻止扩展加载
-            # 使用基本日志输出错误信息
-            import logging
-            logging.getLogger(__name__).warning(f"Failed to configure logging system: {e}")
-            # 不抛出异常，让扩展继续加载
-
-    @classmethod
-    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
-        """
-        工厂方法：兼容 ExtensionManager 的创建方式
-        被 ExtensionManager 调用
-        """
-        # 可以通过 settings 控制是否启用
-        log_file = crawler.settings.get('LOG_FILE')
-        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
-        
-        # 只有当没有配置日志文件且未启用自定义日志时才禁用
-        if not log_file and not log_enable_custom:
-            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
-
-        return cls(crawler.settings)
-
-    def spider_opened(self, spider: Any) -> None:
-        logger_instance = logger()
-        try:
-            logger_instance.info(
-                f"CustomLoggerExtension: Logging initialized. "
-                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
-                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
-            )
-        except Exception as e:
-            # 即使日志初始化信息无法打印，也不应该影响程序运行
+from typing import Any
+from crawlo.exceptions import NotConfigured
+from crawlo.utils.log import get_logger
+
+# 延迟获取logger，确保在日志系统配置之后获取
+_logger = None
+
+def logger():
+    """延迟获取logger实例，确保在日志系统配置之后获取"""
+    global _logger
+    if _logger is None:
+        _logger = get_logger(__name__)
+    return _logger
+
+
+class CustomLoggerExtension:
+    """
+    日志系统初始化扩展
+    遵循与 ExtensionManager 一致的接口规范：使用 create_instance
+    """
+
+    def __init__(self, settings: Any):
+        self.settings = settings
+        # 使用新的日志系统，但要简化配置传递
+        try:
+            from crawlo.logging import configure_logging
+            # 直接传递settings对象，让日志系统内部处理
+            configure_logging(settings)
+        except Exception as e:
+            # 如果日志系统配置失败，不应该阻止扩展加载
+            # 使用基本日志输出错误信息
+            import logging
+            logging.getLogger(__name__).warning(f"Failed to configure logging system: {e}")
+            # 不抛出异常，让扩展继续加载
+
+    @classmethod
+    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
+        """
+        工厂方法：兼容 ExtensionManager 的创建方式
+        被 ExtensionManager 调用
+        """
+        # 可以通过 settings 控制是否启用
+        log_file = crawler.settings.get('LOG_FILE')
+        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
+        
+        # 只有当没有配置日志文件且未启用自定义日志时才禁用
+        if not log_file and not log_enable_custom:
+            raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
+
+        return cls(crawler.settings)
+
+    def spider_opened(self, spider: Any) -> None:
+        logger_instance = logger()
+        try:
+            logger_instance.info(
+                f"CustomLoggerExtension: Logging initialized. "
+                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+            )
+        except Exception as e:
+            # 即使日志初始化信息无法打印，也不应该影响程序运行
             pass