crawlo 1.4.7__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/extension/health_check.py

@@ -1,142 +1,142 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from datetime import datetime
-from typing import Any, Optional, Dict
-
-from crawlo.event import CrawlerEvent
-from crawlo.logging import get_logger
-
-
-class HealthCheckExtension:
-    """
-    健康检查扩展
-    监控爬虫的健康状态，包括响应时间、错误率等指标
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__)
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
-        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
-        
-        # 健康状态统计
-        self.stats: Dict[str, Any] = {
-            'start_time': None,
-            'total_requests': 0,
-            'total_responses': 0,
-            'error_responses': 0,
-            'last_check_time': None,
-            'response_times': [],  # 存储最近的响应时间
-        }
-        
-        self.task: Optional[asyncio.Task] = None
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
-            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
-            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
-            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
-        return o
-
-    async def spider_opened(self) -> None:
-        """爬虫启动时初始化健康检查"""
-        if not self.enabled:
-            return
-            
-        self.stats['start_time'] = datetime.now()
-        self.task = asyncio.create_task(self._health_check_loop())
-        self.logger.info("Health check extension started.")
-
-    async def spider_closed(self) -> None:
-        """爬虫关闭时停止健康检查"""
-        if not self.enabled:
-            return
-            
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-        
-        # 输出最终健康状态
-        await self._check_health()
-        self.logger.info("Health check extension stopped.")
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-        self.stats['total_requests'] += 1
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        self.stats['total_responses'] += 1
-        
-        # 记录错误响应
-        if hasattr(response, 'status_code') and response.status_code >= 400:
-            self.stats['error_responses'] += 1
-
-    async def _health_check_loop(self) -> None:
-        """健康检查循环"""
-        while True:
-            try:
-                await asyncio.sleep(self.check_interval)
-                await self._check_health()
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                self.logger.error(f"Error in health check loop: {e}")
-
-    async def _check_health(self) -> None:
-        """执行健康检查并输出报告"""
-        try:
-            now_time = datetime.now()
-            self.stats['last_check_time'] = now_time
-            
-            # 计算基本统计信息
-            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
-            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
-            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
-            
-            # 计算错误率
-            error_rate = (
-                self.stats['error_responses'] / self.stats['total_responses'] 
-                if self.stats['total_responses'] > 0 else 0
-            )
-            
-            # 输出健康报告
-            health_report = {
-                'runtime_seconds': round(runtime, 2),
-                'total_requests': self.stats['total_requests'],
-                'total_responses': self.stats['total_responses'],
-                'requests_per_second': round(requests_per_second, 2),
-                'responses_per_second': round(responses_per_second, 2),
-                'error_responses': self.stats['error_responses'],
-                'error_rate': f"{error_rate:.2%}",
-            }
-            
-            # 根据错误率判断健康状态
-            if error_rate > 0.1:  # 错误率超过10%
-                self.logger.warning(f"Health check report: {health_report}")
-            elif error_rate > 0.05:  # 错误率超过5%
-                self.logger.info(f"Health check report: {health_report}")
-            else:
-                self.logger.debug(f"Health check report: {health_report}")
-                
-        except Exception as e:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from datetime import datetime
+from typing import Any, Optional, Dict
+
+from crawlo.event import CrawlerEvent
+from crawlo.logging import get_logger
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__)
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+            crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
+            crawler.subscriber.subscribe(o.response_received, event=CrawlerEvent.RESPONSE_RECEIVED)
+            crawler.subscriber.subscribe(o.request_scheduled, event=CrawlerEvent.REQUEST_SCHEDULED)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
             self.logger.error(f"Error in health check: {e}")

crawlo/extension/log_interval.py

@@ -1,95 +1,95 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import Any, Optional
-
-from crawlo.logging import get_logger
-from crawlo.event import CrawlerEvent
-
-
-class LogIntervalExtension:
-
-    def __init__(self, crawler: Any):
-        self.task: Optional[asyncio.Task] = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
-        
-        # 修复时间单位计算逻辑
-        if self.seconds % 60 == 0:
-            self.interval = int(self.seconds / 60)
-            self.unit = 'min'
-        else:
-            self.interval = self.seconds
-            self.unit = 's'
-        
-        # 处理单数情况
-        if self.interval == 1 and self.unit == 'min':
-            self.interval_display = ""
-        else:
-            self.interval_display = str(self.interval)
-
-        self.logger = get_logger(self.__class__.__name__)
-        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
-        crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
-        return o
-
-    async def spider_opened(self) -> None:
-        self.logger.info("Spider opened, starting interval logging task")
-        self.task = asyncio.create_task(self.interval_log())
-        self.logger.info("Interval logging task started")
-
-    async def spider_closed(self) -> None:
-        self.logger.info("Spider closed, stopping interval logging task")
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-            self.task = None
-
-    async def interval_log(self) -> None:
-        iteration = 0
-        while True:
-            try:
-                iteration += 1
-                self.logger.debug(f"Interval log iteration {iteration} starting")
-                last_item_count = self.stats.get_value('item_successful_count', default=0)
-                last_response_count = self.stats.get_value('response_received_count', default=0)
-                item_rate = last_item_count - self.item_count
-                response_rate = last_response_count - self.response_count
-                
-                # 添加调试信息
-                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
-                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
-                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
-                
-                self.item_count, self.response_count = last_item_count, last_response_count
-                
-                # 修复效率计算，确保使用正确的单位
-                if self.unit == 'min' and self.seconds > 0:
-                    # 转换为每分钟速率
-                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
-                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
-                    self.logger.info(
-                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
-                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
-                    )
-                else:
-                    # 使用原始单位
-                    self.logger.info(
-                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
-                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
-                    )
-                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
-                await asyncio.sleep(self.seconds)
-            except Exception as e:
-                self.logger.error(f"Error in interval logging: {e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.logging import get_logger
+from crawlo.event import CrawlerEvent
+
+
+class LogIntervalExtension:
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        
+        # 修复时间单位计算逻辑
+        if self.seconds % 60 == 0:
+            self.interval = int(self.seconds / 60)
+            self.unit = 'min'
+        else:
+            self.interval = self.seconds
+            self.unit = 's'
+        
+        # 处理单数情况
+        if self.interval == 1 and self.unit == 'min':
+            self.interval_display = ""
+        else:
+            self.interval_display = str(self.interval)
+
+        self.logger = get_logger(self.__class__.__name__)
+        self.logger.info(f"LogIntervalExtension initialized with interval: {self.seconds} seconds")
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=CrawlerEvent.SPIDER_OPENED)
+        crawler.subscriber.subscribe(o.spider_closed, event=CrawlerEvent.SPIDER_CLOSED)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.logger.info("Spider opened, starting interval logging task")
+        self.task = asyncio.create_task(self.interval_log())
+        self.logger.info("Interval logging task started")
+
+    async def spider_closed(self) -> None:
+        self.logger.info("Spider closed, stopping interval logging task")
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        iteration = 0
+        while True:
+            try:
+                iteration += 1
+                self.logger.debug(f"Interval log iteration {iteration} starting")
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                
+                # 添加调试信息
+                self.logger.debug(f"Debug info - Iteration: {iteration}, Last item count: {last_item_count}, Last response count: {last_response_count}")
+                self.logger.debug(f"Debug info - Previous item count: {self.item_count}, Previous response count: {self.response_count}")
+                self.logger.debug(f"Debug info - Item rate: {item_rate}, Response rate: {response_rate}")
+                
+                self.item_count, self.response_count = last_item_count, last_response_count
+                
+                # 修复效率计算，确保使用正确的单位
+                if self.unit == 'min' and self.seconds > 0:
+                    # 转换为每分钟速率
+                    pages_per_min = response_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    items_per_min = item_rate * 60 / self.seconds if self.seconds > 0 else 0
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {pages_per_min:.0f} pages/min),'
+                        f' Got {last_item_count} items (at {items_per_min:.0f} items/min).'
+                    )
+                else:
+                    # 使用原始单位
+                    self.logger.info(
+                        f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval_display}{self.unit}),'
+                        f' Got {last_item_count} items (at {item_rate} items/{self.interval_display}{self.unit}).'
+                    )
+                self.logger.debug(f"Interval log iteration {iteration} completed, sleeping for {self.seconds} seconds")
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
                 await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,71 +1,71 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-日志统计扩展
-提供详细的爬虫运行统计信息
-"""
-import asyncio
-from typing import Any
-
-from crawlo.logging import get_logger
-from crawlo.utils import now, time_diff
-
-
-class LogStats:
-    """
-    日志统计扩展，记录和输出爬虫运行过程中的各种统计信息
-    """
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.logger = get_logger(self.__class__.__name__)
-        self._stats = crawler.stats
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls.from_crawler(crawler)
-
-    async def spider_closed(self, reason: str = 'finished') -> None:
-        try:
-            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-            self._stats['reason'] = reason
-        except Exception as e:
-            # 添加日志以便调试
-            self.logger.error(f"Error in spider_closed: {e}")
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_successful(self, _item: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_successful_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
-        try:
-            # 只增加总的丢弃计数，不记录每个丢弃项目的原因详情
-            self._stats.inc_value('item_discard_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def response_received(self, _response: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('response_received_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('request_scheduler_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+日志统计扩展
+提供详细的爬虫运行统计信息
+"""
+import asyncio
+from typing import Any
+
+from crawlo.logging import get_logger
+from crawlo.utils import now, time_diff
+
+
+class LogStats:
+    """
+    日志统计扩展，记录和输出爬虫运行过程中的各种统计信息
+    """
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.logger = get_logger(self.__class__.__name__)
+        self._stats = crawler.stats
+        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    @classmethod
+    def create_instance(cls, crawler):
+        return cls.from_crawler(crawler)
+
+    async def spider_closed(self, reason: str = 'finished') -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+            self._stats['reason'] = reason
+        except Exception as e:
+            # 添加日志以便调试
+            self.logger.error(f"Error in spider_closed: {e}")
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            # 只增加总的丢弃计数，不记录每个丢弃项目的原因详情
+            self._stats.inc_value('item_discard_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
             pass