crawlo 1.1.3__py3-none-any.whl → 1.1.4__py3-none-any.whl

crawlo/exceptions.py

@@ -1,82 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
-    pass
-
-
-class OutputError(Exception):
-    pass
-
-
-class SpiderTypeError(TypeError):
-    pass
-
-
-class ItemInitError(Exception):
-    pass
-
-
-class ItemAttributeError(Exception):
-    pass
-
-
-class DecodeError(Exception):
-    pass
-
-
-class MiddlewareInitError(Exception):
-    pass
-
-
-class PipelineInitError(Exception):
-    pass
-
-
-class InvalidOutputError(Exception):
-    pass
-
-
-class RequestMethodError(Exception):
-    pass
-
-
-class IgnoreRequestError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
-
-
-class ItemDiscard(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
-
-
-class NotConfigured(Exception):
-    pass
-
-
-class NotConfiguredError(Exception):
-    pass
-
-
-class ExtensionInitError(Exception):
-    pass
-
-
-class ReceiverTypeError(Exception):
-    pass
-
-
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
-    pass
-
-
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
-    pass
-
-
-class DropItem(Exception):
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+class TransformTypeError(TypeError):
+    pass
+
+
+class OutputError(Exception):
+    pass
+
+
+class SpiderTypeError(TypeError):
+    pass
+
+
+class ItemInitError(Exception):
+    pass
+
+
+class ItemAttributeError(Exception):
+    pass
+
+
+class DecodeError(Exception):
+    pass
+
+
+class MiddlewareInitError(Exception):
+    pass
+
+
+class PipelineInitError(Exception):
+    pass
+
+
+class InvalidOutputError(Exception):
+    pass
+
+
+class RequestMethodError(Exception):
+    pass
+
+
+class IgnoreRequestError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(IgnoreRequestError, self).__init__(msg)
+
+
+class ItemDiscard(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(ItemDiscard, self).__init__(msg)
+
+
+class NotConfigured(Exception):
+    pass
+
+
+class NotConfiguredError(Exception):
+    pass
+
+
+class ExtensionInitError(Exception):
+    pass
+
+
+class ReceiverTypeError(Exception):
+    pass
+
+
+class SpiderCreationError(Exception):
+    """爬虫实例化失败异常"""
+    pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass
+
+
+class DropItem(Exception):
     pass

crawlo/extension/__init__.py

@@ -1,31 +1,38 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.project import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-
-    @classmethod
-    def create_instance(cls, *args, **kwargs):
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions):
-        for extension in extensions:
-            extension_cls = load_class(extension)
-            if not hasattr(extension_cls, 'create_instance'):
-                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
-            self.extensions.append(extension_cls.create_instance(self.crawler))
-        if extensions:
-            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List, Any
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.project import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler: Any):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_class(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
+        if extensions:
+            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")

crawlo/extension/health_check.py

@@ -0,0 +1,142 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional, Dict
+from datetime import datetime, timedelta
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.response_received, event=response_received)
+            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
+            self.logger.error(f"Error in health check: {e}")

crawlo/extension/log_interval.py

@@ -1,49 +1,58 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler):
-        self.task = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL')
-        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
-        self.interval = "" if self.interval == 1 else self.interval
-        self.unit = 'min' if self.seconds % 60 == 0 else 's'
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self):
-        self.task = asyncio.create_task(self.interval_log())
-        await self.task
-
-    async def spider_closed(self):
-        if self.task:
-            self.task.cancel()
-
-    async def interval_log(self):
-        while True:
-            last_item_count = self.stats.get_value('item_successful_count', default=0)
-            last_response_count = self.stats.get_value('response_received_count', default=0)
-            item_rate = last_item_count - self.item_count
-            response_rate = last_response_count - self.response_count
-            self.item_count, self.response_count = last_item_count, last_response_count
-            self.logger.info(
-                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-            )
-            await asyncio.sleep(self.seconds)
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.task = asyncio.create_task(self.interval_log())
+
+    async def spider_closed(self) -> None:
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        while True:
+            try:
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                self.item_count, self.response_count = last_item_count, last_response_count
+                self.logger.info(
+                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+                )
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
+                await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,44 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from crawlo import event
-from crawlo.utils.date_tools import now, time_diff
-
-
-class LogStats(object):
-
-    def __init__(self, stats):
-        self._stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler):
-        o = cls(crawler.stats)
-        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
-        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
-        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
-
-        return o
-
-    async def spider_opened(self):
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-
-    async def spider_closed(self):
-        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-
-    async def item_successful(self, _item, _spider):
-        self._stats.inc_value('item_successful_count')
-
-    async def item_discard(self, _item, exc, _spider):
-        self._stats.inc_value('item_discard_count')
-        reason = exc.msg
-        if reason:
-            self._stats.inc_value(f"item_discard/{reason}")
-
-    async def response_received(self, _response, _spider):
-        self._stats.inc_value('response_received_count')
-
-    async def request_scheduled(self, _request, _spider):
-        self._stats.inc_value('request_scheduler_count')
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Any
+
+from crawlo import event
+from crawlo.utils.date_tools import now, time_diff
+
+
+class LogStats(object):
+
+    def __init__(self, stats: Any):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogStats':
+        o = cls(crawler.stats)
+        # 订阅所有需要的事件
+        event_subscriptions = [
+            (o.spider_opened, event.spider_opened),
+            (o.spider_closed, event.spider_closed),
+            (o.item_successful, event.item_successful),
+            (o.item_discard, event.item_discard),
+            (o.response_received, event.response_received),
+            (o.request_scheduled, event.request_scheduled),
+        ]
+        
+        for handler, evt in event_subscriptions:
+            try:
+                crawler.subscriber.subscribe(handler, event=evt)
+            except Exception as e:
+                # 获取日志记录器并记录错误
+                from crawlo.utils.log import get_logger
+                logger = get_logger(cls.__name__)
+                logger.error(f"Failed to subscribe to event {evt}: {e}")
+
+        return o
+
+    async def spider_opened(self) -> None:
+        try:
+            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def spider_closed(self) -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_discard_count')
+            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
+            if reason:
+                self._stats.inc_value(f"item_discard/{reason}")
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass