crawlo 1.4.1__py3-none-any.whl → 1.4.3__py3-none-any.whl

crawlo/event.py

@@ -1,11 +1,11 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-spider_error = "spider_error"
-spider_opened = "spider_open"
-spider_closed = "spider_closed"
-ignore_request = "ignore_request"
-request_scheduled = "request_scheduled"
-response_received = "request_received"
-item_successful = "item_successful"
-item_discard = "item_discard"
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+spider_error = "spider_error"
+spider_opened = "spider_open"
+spider_closed = "spider_closed"
+ignore_request = "ignore_request"
+request_scheduled = "request_scheduled"
+response_received = "request_received"
+item_successful = "item_successful"
+item_discard = "item_discard"

crawlo/exceptions.py

@@ -1,82 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
-    pass
-
-
-class OutputError(Exception):
-    pass
-
-
-class SpiderTypeError(TypeError):
-    pass
-
-
-class ItemInitError(Exception):
-    pass
-
-
-class ItemAttributeError(Exception):
-    pass
-
-
-class DecodeError(Exception):
-    pass
-
-
-class MiddlewareInitError(Exception):
-    pass
-
-
-class PipelineInitError(Exception):
-    pass
-
-
-class InvalidOutputError(Exception):
-    pass
-
-
-class RequestMethodError(Exception):
-    pass
-
-
-class IgnoreRequestError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
-
-
-class ItemDiscard(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
-
-
-class NotConfigured(Exception):
-    pass
-
-
-class NotConfiguredError(Exception):
-    pass
-
-
-class ExtensionInitError(Exception):
-    pass
-
-
-class ReceiverTypeError(Exception):
-    pass
-
-
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
-    pass
-
-
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
-    pass
-
-
-class DropItem(Exception):
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+class TransformTypeError(TypeError):
+    pass
+
+
+class OutputError(Exception):
+    pass
+
+
+class SpiderTypeError(TypeError):
+    pass
+
+
+class ItemInitError(Exception):
+    pass
+
+
+class ItemAttributeError(Exception):
+    pass
+
+
+class DecodeError(Exception):
+    pass
+
+
+class MiddlewareInitError(Exception):
+    pass
+
+
+class PipelineInitError(Exception):
+    pass
+
+
+class InvalidOutputError(Exception):
+    pass
+
+
+class RequestMethodError(Exception):
+    pass
+
+
+class IgnoreRequestError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(IgnoreRequestError, self).__init__(msg)
+
+
+class ItemDiscard(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(ItemDiscard, self).__init__(msg)
+
+
+class NotConfigured(Exception):
+    pass
+
+
+class NotConfiguredError(Exception):
+    pass
+
+
+class ExtensionInitError(Exception):
+    pass
+
+
+class ReceiverTypeError(Exception):
+    pass
+
+
+class SpiderCreationError(Exception):
+    """爬虫实例化失败异常"""
+    pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass
+
+
+class DropItem(Exception):
     pass

crawlo/extension/__init__.py

@@ -1,64 +1,64 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List, Any
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.class_loader import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler: Any):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-        self._subscribe_extensions()
-
-    @classmethod
-    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions: List[str]) -> None:
-        for extension_path in extensions:
-            try:
-                extension_cls = load_class(extension_path)
-                if not hasattr(extension_cls, 'create_instance'):
-                    raise ExtensionInitError(
-                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
-                    )
-                self.extensions.append(extension_cls.create_instance(self.crawler))
-            except Exception as e:
-                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
-                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
-        
-        if extensions:
-            # 恢复INFO级别日志，保留关键的启用信息
-            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
-
-    def _subscribe_extensions(self) -> None:
-        """订阅扩展方法到相应的事件"""
-        for extension in self.extensions:
-            # 订阅 spider_closed 方法
-            if hasattr(extension, 'spider_closed'):
-                self.crawler.subscriber.subscribe(extension.spider_closed, event="spider_closed")
-            
-            # 订阅 item_successful 方法
-            if hasattr(extension, 'item_successful'):
-                self.crawler.subscriber.subscribe(extension.item_successful, event="item_successful")
-            
-            # 订阅 item_discard 方法
-            if hasattr(extension, 'item_discard'):
-                self.crawler.subscriber.subscribe(extension.item_discard, event="item_discard")
-            
-            # 订阅 response_received 方法
-            if hasattr(extension, 'response_received'):
-                # 修复：将事件名称从 "request_received" 更正为 "response_received"
-                self.crawler.subscriber.subscribe(extension.response_received, event="response_received")
-            
-            # 订阅 request_scheduled 方法
-            if hasattr(extension, 'request_scheduled'):
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List, Any
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.class_loader import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler: Any):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+        self._subscribe_extensions()
+
+    @classmethod
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_class(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
+        if extensions:
+            # 恢复INFO级别日志，保留关键的启用信息
+            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
+
+    def _subscribe_extensions(self) -> None:
+        """订阅扩展方法到相应的事件"""
+        for extension in self.extensions:
+            # 订阅 spider_closed 方法
+            if hasattr(extension, 'spider_closed'):
+                self.crawler.subscriber.subscribe(extension.spider_closed, event="spider_closed")
+            
+            # 订阅 item_successful 方法
+            if hasattr(extension, 'item_successful'):
+                self.crawler.subscriber.subscribe(extension.item_successful, event="item_successful")
+            
+            # 订阅 item_discard 方法
+            if hasattr(extension, 'item_discard'):
+                self.crawler.subscriber.subscribe(extension.item_discard, event="item_discard")
+            
+            # 订阅 response_received 方法
+            if hasattr(extension, 'response_received'):
+                # 修复：将事件名称从 "request_received" 更正为 "response_received"
+                self.crawler.subscriber.subscribe(extension.response_received, event="response_received")
+            
+            # 订阅 request_scheduled 方法
+            if hasattr(extension, 'request_scheduled'):
                 self.crawler.subscriber.subscribe(extension.request_scheduled, event="request_scheduled")

crawlo/extension/health_check.py

@@ -1,142 +1,142 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from datetime import datetime
-from typing import Any, Optional, Dict
-
-from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
-from crawlo.utils.log import get_logger
-
-
-class HealthCheckExtension:
-    """
-    健康检查扩展
-    监控爬虫的健康状态，包括响应时间、错误率等指标
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
-        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
-        
-        # 健康状态统计
-        self.stats: Dict[str, Any] = {
-            'start_time': None,
-            'total_requests': 0,
-            'total_responses': 0,
-            'error_responses': 0,
-            'last_check_time': None,
-            'response_times': [],  # 存储最近的响应时间
-        }
-        
-        self.task: Optional[asyncio.Task] = None
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-            crawler.subscriber.subscribe(o.response_received, event=response_received)
-            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
-        return o
-
-    async def spider_opened(self) -> None:
-        """爬虫启动时初始化健康检查"""
-        if not self.enabled:
-            return
-            
-        self.stats['start_time'] = datetime.now()
-        self.task = asyncio.create_task(self._health_check_loop())
-        self.logger.info("Health check extension started.")
-
-    async def spider_closed(self) -> None:
-        """爬虫关闭时停止健康检查"""
-        if not self.enabled:
-            return
-            
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-        
-        # 输出最终健康状态
-        await self._check_health()
-        self.logger.info("Health check extension stopped.")
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-        self.stats['total_requests'] += 1
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        self.stats['total_responses'] += 1
-        
-        # 记录错误响应
-        if hasattr(response, 'status_code') and response.status_code >= 400:
-            self.stats['error_responses'] += 1
-
-    async def _health_check_loop(self) -> None:
-        """健康检查循环"""
-        while True:
-            try:
-                await asyncio.sleep(self.check_interval)
-                await self._check_health()
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                self.logger.error(f"Error in health check loop: {e}")
-
-    async def _check_health(self) -> None:
-        """执行健康检查并输出报告"""
-        try:
-            now_time = datetime.now()
-            self.stats['last_check_time'] = now_time
-            
-            # 计算基本统计信息
-            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
-            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
-            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
-            
-            # 计算错误率
-            error_rate = (
-                self.stats['error_responses'] / self.stats['total_responses'] 
-                if self.stats['total_responses'] > 0 else 0
-            )
-            
-            # 输出健康报告
-            health_report = {
-                'runtime_seconds': round(runtime, 2),
-                'total_requests': self.stats['total_requests'],
-                'total_responses': self.stats['total_responses'],
-                'requests_per_second': round(requests_per_second, 2),
-                'responses_per_second': round(responses_per_second, 2),
-                'error_responses': self.stats['error_responses'],
-                'error_rate': f"{error_rate:.2%}",
-            }
-            
-            # 根据错误率判断健康状态
-            if error_rate > 0.1:  # 错误率超过10%
-                self.logger.warning(f"Health check report: {health_report}")
-            elif error_rate > 0.05:  # 错误率超过5%
-                self.logger.info(f"Health check report: {health_report}")
-            else:
-                self.logger.debug(f"Health check report: {health_report}")
-                
-        except Exception as e:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from datetime import datetime
+from typing import Any, Optional, Dict
+
+from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+from crawlo.utils.log import get_logger
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.response_received, event=response_received)
+            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
             self.logger.error(f"Error in health check: {e}")