crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/event.py

@@ -1,11 +1,11 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-
-spider_error = "spider_error"
-spider_opened = "spider_open"
-spider_closed = "spider_closed"
-ignore_request = "ignore_request"
-request_scheduled = "request_scheduled"
-response_received = "request_received"
-item_successful = "item_successful"
-item_discard = "item_discard"
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+
+spider_error = "spider_error"
+spider_opened = "spider_open"
+spider_closed = "spider_closed"
+ignore_request = "ignore_request"
+request_scheduled = "request_scheduled"
+response_received = "request_received"
+item_successful = "item_successful"
+item_discard = "item_discard"

crawlo/exceptions.py

@@ -1,82 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-class TransformTypeError(TypeError):
-    pass
-
-
-class OutputError(Exception):
-    pass
-
-
-class SpiderTypeError(TypeError):
-    pass
-
-
-class ItemInitError(Exception):
-    pass
-
-
-class ItemAttributeError(Exception):
-    pass
-
-
-class DecodeError(Exception):
-    pass
-
-
-class MiddlewareInitError(Exception):
-    pass
-
-
-class PipelineInitError(Exception):
-    pass
-
-
-class InvalidOutputError(Exception):
-    pass
-
-
-class RequestMethodError(Exception):
-    pass
-
-
-class IgnoreRequestError(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(IgnoreRequestError, self).__init__(msg)
-
-
-class ItemDiscard(Exception):
-    def __init__(self, msg):
-        self.msg = msg
-        super(ItemDiscard, self).__init__(msg)
-
-
-class NotConfigured(Exception):
-    pass
-
-
-class NotConfiguredError(Exception):
-    pass
-
-
-class ExtensionInitError(Exception):
-    pass
-
-
-class ReceiverTypeError(Exception):
-    pass
-
-
-class SpiderCreationError(Exception):
-    """爬虫实例化失败异常"""
-    pass
-
-
-class ItemValidationError(Exception):
-    """Item 字段验证错误"""
-    pass
-
-
-class DropItem(Exception):
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+class TransformTypeError(TypeError):
+    pass
+
+
+class OutputError(Exception):
+    pass
+
+
+class SpiderTypeError(TypeError):
+    pass
+
+
+class ItemInitError(Exception):
+    pass
+
+
+class ItemAttributeError(Exception):
+    pass
+
+
+class DecodeError(Exception):
+    pass
+
+
+class MiddlewareInitError(Exception):
+    pass
+
+
+class PipelineInitError(Exception):
+    pass
+
+
+class InvalidOutputError(Exception):
+    pass
+
+
+class RequestMethodError(Exception):
+    pass
+
+
+class IgnoreRequestError(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(IgnoreRequestError, self).__init__(msg)
+
+
+class ItemDiscard(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+        super(ItemDiscard, self).__init__(msg)
+
+
+class NotConfigured(Exception):
+    pass
+
+
+class NotConfiguredError(Exception):
+    pass
+
+
+class ExtensionInitError(Exception):
+    pass
+
+
+class ReceiverTypeError(Exception):
+    pass
+
+
+class SpiderCreationError(Exception):
+    """爬虫实例化失败异常"""
+    pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass
+
+
+class DropItem(Exception):
     pass

crawlo/extension/__init__.py

@@ -1,39 +1,39 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List, Any
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.project import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler: Any):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-
-    @classmethod
-    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions: List[str]) -> None:
-        for extension_path in extensions:
-            try:
-                extension_cls = load_class(extension_path)
-                if not hasattr(extension_cls, 'create_instance'):
-                    raise ExtensionInitError(
-                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
-                    )
-                self.extensions.append(extension_cls.create_instance(self.crawler))
-            except Exception as e:
-                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
-                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
-        
-        if extensions:
-            # 恢复INFO级别日志，保留关键的启用信息
-            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List, Any
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.class_loader import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler: Any):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_class(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
+        if extensions:
+            # 恢复INFO级别日志，保留关键的启用信息
+            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")

crawlo/extension/health_check.py

@@ -1,142 +1,142 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from datetime import datetime
-from typing import Any, Optional, Dict
-
-from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
-from crawlo.utils.log import get_logger
-
-
-class HealthCheckExtension:
-    """
-    健康检查扩展
-    监控爬虫的健康状态，包括响应时间、错误率等指标
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
-        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
-        
-        # 健康状态统计
-        self.stats: Dict[str, Any] = {
-            'start_time': None,
-            'total_requests': 0,
-            'total_responses': 0,
-            'error_responses': 0,
-            'last_check_time': None,
-            'response_times': [],  # 存储最近的响应时间
-        }
-        
-        self.task: Optional[asyncio.Task] = None
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-            crawler.subscriber.subscribe(o.response_received, event=response_received)
-            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
-        return o
-
-    async def spider_opened(self) -> None:
-        """爬虫启动时初始化健康检查"""
-        if not self.enabled:
-            return
-            
-        self.stats['start_time'] = datetime.now()
-        self.task = asyncio.create_task(self._health_check_loop())
-        self.logger.info("Health check extension started.")
-
-    async def spider_closed(self) -> None:
-        """爬虫关闭时停止健康检查"""
-        if not self.enabled:
-            return
-            
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-        
-        # 输出最终健康状态
-        await self._check_health()
-        self.logger.info("Health check extension stopped.")
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-        self.stats['total_requests'] += 1
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        self.stats['total_responses'] += 1
-        
-        # 记录错误响应
-        if hasattr(response, 'status_code') and response.status_code >= 400:
-            self.stats['error_responses'] += 1
-
-    async def _health_check_loop(self) -> None:
-        """健康检查循环"""
-        while True:
-            try:
-                await asyncio.sleep(self.check_interval)
-                await self._check_health()
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                self.logger.error(f"Error in health check loop: {e}")
-
-    async def _check_health(self) -> None:
-        """执行健康检查并输出报告"""
-        try:
-            now_time = datetime.now()
-            self.stats['last_check_time'] = now_time
-            
-            # 计算基本统计信息
-            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
-            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
-            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
-            
-            # 计算错误率
-            error_rate = (
-                self.stats['error_responses'] / self.stats['total_responses'] 
-                if self.stats['total_responses'] > 0 else 0
-            )
-            
-            # 输出健康报告
-            health_report = {
-                'runtime_seconds': round(runtime, 2),
-                'total_requests': self.stats['total_requests'],
-                'total_responses': self.stats['total_responses'],
-                'requests_per_second': round(requests_per_second, 2),
-                'responses_per_second': round(responses_per_second, 2),
-                'error_responses': self.stats['error_responses'],
-                'error_rate': f"{error_rate:.2%}",
-            }
-            
-            # 根据错误率判断健康状态
-            if error_rate > 0.1:  # 错误率超过10%
-                self.logger.warning(f"Health check report: {health_report}")
-            elif error_rate > 0.05:  # 错误率超过5%
-                self.logger.info(f"Health check report: {health_report}")
-            else:
-                self.logger.debug(f"Health check report: {health_report}")
-                
-        except Exception as e:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from datetime import datetime
+from typing import Any, Optional, Dict
+
+from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+from crawlo.utils.log import get_logger
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.response_received, event=response_received)
+            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
             self.logger.error(f"Error in health check: {e}")

crawlo/extension/log_interval.py

@@ -1,58 +1,58 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import Any, Optional
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler: Any):
-        self.task: Optional[asyncio.Task] = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
-        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
-        self.interval = "" if self.interval == 1 else self.interval
-        self.unit = 'min' if self.seconds % 60 == 0 else 's'
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self) -> None:
-        self.task = asyncio.create_task(self.interval_log())
-
-    async def spider_closed(self) -> None:
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-            self.task = None
-
-    async def interval_log(self) -> None:
-        while True:
-            try:
-                last_item_count = self.stats.get_value('item_successful_count', default=0)
-                last_response_count = self.stats.get_value('response_received_count', default=0)
-                item_rate = last_item_count - self.item_count
-                response_rate = last_response_count - self.response_count
-                self.item_count, self.response_count = last_item_count, last_response_count
-                self.logger.info(
-                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-                )
-                await asyncio.sleep(self.seconds)
-            except Exception as e:
-                self.logger.error(f"Error in interval logging: {e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.task = asyncio.create_task(self.interval_log())
+
+    async def spider_closed(self) -> None:
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        while True:
+            try:
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                self.item_count, self.response_count = last_item_count, last_response_count
+                self.logger.info(
+                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+                )
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
                 await asyncio.sleep(self.seconds)  # 即使出错也继续执行