crawlo 1.1.4__py3-none-any.whl → 1.1.6__py3-none-any.whl

crawlo/extension/__init__.py

@@ -1,38 +1,38 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List, Any
-from pprint import pformat
-
-from crawlo.utils.log import get_logger
-from crawlo.project import load_class
-from crawlo.exceptions import ExtensionInitError
-
-
-class ExtensionManager(object):
-
-    def __init__(self, crawler: Any):
-        self.crawler = crawler
-        self.extensions: List = []
-        extensions = self.crawler.settings.get_list('EXTENSIONS')
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        self._add_extensions(extensions)
-
-    @classmethod
-    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
-        return cls(*args, **kwargs)
-
-    def _add_extensions(self, extensions: List[str]) -> None:
-        for extension_path in extensions:
-            try:
-                extension_cls = load_class(extension_path)
-                if not hasattr(extension_cls, 'create_instance'):
-                    raise ExtensionInitError(
-                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
-                    )
-                self.extensions.append(extension_cls.create_instance(self.crawler))
-            except Exception as e:
-                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
-                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
-        
-        if extensions:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List, Any
+from pprint import pformat
+
+from crawlo.utils.log import get_logger
+from crawlo.project import load_class
+from crawlo.exceptions import ExtensionInitError
+
+
+class ExtensionManager(object):
+
+    def __init__(self, crawler: Any):
+        self.crawler = crawler
+        self.extensions: List = []
+        extensions = self.crawler.settings.get_list('EXTENSIONS')
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        self._add_extensions(extensions)
+
+    @classmethod
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
+        return cls(*args, **kwargs)
+
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_class(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
+        if extensions:
             self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")

crawlo/extension/health_check.py

@@ -1,142 +1,142 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import Any, Optional, Dict
-from datetime import datetime, timedelta
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
-
-
-class HealthCheckExtension:
-    """
-    健康检查扩展
-    监控爬虫的健康状态，包括响应时间、错误率等指标
-    """
-
-    def __init__(self, crawler: Any):
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-        
-        # 获取配置参数
-        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
-        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
-        
-        # 健康状态统计
-        self.stats: Dict[str, Any] = {
-            'start_time': None,
-            'total_requests': 0,
-            'total_responses': 0,
-            'error_responses': 0,
-            'last_check_time': None,
-            'response_times': [],  # 存储最近的响应时间
-        }
-        
-        self.task: Optional[asyncio.Task] = None
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
-        # 只有当配置启用时才创建实例
-        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
-            from crawlo.exceptions import NotConfigured
-            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
-        
-        o = cls(crawler)
-        if o.enabled:
-            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-            crawler.subscriber.subscribe(o.response_received, event=response_received)
-            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
-        return o
-
-    async def spider_opened(self) -> None:
-        """爬虫启动时初始化健康检查"""
-        if not self.enabled:
-            return
-            
-        self.stats['start_time'] = datetime.now()
-        self.task = asyncio.create_task(self._health_check_loop())
-        self.logger.info("Health check extension started.")
-
-    async def spider_closed(self) -> None:
-        """爬虫关闭时停止健康检查"""
-        if not self.enabled:
-            return
-            
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-        
-        # 输出最终健康状态
-        await self._check_health()
-        self.logger.info("Health check extension stopped.")
-
-    async def request_scheduled(self, request: Any, spider: Any) -> None:
-        """记录调度的请求"""
-        if not self.enabled:
-            return
-        self.stats['total_requests'] += 1
-
-    async def response_received(self, response: Any, spider: Any) -> None:
-        """记录接收到的响应"""
-        if not self.enabled:
-            return
-            
-        self.stats['total_responses'] += 1
-        
-        # 记录错误响应
-        if hasattr(response, 'status_code') and response.status_code >= 400:
-            self.stats['error_responses'] += 1
-
-    async def _health_check_loop(self) -> None:
-        """健康检查循环"""
-        while True:
-            try:
-                await asyncio.sleep(self.check_interval)
-                await self._check_health()
-            except asyncio.CancelledError:
-                break
-            except Exception as e:
-                self.logger.error(f"Error in health check loop: {e}")
-
-    async def _check_health(self) -> None:
-        """执行健康检查并输出报告"""
-        try:
-            now_time = datetime.now()
-            self.stats['last_check_time'] = now_time
-            
-            # 计算基本统计信息
-            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
-            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
-            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
-            
-            # 计算错误率
-            error_rate = (
-                self.stats['error_responses'] / self.stats['total_responses'] 
-                if self.stats['total_responses'] > 0 else 0
-            )
-            
-            # 输出健康报告
-            health_report = {
-                'runtime_seconds': round(runtime, 2),
-                'total_requests': self.stats['total_requests'],
-                'total_responses': self.stats['total_responses'],
-                'requests_per_second': round(requests_per_second, 2),
-                'responses_per_second': round(responses_per_second, 2),
-                'error_responses': self.stats['error_responses'],
-                'error_rate': f"{error_rate:.2%}",
-            }
-            
-            # 根据错误率判断健康状态
-            if error_rate > 0.1:  # 错误率超过10%
-                self.logger.warning(f"Health check report: {health_report}")
-            elif error_rate > 0.05:  # 错误率超过5%
-                self.logger.info(f"Health check report: {health_report}")
-            else:
-                self.logger.debug(f"Health check report: {health_report}")
-                
-        except Exception as e:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional, Dict
+from datetime import datetime, timedelta
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.response_received, event=response_received)
+            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
             self.logger.error(f"Error in health check: {e}")

crawlo/extension/log_interval.py

@@ -1,58 +1,58 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from typing import Any, Optional
-
-from crawlo.utils.log import get_logger
-from crawlo.event import spider_opened, spider_closed
-
-
-class LogIntervalExtension(object):
-
-    def __init__(self, crawler: Any):
-        self.task: Optional[asyncio.Task] = None
-        self.stats = crawler.stats
-        self.item_count = 0
-        self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
-        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
-        self.interval = "" if self.interval == 1 else self.interval
-        self.unit = 'min' if self.seconds % 60 == 0 else 's'
-
-        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
-        o = cls(crawler)
-        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
-        return o
-
-    async def spider_opened(self) -> None:
-        self.task = asyncio.create_task(self.interval_log())
-
-    async def spider_closed(self) -> None:
-        if self.task:
-            self.task.cancel()
-            try:
-                await self.task
-            except asyncio.CancelledError:
-                pass
-            self.task = None
-
-    async def interval_log(self) -> None:
-        while True:
-            try:
-                last_item_count = self.stats.get_value('item_successful_count', default=0)
-                last_response_count = self.stats.get_value('response_received_count', default=0)
-                item_rate = last_item_count - self.item_count
-                response_rate = last_response_count - self.response_count
-                self.item_count, self.response_count = last_item_count, last_response_count
-                self.logger.info(
-                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-                )
-                await asyncio.sleep(self.seconds)
-            except Exception as e:
-                self.logger.error(f"Error in interval logging: {e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class LogIntervalExtension(object):
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.stats = crawler.stats
+        self.item_count = 0
+        self.response_count = 0
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
+        self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
+        self.interval = "" if self.interval == 1 else self.interval
+        self.unit = 'min' if self.seconds % 60 == 0 else 's'
+
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        self.task = asyncio.create_task(self.interval_log())
+
+    async def spider_closed(self) -> None:
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+
+    async def interval_log(self) -> None:
+        while True:
+            try:
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                self.item_count, self.response_count = last_item_count, last_response_count
+                self.logger.info(
+                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+                )
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
                 await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,82 +1,82 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import Any
-
-from crawlo import event
-from crawlo.utils.date_tools import now, time_diff
-
-
-class LogStats(object):
-
-    def __init__(self, stats: Any):
-        self._stats = stats
-
-    @classmethod
-    def create_instance(cls, crawler: Any) -> 'LogStats':
-        o = cls(crawler.stats)
-        # 订阅所有需要的事件
-        event_subscriptions = [
-            (o.spider_opened, event.spider_opened),
-            (o.spider_closed, event.spider_closed),
-            (o.item_successful, event.item_successful),
-            (o.item_discard, event.item_discard),
-            (o.response_received, event.response_received),
-            (o.request_scheduled, event.request_scheduled),
-        ]
-        
-        for handler, evt in event_subscriptions:
-            try:
-                crawler.subscriber.subscribe(handler, event=evt)
-            except Exception as e:
-                # 获取日志记录器并记录错误
-                from crawlo.utils.log import get_logger
-                logger = get_logger(cls.__name__)
-                logger.error(f"Failed to subscribe to event {evt}: {e}")
-
-        return o
-
-    async def spider_opened(self) -> None:
-        try:
-            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def spider_closed(self) -> None:
-        try:
-            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_successful(self, _item: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_successful_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('item_discard_count')
-            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
-            if reason:
-                self._stats.inc_value(f"item_discard/{reason}")
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def response_received(self, _response: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('response_received_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
-            pass
-
-    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
-        try:
-            self._stats.inc_value('request_scheduler_count')
-        except Exception as e:
-            # 静默处理，避免影响爬虫运行
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Any
+
+from crawlo import event
+from crawlo.utils.date_tools import now, time_diff
+
+
+class LogStats(object):
+
+    def __init__(self, stats: Any):
+        self._stats = stats
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'LogStats':
+        o = cls(crawler.stats)
+        # 订阅所有需要的事件
+        event_subscriptions = [
+            (o.spider_opened, event.spider_opened),
+            (o.spider_closed, event.spider_closed),
+            (o.item_successful, event.item_successful),
+            (o.item_discard, event.item_discard),
+            (o.response_received, event.response_received),
+            (o.request_scheduled, event.request_scheduled),
+        ]
+        
+        for handler, evt in event_subscriptions:
+            try:
+                crawler.subscriber.subscribe(handler, event=evt)
+            except Exception as e:
+                # 获取日志记录器并记录错误
+                from crawlo.utils.log import get_logger
+                logger = get_logger(cls.__name__)
+                logger.error(f"Failed to subscribe to event {evt}: {e}")
+
+        return o
+
+    async def spider_opened(self) -> None:
+        try:
+            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def spider_closed(self) -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_discard_count')
+            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
+            if reason:
+                self._stats.inc_value(f"item_discard/{reason}")
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
+
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
             pass