crawlo 1.1.2__py3-none-any.whl → 1.1.4__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.2"
+__version__ = "1.1.4"

crawlo/core/scheduler.py

@@ -34,21 +34,25 @@ class Scheduler:
 
     async def open(self):
         """初始化调度器和队列"""
-        # 创建队列配置
-        queue_config = QueueConfig.from_settings(self.crawler.settings)
-        
-        # 创建队列管理器
-        self.queue_manager = QueueManager(queue_config)
-        
-        # 初始化队列
-        success = await self.queue_manager.initialize()
-        if not success:
-            raise RuntimeError("队列初始化失败")
-        
-        # 输出队列状态
-        status = self.queue_manager.get_status()
-        self.logger.info(f'队列类型: {status["type"]}, 状态: {status["health"]}')
-        self.logger.info(f'requesting filter: {self.dupe_filter}')
+        try:
+            # 创建队列配置
+            queue_config = QueueConfig.from_settings(self.crawler.settings)
+            
+            # 创建队列管理器
+            self.queue_manager = QueueManager(queue_config)
+            
+            # 初始化队列
+            success = await self.queue_manager.initialize()
+            if not success:
+                raise RuntimeError("队列初始化失败")
+            
+            # 输出队列状态
+            status = self.queue_manager.get_status()
+            self.logger.info(f'队列类型: {status["type"]}, 状态: {status["health"]}')
+            self.logger.info(f'requesting filter: {self.dupe_filter}')
+        except Exception as e:
+            self.logger.error(f"❌ 调度器初始化失败: {e}")
+            raise
 
     async def next_request(self):
         """获取下一个请求"""
@@ -159,4 +163,4 @@ class Scheduler:
 #             await closed()
 #
 #     def __len__(self):
-#         return self.request_queue.qsize()
+#         return self.request_queue.qsize()

crawlo/downloader/httpx_downloader.py

@@ -26,6 +26,7 @@ except ImportError:
 # 定义我们认为是网络问题，应该触发降级的异常
 NETWORK_EXCEPTIONS = (ConnectError, TimeoutException, NetworkError)
 
+
 class HttpXDownloader(DownloaderBase):
     """
     基于 httpx 的高性能异步下载器
@@ -66,15 +67,15 @@ class HttpXDownloader(DownloaderBase):
         self._client_timeout = Timeout(
             connect=10.0,  # 建立连接超时
             read=timeout_total - 10.0 if timeout_total > 10 else timeout_total / 2,  # 读取数据超时
-            write=10.0,   # 发送数据超时
-            pool=1.0      # 从连接池获取连接的超时
+            write=10.0,  # 发送数据超时
+            pool=1.0  # 从连接池获取连接的超时
         )
         self._client_limits = Limits(
             max_connections=pool_limit,
             max_keepalive_connections=pool_per_host
         )
         self._client_verify = self.crawler.settings.get_bool("VERIFY_SSL", True)
-        self._client_http2 = True # 启用 HTTP/2 支持
+        self._client_http2 = True  # 启用 HTTP/2 支持
         # ----------------------------
 
         # 创建持久化客户端 (不在此处设置全局代理)
@@ -102,7 +103,7 @@ class HttpXDownloader(DownloaderBase):
         # --- 1. 确定要使用的 client 实例 ---
         effective_client = self._client  # 默认使用共享的主 client
         temp_client = None  # 用于可能创建的临时 client
-        used_proxy = None # 记录当前尝试使用的代理
+        used_proxy = None  # 记录当前尝试使用的代理
 
         try:
             # --- 2. 构造发送参数 (不包含 proxy/proxies) ---
@@ -123,7 +124,7 @@ class HttpXDownloader(DownloaderBase):
                 kwargs["content"] = request.body  # 使用 content 而不是 data
 
             # --- 3. 处理代理 ---
-            httpx_proxy_config = None # 用于初始化临时 client 的代理配置
+            httpx_proxy_config = None  # 用于初始化临时 client 的代理配置
             if request.proxy:
                 # 根据 request.proxy 的类型准备 httpx 的 proxy 参数
                 if isinstance(request.proxy, str):
@@ -156,14 +157,15 @@ class HttpXDownloader(DownloaderBase):
                             limits=self._client_limits,
                             verify=self._client_verify,
                             http2=self._client_http2,
-                            follow_redirects=True, # 确保继承
-                            proxy=httpx_proxy_config, # 设置代理
+                            follow_redirects=True,  # 确保继承
+                            proxy=httpx_proxy_config,  # 设置代理
                         )
                         effective_client = temp_client
-                        used_proxy = httpx_proxy_config # 记录使用的代理
+                        used_proxy = httpx_proxy_config  # 记录使用的代理
                         self.logger.debug(f"Using temporary client with proxy: {httpx_proxy_config} for {request.url}")
                     except Exception as e:
-                        self.logger.error(f"Failed to create temporary client with proxy {httpx_proxy_config} for {request.url}: {e}")
+                        self.logger.error(
+                            f"Failed to create temporary client with proxy {httpx_proxy_config} for {request.url}: {e}")
                         # 出错则回退到使用主 client（无代理）
                         # 可以选择抛出异常或继续
                         # raise # 如果希望代理失败导致请求失败，取消注释
@@ -181,7 +183,7 @@ class HttpXDownloader(DownloaderBase):
                     )
                     # 关闭失败的临时客户端
                     await temp_client.aclose()
-                    temp_client = None # 防止 finally 再次关闭
+                    temp_client = None  # 防止 finally 再次关闭
 
                     # 切换到主客户端（直连）
                     effective_client = self._client
@@ -223,7 +225,7 @@ class HttpXDownloader(DownloaderBase):
             try:
                 error_body = await e.response.aread()
             except Exception:
-                error_body = b"" # 如果读取错误响应体失败，则为空
+                error_body = b""  # 如果读取错误响应体失败，则为空
             return self.structure_response(request=request, response=e.response, body=error_body)
         except Exception as e:
             self.logger.critical(f"Unexpected error for {request.url}: {e}", exc_info=True)
@@ -244,7 +246,7 @@ class HttpXDownloader(DownloaderBase):
         return Response(
             url=str(response.url),  # httpx 的 URL 是对象，需转字符串
             headers=dict(response.headers),
-            status_code=response.status_code, # 注意：使用 status_code
+            status_code=response.status_code,  # 注意：使用 status_code
             body=body,
             request=request
         )

crawlo/exceptions.py

@@ -76,3 +76,7 @@ class SpiderCreationError(Exception):
 class ItemValidationError(Exception):
     """Item 字段验证错误"""
     pass
+
+
+class DropItem(Exception):
+    pass

crawlo/extension/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from typing import List
+from typing import List, Any
 from pprint import pformat
 
 from crawlo.utils.log import get_logger
@@ -10,7 +10,7 @@ from crawlo.exceptions import ExtensionInitError
 
 class ExtensionManager(object):
 
-    def __init__(self, crawler):
+    def __init__(self, crawler: Any):
         self.crawler = crawler
         self.extensions: List = []
         extensions = self.crawler.settings.get_list('EXTENSIONS')
@@ -18,14 +18,21 @@ class ExtensionManager(object):
         self._add_extensions(extensions)
 
     @classmethod
-    def create_instance(cls, *args, **kwargs):
+    def create_instance(cls, *args: Any, **kwargs: Any) -> 'ExtensionManager':
         return cls(*args, **kwargs)
 
-    def _add_extensions(self, extensions):
-        for extension in extensions:
-            extension_cls = load_class(extension)
-            if not hasattr(extension_cls, 'create_instance'):
-                raise ExtensionInitError(f"extension init failed, Must have method 'create_instance()")
-            self.extensions.append(extension_cls.create_instance(self.crawler))
+    def _add_extensions(self, extensions: List[str]) -> None:
+        for extension_path in extensions:
+            try:
+                extension_cls = load_class(extension_path)
+                if not hasattr(extension_cls, 'create_instance'):
+                    raise ExtensionInitError(
+                        f"Extension '{extension_path}' init failed: Must have method 'create_instance()'"
+                    )
+                self.extensions.append(extension_cls.create_instance(self.crawler))
+            except Exception as e:
+                self.logger.error(f"Failed to load extension '{extension_path}': {e}")
+                raise ExtensionInitError(f"Failed to load extension '{extension_path}': {e}")
+        
         if extensions:
-            self.logger.info(f"enabled extensions: \n {pformat(extensions)}")
+            self.logger.info(f"Enabled extensions: \n{pformat(extensions)}")

crawlo/extension/health_check.py

@@ -0,0 +1,142 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from typing import Any, Optional, Dict
+from datetime import datetime, timedelta
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed, response_received, request_scheduled
+
+
+class HealthCheckExtension:
+    """
+    健康检查扩展
+    监控爬虫的健康状态，包括响应时间、错误率等指标
+    """
+
+    def __init__(self, crawler: Any):
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.enabled = self.settings.get_bool('HEALTH_CHECK_ENABLED', True)
+        self.check_interval = self.settings.get_int('HEALTH_CHECK_INTERVAL', 60)  # 默认60秒
+        
+        # 健康状态统计
+        self.stats: Dict[str, Any] = {
+            'start_time': None,
+            'total_requests': 0,
+            'total_responses': 0,
+            'error_responses': 0,
+            'last_check_time': None,
+            'response_times': [],  # 存储最近的响应时间
+        }
+        
+        self.task: Optional[asyncio.Task] = None
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'HealthCheckExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('HEALTH_CHECK_ENABLED', True):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("HealthCheckExtension: HEALTH_CHECK_ENABLED is False")
+        
+        o = cls(crawler)
+        if o.enabled:
+            crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+            crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+            crawler.subscriber.subscribe(o.response_received, event=response_received)
+            crawler.subscriber.subscribe(o.request_scheduled, event=request_scheduled)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时初始化健康检查"""
+        if not self.enabled:
+            return
+            
+        self.stats['start_time'] = datetime.now()
+        self.task = asyncio.create_task(self._health_check_loop())
+        self.logger.info("Health check extension started.")
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止健康检查"""
+        if not self.enabled:
+            return
+            
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+        
+        # 输出最终健康状态
+        await self._check_health()
+        self.logger.info("Health check extension stopped.")
+
+    async def request_scheduled(self, request: Any, spider: Any) -> None:
+        """记录调度的请求"""
+        if not self.enabled:
+            return
+        self.stats['total_requests'] += 1
+
+    async def response_received(self, response: Any, spider: Any) -> None:
+        """记录接收到的响应"""
+        if not self.enabled:
+            return
+            
+        self.stats['total_responses'] += 1
+        
+        # 记录错误响应
+        if hasattr(response, 'status_code') and response.status_code >= 400:
+            self.stats['error_responses'] += 1
+
+    async def _health_check_loop(self) -> None:
+        """健康检查循环"""
+        while True:
+            try:
+                await asyncio.sleep(self.check_interval)
+                await self._check_health()
+            except asyncio.CancelledError:
+                break
+            except Exception as e:
+                self.logger.error(f"Error in health check loop: {e}")
+
+    async def _check_health(self) -> None:
+        """执行健康检查并输出报告"""
+        try:
+            now_time = datetime.now()
+            self.stats['last_check_time'] = now_time
+            
+            # 计算基本统计信息
+            runtime = (now_time - self.stats['start_time']).total_seconds() if self.stats['start_time'] else 0
+            requests_per_second = self.stats['total_requests'] / runtime if runtime > 0 else 0
+            responses_per_second = self.stats['total_responses'] / runtime if runtime > 0 else 0
+            
+            # 计算错误率
+            error_rate = (
+                self.stats['error_responses'] / self.stats['total_responses'] 
+                if self.stats['total_responses'] > 0 else 0
+            )
+            
+            # 输出健康报告
+            health_report = {
+                'runtime_seconds': round(runtime, 2),
+                'total_requests': self.stats['total_requests'],
+                'total_responses': self.stats['total_responses'],
+                'requests_per_second': round(requests_per_second, 2),
+                'responses_per_second': round(responses_per_second, 2),
+                'error_responses': self.stats['error_responses'],
+                'error_rate': f"{error_rate:.2%}",
+            }
+            
+            # 根据错误率判断健康状态
+            if error_rate > 0.1:  # 错误率超过10%
+                self.logger.warning(f"Health check report: {health_report}")
+            elif error_rate > 0.05:  # 错误率超过5%
+                self.logger.info(f"Health check report: {health_report}")
+            else:
+                self.logger.debug(f"Health check report: {health_report}")
+                
+        except Exception as e:
+            self.logger.error(f"Error in health check: {e}")

crawlo/extension/log_interval.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+from typing import Any, Optional
 
 from crawlo.utils.log import get_logger
 from crawlo.event import spider_opened, spider_closed
@@ -8,12 +9,12 @@ from crawlo.event import spider_opened, spider_closed
 
 class LogIntervalExtension(object):
 
-    def __init__(self, crawler):
-        self.task = None
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
         self.stats = crawler.stats
         self.item_count = 0
         self.response_count = 0
-        self.seconds = crawler.settings.get('INTERVAL')
+        self.seconds = crawler.settings.get('INTERVAL', 60)  # 默认60秒
         self.interval = int(self.seconds / 60) if self.seconds % 60 == 0 else self.seconds
         self.interval = "" if self.interval == 1 else self.interval
         self.unit = 'min' if self.seconds % 60 == 0 else 's'
@@ -21,29 +22,37 @@ class LogIntervalExtension(object):
         self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
 
     @classmethod
-    def create_instance(cls, crawler):
+    def create_instance(cls, crawler: Any) -> 'LogIntervalExtension':
         o = cls(crawler)
         crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
         crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
         return o
 
-    async def spider_opened(self):
+    async def spider_opened(self) -> None:
         self.task = asyncio.create_task(self.interval_log())
-        await self.task
 
-    async def spider_closed(self):
+    async def spider_closed(self) -> None:
         if self.task:
             self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
 
-    async def interval_log(self):
+    async def interval_log(self) -> None:
         while True:
-            last_item_count = self.stats.get_value('item_successful_count', default=0)
-            last_response_count = self.stats.get_value('response_received_count', default=0)
-            item_rate = last_item_count - self.item_count
-            response_rate = last_response_count - self.response_count
-            self.item_count, self.response_count = last_item_count, last_response_count
-            self.logger.info(
-                f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
-                f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
-            )
-            await asyncio.sleep(self.seconds)
+            try:
+                last_item_count = self.stats.get_value('item_successful_count', default=0)
+                last_response_count = self.stats.get_value('response_received_count', default=0)
+                item_rate = last_item_count - self.item_count
+                response_rate = last_response_count - self.response_count
+                self.item_count, self.response_count = last_item_count, last_response_count
+                self.logger.info(
+                    f'Crawled {last_response_count} pages (at {response_rate} pages/{self.interval}{self.unit}),'
+                    f' Got {last_item_count} items (at {item_rate} items/{self.interval}{self.unit}).'
+                )
+                await asyncio.sleep(self.seconds)
+            except Exception as e:
+                self.logger.error(f"Error in interval logging: {e}")
+                await asyncio.sleep(self.seconds)  # 即使出错也继续执行

crawlo/extension/log_stats.py

@@ -1,44 +1,82 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+from typing import Any
+
 from crawlo import event
 from crawlo.utils.date_tools import now, time_diff
 
 
 class LogStats(object):
 
-    def __init__(self, stats):
+    def __init__(self, stats: Any):
         self._stats = stats
 
     @classmethod
-    def create_instance(cls, crawler):
+    def create_instance(cls, crawler: Any) -> 'LogStats':
         o = cls(crawler.stats)
-        crawler.subscriber.subscribe(o.spider_opened, event=event.spider_opened)
-        crawler.subscriber.subscribe(o.spider_closed, event=event.spider_closed)
-        crawler.subscriber.subscribe(o.item_successful, event=event.item_successful)
-        crawler.subscriber.subscribe(o.item_discard, event=event.item_discard)
-        crawler.subscriber.subscribe(o.response_received, event=event.response_received)
-        crawler.subscriber.subscribe(o.request_scheduled, event=event.request_scheduled)
+        # 订阅所有需要的事件
+        event_subscriptions = [
+            (o.spider_opened, event.spider_opened),
+            (o.spider_closed, event.spider_closed),
+            (o.item_successful, event.item_successful),
+            (o.item_discard, event.item_discard),
+            (o.response_received, event.response_received),
+            (o.request_scheduled, event.request_scheduled),
+        ]
+        
+        for handler, evt in event_subscriptions:
+            try:
+                crawler.subscriber.subscribe(handler, event=evt)
+            except Exception as e:
+                # 获取日志记录器并记录错误
+                from crawlo.utils.log import get_logger
+                logger = get_logger(cls.__name__)
+                logger.error(f"Failed to subscribe to event {evt}: {e}")
 
         return o
 
-    async def spider_opened(self):
-        self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+    async def spider_opened(self) -> None:
+        try:
+            self._stats['start_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def spider_closed(self):
-        self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
-        self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+    async def spider_closed(self) -> None:
+        try:
+            self._stats['end_time'] = now(fmt='%Y-%m-%d %H:%M:%S')
+            self._stats['cost_time(s)'] = time_diff(start=self._stats['start_time'], end=self._stats['end_time'])
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def item_successful(self, _item, _spider):
-        self._stats.inc_value('item_successful_count')
+    async def item_successful(self, _item: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_successful_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def item_discard(self, _item, exc, _spider):
-        self._stats.inc_value('item_discard_count')
-        reason = exc.msg
-        if reason:
-            self._stats.inc_value(f"item_discard/{reason}")
+    async def item_discard(self, _item: Any, exc: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('item_discard_count')
+            reason = getattr(exc, 'msg', None)  # 更安全地获取属性
+            if reason:
+                self._stats.inc_value(f"item_discard/{reason}")
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def response_received(self, _response, _spider):
-        self._stats.inc_value('response_received_count')
+    async def response_received(self, _response: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('response_received_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass
 
-    async def request_scheduled(self, _request, _spider):
-        self._stats.inc_value('request_scheduler_count')
+    async def request_scheduled(self, _request: Any, _spider: Any) -> None:
+        try:
+            self._stats.inc_value('request_scheduler_count')
+        except Exception as e:
+            # 静默处理，避免影响爬虫运行
+            pass

crawlo/extension/logging_extension.py

@@ -1,3 +1,4 @@
+from typing import Any
 from crawlo.exceptions import NotConfigured
 from crawlo.utils.log import get_logger
 from crawlo.utils.log import LoggerManager
@@ -9,27 +10,35 @@ class CustomLoggerExtension:
     遵循与 ExtensionManager 一致的接口规范：使用 create_instance
     """
 
-    def __init__(self, settings):
+    def __init__(self, settings: Any):
         self.settings = settings
         # 初始化全局日志配置
         LoggerManager.configure(settings)
 
     @classmethod
-    def create_instance(cls, crawler, *args, **kwargs):
+    def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
         """
         工厂方法：兼容 ExtensionManager 的创建方式
         被 ExtensionManager 调用
         """
         # 可以通过 settings 控制是否启用
-        if not crawler.settings.get('LOG_FILE') and not crawler.settings.get('LOG_ENABLE_CUSTOM'):
+        log_file = crawler.settings.get('LOG_FILE')
+        log_enable_custom = crawler.settings.get('LOG_ENABLE_CUSTOM', False)
+        
+        # 只有当没有配置日志文件且未启用自定义日志时才禁用
+        if not log_file and not log_enable_custom:
             raise NotConfigured("CustomLoggerExtension: LOG_FILE not set and LOG_ENABLE_CUSTOM=False")
 
         return cls(crawler.settings)
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Any) -> None:
         logger = get_logger(__name__)
-        logger.info(
-            f"CustomLoggerExtension: Logging initialized. "
-            f"LOG_FILE={self.settings.get('LOG_FILE')}, "
-            f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
-        )
+        try:
+            logger.info(
+                f"CustomLoggerExtension: Logging initialized. "
+                f"LOG_FILE={self.settings.get('LOG_FILE')}, "
+                f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"
+            )
+        except Exception as e:
+            # 即使日志初始化信息无法打印，也不应该影响程序运行
+            pass

crawlo/extension/memory_monitor.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+import psutil
+from typing import Any, Optional
+
+from crawlo.utils.log import get_logger
+from crawlo.event import spider_opened, spider_closed
+
+
+class MemoryMonitorExtension:
+    """
+    内存监控扩展
+    定期监控爬虫进程的内存使用情况，并在超出阈值时发出警告
+    """
+
+    def __init__(self, crawler: Any):
+        self.task: Optional[asyncio.Task] = None
+        self.process = psutil.Process()
+        self.settings = crawler.settings
+        self.logger = get_logger(self.__class__.__name__, crawler.settings.get('LOG_LEVEL'))
+        
+        # 获取配置参数
+        self.interval = self.settings.get_int('MEMORY_MONITOR_INTERVAL', 60)  # 默认60秒检查一次
+        self.warning_threshold = self.settings.get_float('MEMORY_WARNING_THRESHOLD', 80.0)  # 默认80%警告阈值
+        self.critical_threshold = self.settings.get_float('MEMORY_CRITICAL_THRESHOLD', 90.0)  # 默认90%严重阈值
+
+    @classmethod
+    def create_instance(cls, crawler: Any) -> 'MemoryMonitorExtension':
+        # 只有当配置启用时才创建实例
+        if not crawler.settings.get_bool('MEMORY_MONITOR_ENABLED', False):
+            from crawlo.exceptions import NotConfigured
+            raise NotConfigured("MemoryMonitorExtension: MEMORY_MONITOR_ENABLED is False")
+        
+        o = cls(crawler)
+        crawler.subscriber.subscribe(o.spider_opened, event=spider_opened)
+        crawler.subscriber.subscribe(o.spider_closed, event=spider_closed)
+        return o
+
+    async def spider_opened(self) -> None:
+        """爬虫启动时开始监控"""
+        self.task = asyncio.create_task(self._monitor_loop())
+        self.logger.info(
+            f"Memory monitor started. Interval: {self.interval}s, "
+            f"Warning threshold: {self.warning_threshold}%, Critical threshold: {self.critical_threshold}%"
+        )
+
+    async def spider_closed(self) -> None:
+        """爬虫关闭时停止监控"""
+        if self.task:
+            self.task.cancel()
+            try:
+                await self.task
+            except asyncio.CancelledError:
+                pass
+            self.task = None
+            self.logger.info("Memory monitor stopped.")
+
+    async def _monitor_loop(self) -> None:
+        """内存监控循环"""
+        while True:
+            try:
+                # 获取内存使用信息
+                memory_info = self.process.memory_info()
+                memory_percent = self.process.memory_percent()
+                
+                # 记录内存使用情况
+                self.logger.debug(
+                    f"Memory usage: {memory_percent:.2f}% "
+                    f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB, "
+                    f"VMS: {memory_info.vms / 1024 / 1024:.2f} MB)"
+                )
+                
+                # 检查是否超过阈值
+                if memory_percent >= self.critical_threshold:
+                    self.logger.critical(
+                        f"Memory usage critical: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                elif memory_percent >= self.warning_threshold:
+                    self.logger.warning(
+                        f"Memory usage high: {memory_percent:.2f}% "
+                        f"(RSS: {memory_info.rss / 1024 / 1024:.2f} MB)"
+                    )
+                
+                await asyncio.sleep(self.interval)
+            except Exception as e:
+                self.logger.error(f"Error in memory monitoring: {e}")
+                await asyncio.sleep(self.interval)