crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/filters/aioredis_filter.py

@@ -16,6 +16,7 @@ from typing import Optional
 from crawlo.filters import BaseFilter
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import request_fingerprint
+from crawlo.utils.redis_connection_pool import get_redis_pool
 
 
 class AioRedisFilter(BaseFilter):
@@ -48,7 +49,7 @@ class AioRedisFilter(BaseFilter):
         初始化Redis过滤器
         
         :param redis_key: Redis中存储指纹的键名
-        :param client: Redis客户端实例
+        :param client: Redis客户端实例（可以为None，稍后初始化）
         :param stats: 统计信息存储
         :param debug: 是否启用调试模式
         :param log_level: 日志级别
@@ -63,6 +64,9 @@ class AioRedisFilter(BaseFilter):
         self.cleanup_fp = cleanup_fp
         self.ttl = ttl
         
+        # 保存连接池引用（用于延迟初始化）
+        self._redis_pool = None
+        
         # 性能计数器
         self._redis_operations = 0
         self._pipeline_operations = 0
@@ -80,17 +84,30 @@ class AioRedisFilter(BaseFilter):
             ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
 
         try:
-            redis_client = aioredis.from_url(
+            # 使用优化的连接池
+            redis_pool = get_redis_pool(
                 redis_url,
-                decode_responses=decode_responses,
                 max_connections=20,
+                socket_connect_timeout=5,
+                socket_timeout=30,
+                health_check_interval=30,
+                retry_on_timeout=True,
+                decode_responses=decode_responses,
                 encoding='utf-8'
             )
+            
+            # 注意：这里不应该使用 await，因为 create_instance 不是异步方法
+            # 我们将在实际使用时获取连接
+            redis_client = None  # 延迟初始化
         except Exception as e:
-            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
+            raise RuntimeError(f"Redis连接池初始化失败: {redis_url} - {str(e)}")
 
-        return cls(
-            redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
+        # 使用统一的Redis key命名规范: crawlo:{project_name}:filter:fingerprint
+        project_name = crawler.settings.get('PROJECT_NAME', 'default')
+        redis_key = f"crawlo:{project_name}:filter:fingerprint"
+
+        instance = cls(
+            redis_key=redis_key,
             client=redis_client,
             stats=crawler.stats,
             cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
@@ -98,6 +115,16 @@ class AioRedisFilter(BaseFilter):
             debug=crawler.settings.get_bool('FILTER_DEBUG', False),
             log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
         )
+        
+        # 保存连接池引用，以便在需要时获取连接
+        instance._redis_pool = redis_pool
+        return instance
+
+    async def _get_redis_client(self):
+        """获取Redis客户端实例（延迟初始化）"""
+        if self.redis is None and self._redis_pool is not None:
+            self.redis = await self._redis_pool.get_connection()
+        return self.redis
 
     async def requested(self, request) -> bool:
         """
@@ -107,6 +134,9 @@ class AioRedisFilter(BaseFilter):
         :return: True 表示重复，False 表示新请求
         """
         try:
+            # 确保Redis客户端已初始化
+            await self._get_redis_client()
+            
             fp = str(request_fingerprint(request))
             self._redis_operations += 1
 
@@ -141,6 +171,9 @@ class AioRedisFilter(BaseFilter):
         :return: 是否成功添加（True 表示新添加，False 表示已存在）
         """
         try:
+            # 确保Redis客户端已初始化
+            await self._get_redis_client()
+            
             fp = str(fp)
             
             # 使用 pipeline 优化性能
@@ -178,6 +211,9 @@ class AioRedisFilter(BaseFilter):
     async def get_stats(self) -> dict:
         """获取过滤器详细统计信息"""
         try:
+            # 确保Redis客户端已初始化
+            await self._get_redis_client()
+            
             count = await self.redis.scard(self.redis_key)
             
             # 获取TTL信息
@@ -212,6 +248,9 @@ class AioRedisFilter(BaseFilter):
     async def clear_all(self) -> int:
         """清空所有指纹数据"""
         try:
+            # 确保Redis客户端已初始化
+            await self._get_redis_client()
+            
             deleted = await self.redis.delete(self.redis_key)
             self.logger.info(f"已清除指纹数: {deleted}")
             return deleted
@@ -222,6 +261,9 @@ class AioRedisFilter(BaseFilter):
     async def closed(self, reason: Optional[str] = None) -> None:
         """爬虫关闭时的清理操作"""
         try:
+            # 确保Redis客户端已初始化
+            await self._get_redis_client()
+            
             if self.cleanup_fp:
                 deleted = await self.redis.delete(self.redis_key)
                 self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
@@ -234,9 +276,5 @@ class AioRedisFilter(BaseFilter):
 
     async def _close_redis(self) -> None:
         """安全关闭Redis连接"""
-        try:
-            if hasattr(self.redis, 'close'):
-                await self.redis.close()
-                self.logger.debug("Redis连接已关闭")
-        except Exception as e:
-            self.logger.warning(f"Redis关闭时出错：{e}")
+        # 连接池会自动管理连接，这里不需要显式关闭
+        self.logger.debug("Redis连接已释放")

crawlo/middleware/proxy.py

@@ -173,7 +173,7 @@ class ProxyMiddleware:
             if isinstance(result, str) and result.strip():
                 return result.strip()
             elif isinstance(result, dict):
-                cleaned = {k: v.strip() for k, v in result.items() if v and isinstance(v, str)}
+                cleaned = {k: v.strip() if isinstance(v, str) else v for k, v in result.items()}
                 return cleaned if cleaned else None
             return None
         except Exception as e:
@@ -225,7 +225,31 @@ class ProxyMiddleware:
 
         proxy = await self._get_cached_proxy()
         if proxy:
-            request.proxy = proxy
+            # 处理带认证的代理URL
+            if isinstance(proxy, str) and "@" in proxy and "://" in proxy:
+                # 解析带认证的代理URL
+                parsed = urlparse(proxy)
+                if parsed.username and parsed.password:
+                    # 对于AioHttp下载器，需要特殊处理认证信息
+                    downloader_type = spider.crawler.settings.get("DOWNLOADER_TYPE", "aiohttp")
+                    if downloader_type == "aiohttp":
+                        # 将认证信息存储在meta中，由下载器处理
+                        request.meta["proxy_auth"] = {
+                            "username": parsed.username,
+                            "password": parsed.password
+                        }
+                        # 清理URL中的认证信息
+                        clean_proxy = f"{parsed.scheme}://{parsed.hostname}"
+                        if parsed.port:
+                            clean_proxy += f":{parsed.port}"
+                        request.proxy = clean_proxy
+                    else:
+                        # 其他下载器可以直接使用带认证的URL
+                        request.proxy = proxy
+                else:
+                    request.proxy = proxy
+            else:
+                request.proxy = proxy
             self.logger.info(f"分配代理 → {proxy} | {request.url}")
         else:
             self.logger.warning(f"未获取到代理，请求直连: {request.url}")

crawlo/mode_manager.py

@@ -63,8 +63,9 @@ class ModeManager:
             'REDIS_PORT': redis_port,
             'REDIS_PASSWORD': redis_password,
             'REDIS_URL': redis_url,
-            'SCHEDULER_QUEUE_NAME': f'{project_name}:requests',
-            'REDIS_KEY': f'{project_name}:fingerprint',
+            'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',  # 使用统一命名规范
+            # Redis key配置已移至各组件中，使用统一的命名规范
+            # crawlo:{project_name}:filter:fingerprint (请求去重)
             'CONCURRENCY': 16,
             'MAX_RUNNING_SPIDERS': 1,
             'DOWNLOAD_DELAY': 1.0,
@@ -181,21 +182,25 @@ def auto_mode(**kwargs) -> Dict[str, Any]:
 # 环境变量支持
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
-    mode = os.getenv('CRAWLO_MODE', default_mode).lower()
+    # 移除直接使用 os.getenv()，要求通过 settings 配置
+    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
     
-    if mode == 'distributed':
-        return distributed_mode(
-            redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
-            redis_port=int(os.getenv('REDIS_PORT', 6379)),
-            redis_password=os.getenv('REDIS_PASSWORD'),
-            project_name=os.getenv('PROJECT_NAME', 'crawlo'),
-            CONCURRENCY=int(os.getenv('CONCURRENCY', 16)),
-        )
-    elif mode == 'auto':
-        return auto_mode(
-            CONCURRENCY=int(os.getenv('CONCURRENCY', 12)),
-        )
-    else:  # standalone
-        return standalone_mode(
-            CONCURRENCY=int(os.getenv('CONCURRENCY', 8)),
-        )
+    # 保留原有代码作为参考
+    # mode = os.getenv('CRAWLO_MODE', default_mode).lower()
+    # 
+    # if mode == 'distributed':
+    #     return distributed_mode(
+    #         redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
+    #         redis_port=int(os.getenv('REDIS_PORT', 6379)),
+    #         redis_password=os.getenv('REDIS_PASSWORD'),
+    #         project_name=os.getenv('PROJECT_NAME', 'crawlo'),
+    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 16)),
+    #     )
+    # elif mode == 'auto':
+    #     return auto_mode(
+    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 12)),
+    #     )
+    # else:  # standalone
+    #     return standalone_mode(
+    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 8)),
+    #     )

crawlo/network/request.py

@@ -76,7 +76,9 @@ class Request:
         'verify',
         'flags',
         '_json_body',
-        '_form_data'
+        '_form_data',
+        'use_dynamic_loader',
+        'dynamic_loader_options'
     )
 
     def __init__(
@@ -99,7 +101,10 @@ class Request:
         auth: Optional[tuple] = None,
         verify: bool = True,
         flags: Optional[List[str]] = None,
-        encoding: str = 'utf-8'
+        encoding: str = 'utf-8',
+        # 动态加载相关参数
+        use_dynamic_loader: bool = False,
+        dynamic_loader_options: Optional[Dict[str, Any]] = None
     ):
         """
         初始化请求对象。
@@ -145,6 +150,10 @@ class Request:
         # 保存高层语义参数（用于 copy）
         self._json_body = json_body
         self._form_data = form_data
+        
+        # 动态加载相关属性
+        self.use_dynamic_loader = use_dynamic_loader
+        self.dynamic_loader_options = dynamic_loader_options or {}
 
         # 构建 body
         if json_body is not None:
@@ -228,7 +237,9 @@ class Request:
             auth=self.auth,
             verify=self.verify,
             flags=self.flags.copy(),
-            encoding=self.encoding
+            encoding=self.encoding,
+            use_dynamic_loader=self.use_dynamic_loader,
+            dynamic_loader_options=deepcopy(self.dynamic_loader_options)
         )
 
     def set_meta(self, key: str, value: Any) -> 'Request':
@@ -267,6 +278,22 @@ class Request:
         if flag in self.flags:
             self.flags.remove(flag)
         return self
+    
+    def set_dynamic_loader(self, use_dynamic: bool = True, options: Optional[Dict[str, Any]] = None) -> 'Request':
+        """设置使用动态加载器，支持链式调用。"""
+        self.use_dynamic_loader = use_dynamic
+        if options:
+            self.dynamic_loader_options = options
+        # 同时在meta中设置标记，供混合下载器使用
+        self._meta['use_dynamic_loader'] = use_dynamic
+        return self
+    
+    def set_protocol_loader(self) -> 'Request':
+        """强制使用协议加载器，支持链式调用。"""
+        self.use_dynamic_loader = False
+        self._meta['use_dynamic_loader'] = False
+        self._meta['use_protocol_loader'] = True
+        return self
 
     def _set_url(self, url: str) -> None:
         """安全设置 URL，确保格式正确。"""

crawlo/network/response.py

@@ -196,50 +196,139 @@ class Response:
         """使用 CSS 选择器查询文档。"""
         return self._selector.css(query)
 
-    def xpath_text(self, query: str) -> str:
-        """使用 XPath 提取并返回纯文本。"""
-        fragments = self.xpath(f"{query}//text()").getall()
-        return " ".join(text.strip() for text in fragments if text.strip())
+    def _is_xpath(self, query: str) -> bool:
+        """判断查询语句是否为XPath"""
+        return query.startswith(('/', '//', './'))
 
-    def css_text(self, query: str) -> str:
-        """使用 CSS 选择器提取并返回纯文本。"""
-        fragments = self.css(f"{query} ::text").getall()
-        return " ".join(text.strip() for text in fragments if text.strip())
-
-    def get_text(self, xpath_or_css: str, join_str: str = " ") -> str:
+    def _extract_text_from_elements(self, elements: SelectorList, join_str: str = " ") -> str:
+        """
+        从元素列表中提取文本并拼接
+        
+        :param elements: SelectorList元素列表
+        :param join_str: 文本拼接分隔符
+        :return: 拼接后的文本
+        """
+        texts = []
+        for element in elements:
+            # 获取元素的所有文本节点
+            if hasattr(element, 'xpath'):
+                element_texts = element.xpath('.//text()').getall()
+            else:
+                element_texts = [str(element)]
+            # 清理并添加非空文本
+            for text in element_texts:
+                cleaned = text.strip()
+                if cleaned:
+                    texts.append(cleaned)
+        return join_str.join(texts)
+
+    def extract_text(self, xpath_or_css: str, join_str: str = " ", default: str = '') -> str:
         """
-        获取指定节点的纯文本(自动拼接子节点文本)
+        提取单个元素的文本内容，支持CSS和XPath选择器
 
         参数:
             xpath_or_css: XPath或CSS选择器
             join_str: 文本拼接分隔符(默认为空格)
+            default: 默认返回值，当未找到元素时返回
 
         返回:
             拼接后的纯文本字符串
         """
-        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
-        texts = elements.xpath('.//text()').getall()
-        return join_str.join(text.strip() for text in texts if text.strip())
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+            return self._extract_text_from_elements(elements, join_str)
+        except Exception:
+            return default
 
-    def get_all_text(self, xpath_or_css: str, join_str: str = " ") -> List[str]:
+    def extract_texts(self, xpath_or_css: str, join_str: str = " ", default: List[str] = None) -> List[str]:
         """
-        获取多个节点的纯文本列表
+        提取多个元素的文本内容列表，支持CSS和XPath选择器
 
         参数:
             xpath_or_css: XPath或CSS选择器
             join_str: 单个节点内文本拼接分隔符
+            default: 默认返回值，当未找到元素时返回
 
         返回:
             纯文本列表(每个元素对应一个节点的文本)
         """
-        elements = self.xpath(xpath_or_css) if xpath_or_css.startswith(('/', '//', './')) else self.css(xpath_or_css)
-        result = []
-        for element in elements:
-            texts = element.xpath('.//text()').getall()
-            clean_text = join_str.join(text.strip() for text in texts if text.strip())
-            if clean_text:
-                result.append(clean_text)
-        return result
+        if default is None:
+            default = []
+            
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+                
+            result = []
+            for element in elements:
+                # 对每个元素提取文本
+                if hasattr(element, 'xpath'):
+                    texts = element.xpath('.//text()').getall()
+                else:
+                    texts = [str(element)]
+                    
+                # 清理文本并拼接
+                clean_texts = [text.strip() for text in texts if text.strip()]
+                if clean_texts:
+                    result.append(join_str.join(clean_texts))
+                    
+            return result if result else default
+        except Exception:
+            return default
+
+    def extract_attr(self, xpath_or_css: str, attr_name: str, default: Any = None) -> Any:
+        """
+        提取单个元素的属性值，支持CSS和XPath选择器
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            attr_name: 属性名称
+            default: 默认返回值
+
+        返回:
+            属性值或默认值
+        """
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+            return elements.attrib.get(attr_name, default)
+        except Exception:
+            return default
+
+    def extract_attrs(self, xpath_or_css: str, attr_name: str, default: List[Any] = None) -> List[Any]:
+        """
+        提取多个元素的属性值列表，支持CSS和XPath选择器
+
+        参数:
+            xpath_or_css: XPath或CSS选择器
+            attr_name: 属性名称
+            default: 默认返回值
+
+        返回:
+            属性值列表
+        """
+        if default is None:
+            default = []
+            
+        try:
+            elements = self.xpath(xpath_or_css) if self._is_xpath(xpath_or_css) else self.css(xpath_or_css)
+            if not elements:
+                return default
+                
+            result = []
+            for element in elements:
+                if hasattr(element, 'attrib'):
+                    attr_value = element.attrib.get(attr_name)
+                    if attr_value is not None:
+                        result.append(attr_value)
+                        
+            return result if result else default
+        except Exception:
+            return default
 
     def re_search(self, pattern: str, flags: int = re.DOTALL) -> Optional[re.Match]:
         """在响应文本上执行正则表达式搜索。"""
@@ -268,4 +357,4 @@ class Response:
         return self.request.meta if self.request else {}
 
     def __str__(self):
-        return f"<{self.status_code} {self.url}>"
+        return f"<{self.status_code} {self.url}>"

crawlo/pipelines/mongo_pipeline.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from typing import Optional
+from typing import Optional, List, Dict
 from motor.motor_asyncio import AsyncIOMotorClient
 from pymongo.errors import PyMongoError
 from crawlo.utils.log import get_logger
@@ -21,6 +21,17 @@ class MongoPipeline:
         self.mongo_uri = self.settings.get('MONGO_URI', 'mongodb://localhost:27017')
         self.db_name = self.settings.get('MONGO_DATABASE', 'scrapy_db')
         self.collection_name = self.settings.get('MONGO_COLLECTION', crawler.spider.name)
+        
+        # 连接池配置
+        self.max_pool_size = self.settings.getint('MONGO_MAX_POOL_SIZE', 100)
+        self.min_pool_size = self.settings.getint('MONGO_MIN_POOL_SIZE', 10)
+        self.connect_timeout_ms = self.settings.getint('MONGO_CONNECT_TIMEOUT_MS', 5000)
+        self.socket_timeout_ms = self.settings.getint('MONGO_SOCKET_TIMEOUT_MS', 30000)
+
+        # 批量插入配置
+        self.batch_size = self.settings.getint('MONGO_BATCH_SIZE', 100)
+        self.use_batch = self.settings.getbool('MONGO_USE_BATCH', False)
+        self.batch_buffer: List[Dict] = []  # 批量缓冲区
 
         # 注册关闭事件
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
@@ -32,86 +43,90 @@ class MongoPipeline:
     async def _ensure_connection(self):
         """确保连接已建立"""
         if self.client is None:
-            self.client = AsyncIOMotorClient(self.mongo_uri)
+            # 使用连接池配置创建客户端
+            self.client = AsyncIOMotorClient(
+                self.mongo_uri,
+                maxPoolSize=self.max_pool_size,
+                minPoolSize=self.min_pool_size,
+                connectTimeoutMS=self.connect_timeout_ms,
+                socketTimeoutMS=self.socket_timeout_ms
+            )
             self.db = self.client[self.db_name]
             self.collection = self.db[self.collection_name]
             self.logger.info(f"MongoDB连接建立 (集合: {self.collection_name})")
 
     async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item的核心方法"""
-        try:
-            await self._ensure_connection()
-
-            item_dict = dict(item)
-            result = await self.collection.insert_one(item_dict)
-
-            # 统计计数
-            self.crawler.stats.inc_value('mongodb/inserted')
-            self.logger.debug(f"插入文档ID: {result.inserted_id}")
-
+        """处理item的核心方法（带重试机制）"""
+        # 如果启用批量插入，将item添加到缓冲区
+        if self.use_batch:
+            self.batch_buffer.append(dict(item))
+            
+            # 如果缓冲区达到批量大小，执行批量插入
+            if len(self.batch_buffer) >= self.batch_size:
+                await self._flush_batch(spider)
+                
             return item
+        else:
+            # 单条插入逻辑
+            try:
+                await self._ensure_connection()
+
+                item_dict = dict(item)
+
+                # 带重试的插入操作
+                for attempt in range(3):
+                    try:
+                        result = await self.collection.insert_one(item_dict)
+                        # 统一使用insert_success统计键名
+                        self.crawler.stats.inc_value('mongodb/insert_success')
+                        self.logger.debug(f"插入成功 [attempt {attempt + 1}]: {result.inserted_id}")
+                        return item
+                    except PyMongoError as e:
+                        if attempt == 2:  # 最后一次尝试仍失败
+                            raise
+                        self.logger.warning(f"插入重试中 [attempt {attempt + 1}]: {e}")
+
+            except Exception as e:
+                # 统一使用insert_failed统计键名
+                self.crawler.stats.inc_value('mongodb/insert_failed')
+                self.logger.error(f"MongoDB操作最终失败: {e}")
+                raise ItemDiscard(f"MongoDB操作失败: {e}")
+
+    async def _flush_batch(self, spider):
+        """刷新批量缓冲区并执行批量插入"""
+        if not self.batch_buffer:
+            return
 
-        except Exception as e:
-            self.crawler.stats.inc_value('mongodb/failed')
-            self.logger.error(f"MongoDB插入失败: {e}")
-            raise ItemDiscard(f"MongoDB操作失败: {e}")
-
-    async def spider_closed(self):
-        """关闭爬虫时清理资源"""
-        if self.client:
-            self.client.close()
-            self.logger.info("MongoDB连接已关闭")
-
-
-class MongoPoolPipeline:
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
-
-        # 连接池配置
-        self.client = AsyncIOMotorClient(
-            self.settings.get('MONGO_URI', 'mongodb://localhost:27017'),
-            maxPoolSize=self.settings.getint('MONGO_MAX_POOL_SIZE', 100),
-            minPoolSize=self.settings.getint('MONGO_MIN_POOL_SIZE', 10),
-            connectTimeoutMS=5000,
-            socketTimeoutMS=30000
-        )
-
-        self.db = self.client[self.settings.get('MONGO_DATABASE', 'scrapy_db')]
-        self.collection = self.db[self.settings.get('MONGO_COLLECTION', crawler.spider.name)]
-
-        crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
-        self.logger.info(f"MongoDB连接池已初始化 (集合: {self.collection.name})")
-
-    @classmethod
-    def create_instance(cls, crawler):
-        return cls(crawler)
-
-    async def process_item(self, item, spider) -> Optional[dict]:
-        """处理item方法（带重试机制）"""
         try:
-            item_dict = dict(item)
+            await self._ensure_connection()
 
-            # 带重试的插入操作
+            # 带重试的批量插入操作
             for attempt in range(3):
                 try:
-                    result = await self.collection.insert_one(item_dict)
-                    self.crawler.stats.inc_value('mongodb/insert_success')
-                    self.logger.debug(f"插入成功 [attempt {attempt + 1}]: {result.inserted_id}")
-                    return item
+                    result = await self.collection.insert_many(self.batch_buffer, ordered=False)
+                    # 统一使用insert_success统计键名
+                    inserted_count = len(result.inserted_ids)
+                    self.crawler.stats.inc_value('mongodb/insert_success', inserted_count)
+                    self.logger.debug(f"批量插入成功 [attempt {attempt + 1}]: {inserted_count} 条记录")
+                    self.batch_buffer.clear()
+                    return
                 except PyMongoError as e:
                     if attempt == 2:  # 最后一次尝试仍失败
                         raise
-                    self.logger.warning(f"插入重试中 [attempt {attempt + 1}]: {e}")
-
+                    self.logger.warning(f"批量插入重试中 [attempt {attempt + 1}]: {e}")
         except Exception as e:
-            self.crawler.stats.inc_value('mongodb/insert_failed')
-            self.logger.error(f"MongoDB操作最终失败: {e}")
-            raise ItemDiscard(f"MongoDB操作失败: {e}")
+            # 统一使用insert_failed统计键名
+            failed_count = len(self.batch_buffer)
+            self.crawler.stats.inc_value('mongodb/insert_failed', failed_count)
+            self.logger.error(f"MongoDB批量插入最终失败: {e}")
+            raise ItemDiscard(f"MongoDB批量插入失败: {e}")
 
     async def spider_closed(self):
-        """资源清理"""
-        if hasattr(self, 'client'):
+        """关闭爬虫时清理资源"""
+        # 在关闭前刷新剩余的批量数据
+        if self.use_batch and self.batch_buffer:
+            await self._flush_batch(self.crawler.spider)
+            
+        if self.client:
             self.client.close()
-            self.logger.info("MongoDB连接池已释放")
+            self.logger.info("MongoDB连接已关闭")