crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/filters/aioredis_filter.py

@@ -1,150 +1,242 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import aioredis
-from typing import Optional
-from crawlo import Request
-from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
-
-
-class AioRedisFilter(BaseFilter):
-    """基于Redis集合实现的异步请求去重过滤器（支持分布式爬虫），提供TTL和清理控制"""
-
-    def __init__(
-            self,
-            redis_key: str,
-            client: aioredis.Redis,
-            stats: dict,
-            debug: bool,
-            log_level: str,
-            cleanup_fp: bool = False,
-            ttl: Optional[int] = None
-    ):
-        """初始化过滤器"""
-        self.logger = get_logger(self.__class__.__name__, log_level)
-        super().__init__(self.logger, stats, debug)
-
-        self.redis_key = redis_key
-        self.redis = client
-        self.cleanup_fp = cleanup_fp
-        self.ttl = ttl
-
-    @classmethod
-    def create_instance(cls, crawler) -> 'BaseFilter':
-        """从爬虫配置创建过滤器实例"""
-        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
-        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
-        ttl_setting = crawler.settings.get_int('REDIS_TTL')
-
-        # 处理TTL设置
-        ttl = None
-        if ttl_setting is not None:
-            ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
-
-        try:
-            redis_client = aioredis.from_url(
-                redis_url,
-                decode_responses=decode_responses,
-                max_connections=20,
-                encoding='utf-8'
-            )
-        except Exception as e:
-            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
-
-        return cls(
-            redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
-            client=redis_client,
-            stats=crawler.stats,
-            cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
-            ttl=ttl,
-            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
-            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
-        )
-
-    async def requested(self, request: Request) -> bool:
-        """检查请求是否已存在"""
-        try:
-            fp = str(request_fingerprint(request))
-
-            # 1. 检查指纹是否存在
-            pipe = self.redis.pipeline()
-            pipe.sismember(self.redis_key, fp)  # 不单独 await
-            exists = (await pipe.execute())[0]  # 执行并获取结果
-
-            if exists:  # 如果已存在，返回 True
-                return True
-
-            # 2. 如果不存在，添加指纹并设置 TTL
-            pipe = self.redis.pipeline()
-            pipe.sadd(self.redis_key, fp)  # 不单独 await
-            if self.ttl and self.ttl > 0:
-                pipe.expire(self.redis_key, self.ttl)  # 不单独 await
-            await pipe.execute()  # 一次性执行所有命令
-
-            return False  # 表示是新请求
-
-        except Exception as e:
-            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')}")
-            raise
-
-    async def add_fingerprint(self, fp: str) -> bool:
-        """添加新指纹到Redis集合"""
-        try:
-            fp = str(fp)
-            added = await self.redis.sadd(self.redis_key, fp)
-
-            if self.ttl and self.ttl > 0:
-                await self.redis.expire(self.redis_key, self.ttl)
-
-            return added == 1
-        except Exception as e:
-            self.logger.error("添加指纹失败")
-            raise
-
-    async def get_stats(self) -> dict:
-        """获取过滤器统计信息"""
-        try:
-            count = await self.redis.scard(self.redis_key)
-            stats = {
-                '指纹总数': count,
-                'Redis键名': self.redis_key,
-                'TTL配置': f"{self.ttl}秒" if self.ttl else "持久化"
-            }
-            stats.update(self.stats)
-            return stats
-        except Exception as e:
-            self.logger.error("获取统计信息失败")
-            return self.stats
-
-    async def clear_all(self) -> int:
-        """清空所有指纹数据"""
-        try:
-            deleted = await self.redis.delete(self.redis_key)
-            self.logger.info(f"已清除指纹数: {deleted}")
-            return deleted
-        except Exception as e:
-            self.logger.error("清空指纹失败")
-            raise
-
-    async def closed(self, reason: Optional[str] = None) -> None:
-        """爬虫关闭时的清理操作"""
-        try:
-            if self.cleanup_fp:
-                deleted = await self.redis.delete(self.redis_key)
-                self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
-            else:
-                count = await self.redis.scard(self.redis_key)
-                ttl_info = f"{self.ttl}秒" if self.ttl else "持久化"
-                self.logger.info(f"保留指纹数: {count} (TTL: {ttl_info})")
-        finally:
-            await self._close_redis()
-
-    async def _close_redis(self) -> None:
-        """安全关闭Redis连接"""
-        try:
-            if hasattr(self.redis, 'close'):
-                await self.redis.close()
-                self.logger.debug("Redis连接已关闭")
-        except Exception as e:
-            self.logger.warning(f"Redis关闭时出错：{e}")
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+"""
+Redis 过滤器实现
+=================
+提供基于 Redis 的分布式请求去重功能。
+
+特点:
+- 分布式支持: 多节点共享去重数据
+- TTL 支持: 自动过期清理
+- 高性能: 使用 Redis pipeline 优化
+- 容错设计: 网络异常自动重试
+"""
+from typing import Optional
+from redis import asyncio as aioredis
+from crawlo.filters import BaseFilter
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import request_fingerprint
+
+
+class AioRedisFilter(BaseFilter):
+    """
+    基于Redis集合实现的异步请求去重过滤器
+    
+    支持特性:
+    - 分布式爬虫多节点共享去重数据
+    - TTL 自动过期清理机制
+    - Pipeline 批量操作优化性能
+    - 容错设计和连接池管理
+    
+    适用场景:
+    - 分布式爬虫系统
+    - 大规模数据处理
+    - 需要持久化去重的场景
+    """
+
+    def __init__(
+            self,
+            redis_key: str,
+            client: aioredis.Redis,
+            stats: dict,
+            debug: bool = False,
+            log_level: str = 'INFO',
+            cleanup_fp: bool = False,
+            ttl: Optional[int] = None
+    ):
+        """
+        初始化Redis过滤器
+        
+        :param redis_key: Redis中存储指纹的键名
+        :param client: Redis客户端实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试模式
+        :param log_level: 日志级别
+        :param cleanup_fp: 关闭时是否清理指纹
+        :param ttl: 指纹过期时间（秒）
+        """
+        self.logger = get_logger(self.__class__.__name__, log_level)
+        super().__init__(self.logger, stats, debug)
+
+        self.redis_key = redis_key
+        self.redis = client
+        self.cleanup_fp = cleanup_fp
+        self.ttl = ttl
+        
+        # 性能计数器
+        self._redis_operations = 0
+        self._pipeline_operations = 0
+
+    @classmethod
+    def create_instance(cls, crawler) -> 'BaseFilter':
+        """从爬虫配置创建过滤器实例"""
+        redis_url = crawler.settings.get('REDIS_URL', 'redis://localhost:6379')
+        decode_responses = crawler.settings.get_bool('DECODE_RESPONSES', False)
+        ttl_setting = crawler.settings.get_int('REDIS_TTL')
+
+        # 处理TTL设置
+        ttl = None
+        if ttl_setting is not None:
+            ttl = max(0, int(ttl_setting)) if ttl_setting > 0 else None
+
+        try:
+            redis_client = aioredis.from_url(
+                redis_url,
+                decode_responses=decode_responses,
+                max_connections=20,
+                encoding='utf-8'
+            )
+        except Exception as e:
+            raise RuntimeError(f"Redis连接失败: {redis_url} - {str(e)}")
+
+        return cls(
+            redis_key=f"{crawler.settings.get('PROJECT_NAME', 'default')}:{crawler.settings.get('REDIS_KEY', 'request_fingerprints')}",
+            client=redis_client,
+            stats=crawler.stats,
+            cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
+            ttl=ttl,
+            debug=crawler.settings.get_bool('FILTER_DEBUG', False),
+            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
+        )
+
+    async def requested(self, request) -> bool:
+        """
+        检查请求是否已存在（优化版本）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        try:
+            fp = str(request_fingerprint(request))
+            self._redis_operations += 1
+
+            # 使用 pipeline 优化性能
+            pipe = self.redis.pipeline()
+            pipe.sismember(self.redis_key, fp)
+            
+            results = await pipe.execute()
+            exists = results[0]
+            
+            self._pipeline_operations += 1
+
+            if exists:
+                if self.debug:
+                    self.logger.debug(f"发现重复请求: {fp[:20]}...")
+                return True
+
+            # 如果不存在，添加指纹并设置TTL
+            await self.add_fingerprint(fp)
+            return False
+
+        except Exception as e:
+            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')} - {e}")
+            # 在网络异常时返回False，避免丢失请求
+            return False
+
+    async def add_fingerprint(self, fp: str) -> bool:
+        """
+        添加新指纹到Redis集合（优化版本）
+        
+        :param fp: 请求指纹字符串
+        :return: 是否成功添加（True 表示新添加，False 表示已存在）
+        """
+        try:
+            fp = str(fp)
+            
+            # 使用 pipeline 优化性能
+            pipe = self.redis.pipeline()
+            pipe.sadd(self.redis_key, fp)
+            
+            if self.ttl and self.ttl > 0:
+                pipe.expire(self.redis_key, self.ttl)
+            
+            results = await pipe.execute()
+            added = results[0] == 1  # sadd 返回 1 表示新添加
+            
+            self._pipeline_operations += 1
+            
+            if self.debug and added:
+                self.logger.debug(f"添加新指纹: {fp[:20]}...")
+            
+            return added
+            
+        except Exception as e:
+            self.logger.error(f"添加指纹失败: {fp[:20]}... - {e}")
+            return False
+    
+    def __contains__(self, item: str) -> bool:
+        """
+        同步版本的包含检查（不推荐在异步环境中使用）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        # 这是一个同步方法，不能直接调用异步Redis操作
+        # 建议使用 requested() 方法替代
+        raise NotImplementedError("请使用 requested() 方法进行异步检查")
+
+    async def get_stats(self) -> dict:
+        """获取过滤器详细统计信息"""
+        try:
+            count = await self.redis.scard(self.redis_key)
+            
+            # 获取TTL信息
+            ttl_info = "TTL未设置"
+            if self.ttl:
+                remaining_ttl = await self.redis.ttl(self.redis_key)
+                if remaining_ttl > 0:
+                    ttl_info = f"剩余 {remaining_ttl} 秒"
+                else:
+                    ttl_info = f"配置 {self.ttl} 秒"
+            
+            stats = {
+                'filter_type': 'AioRedisFilter',
+                '指纹总数': count,
+                'Redis键名': self.redis_key,
+                'TTL配置': ttl_info,
+                'Redis操作数': self._redis_operations,
+                'Pipeline操作数': self._pipeline_operations,
+                '性能优化率': f"{self._pipeline_operations / max(1, self._redis_operations) * 100:.1f}%"
+            }
+            
+            # 合并基类统计
+            base_stats = super().get_stats()
+            stats.update(base_stats)
+            
+            return stats
+            
+        except Exception as e:
+            self.logger.error(f"获取统计信息失败: {e}")
+            return super().get_stats()
+
+    async def clear_all(self) -> int:
+        """清空所有指纹数据"""
+        try:
+            deleted = await self.redis.delete(self.redis_key)
+            self.logger.info(f"已清除指纹数: {deleted}")
+            return deleted
+        except Exception as e:
+            self.logger.error("清空指纹失败")
+            raise
+
+    async def closed(self, reason: Optional[str] = None) -> None:
+        """爬虫关闭时的清理操作"""
+        try:
+            if self.cleanup_fp:
+                deleted = await self.redis.delete(self.redis_key)
+                self.logger.info(f"爬虫关闭清理: 已删除{deleted}个指纹")
+            else:
+                count = await self.redis.scard(self.redis_key)
+                ttl_info = f"{self.ttl}秒" if self.ttl else "持久化"
+                self.logger.info(f"保留指纹数: {count} (TTL: {ttl_info})")
+        finally:
+            await self._close_redis()
+
+    async def _close_redis(self) -> None:
+        """安全关闭Redis连接"""
+        try:
+            if hasattr(self.redis, 'close'):
+                await self.redis.close()
+                self.logger.debug("Redis连接已关闭")
+        except Exception as e:
+            self.logger.warning(f"Redis关闭时出错：{e}")