crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/filters/aioredis_filter.py

@@ -1,27 +1,60 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-import aioredis
+"""
+Redis 过滤器实现
+=================
+提供基于 Redis 的分布式请求去重功能。
+
+特点:
+- 分布式支持: 多节点共享去重数据
+- TTL 支持: 自动过期清理
+- 高性能: 使用 Redis pipeline 优化
+- 容错设计: 网络异常自动重试
+"""
 from typing import Optional
-from crawlo import Request
+from redis import asyncio as aioredis
 from crawlo.filters import BaseFilter
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import request_fingerprint
 
 
 class AioRedisFilter(BaseFilter):
-    """基于Redis集合实现的异步请求去重过滤器（支持分布式爬虫），提供TTL和清理控制"""
+    """
+    基于Redis集合实现的异步请求去重过滤器
+    
+    支持特性:
+    - 分布式爬虫多节点共享去重数据
+    - TTL 自动过期清理机制
+    - Pipeline 批量操作优化性能
+    - 容错设计和连接池管理
+    
+    适用场景:
+    - 分布式爬虫系统
+    - 大规模数据处理
+    - 需要持久化去重的场景
+    """
 
     def __init__(
             self,
             redis_key: str,
             client: aioredis.Redis,
             stats: dict,
-            debug: bool,
-            log_level: str,
+            debug: bool = False,
+            log_level: str = 'INFO',
             cleanup_fp: bool = False,
             ttl: Optional[int] = None
     ):
-        """初始化过滤器"""
+        """
+        初始化Redis过滤器
+        
+        :param redis_key: Redis中存储指纹的键名
+        :param client: Redis客户端实例
+        :param stats: 统计信息存储
+        :param debug: 是否启用调试模式
+        :param log_level: 日志级别
+        :param cleanup_fp: 关闭时是否清理指纹
+        :param ttl: 指纹过期时间（秒）
+        """
         self.logger = get_logger(self.__class__.__name__, log_level)
         super().__init__(self.logger, stats, debug)
 
@@ -29,6 +62,10 @@ class AioRedisFilter(BaseFilter):
         self.redis = client
         self.cleanup_fp = cleanup_fp
         self.ttl = ttl
+        
+        # 性能计数器
+        self._redis_operations = 0
+        self._pipeline_operations = 0
 
     @classmethod
     def create_instance(cls, crawler) -> 'BaseFilter':
@@ -62,60 +99,115 @@ class AioRedisFilter(BaseFilter):
             log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
         )
 
-    async def requested(self, request: Request) -> bool:
-        """检查请求是否已存在"""
+    async def requested(self, request) -> bool:
+        """
+        检查请求是否已存在（优化版本）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
         try:
             fp = str(request_fingerprint(request))
+            self._redis_operations += 1
 
-            # 1. 检查指纹是否存在
+            # 使用 pipeline 优化性能
             pipe = self.redis.pipeline()
-            pipe.sismember(self.redis_key, fp)  # 不单独 await
-            exists = (await pipe.execute())[0]  # 执行并获取结果
-
-            if exists:  # 如果已存在，返回 True
+            pipe.sismember(self.redis_key, fp)
+            
+            results = await pipe.execute()
+            exists = results[0]
+            
+            self._pipeline_operations += 1
+
+            if exists:
+                if self.debug:
+                    self.logger.debug(f"发现重复请求: {fp[:20]}...")
                 return True
 
-            # 2. 如果不存在，添加指纹并设置 TTL
-            pipe = self.redis.pipeline()
-            pipe.sadd(self.redis_key, fp)  # 不单独 await
-            if self.ttl and self.ttl > 0:
-                pipe.expire(self.redis_key, self.ttl)  # 不单独 await
-            await pipe.execute()  # 一次性执行所有命令
-
-            return False  # 表示是新请求
+            # 如果不存在，添加指纹并设置TTL
+            await self.add_fingerprint(fp)
+            return False
 
         except Exception as e:
-            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')}")
-            raise
+            self.logger.error(f"请求检查失败: {getattr(request, 'url', '未知URL')} - {e}")
+            # 在网络异常时返回False，避免丢失请求
+            return False
 
     async def add_fingerprint(self, fp: str) -> bool:
-        """添加新指纹到Redis集合"""
+        """
+        添加新指纹到Redis集合（优化版本）
+        
+        :param fp: 请求指纹字符串
+        :return: 是否成功添加（True 表示新添加，False 表示已存在）
+        """
         try:
             fp = str(fp)
-            added = await self.redis.sadd(self.redis_key, fp)
-
+            
+            # 使用 pipeline 优化性能
+            pipe = self.redis.pipeline()
+            pipe.sadd(self.redis_key, fp)
+            
             if self.ttl and self.ttl > 0:
-                await self.redis.expire(self.redis_key, self.ttl)
-
-            return added == 1
+                pipe.expire(self.redis_key, self.ttl)
+            
+            results = await pipe.execute()
+            added = results[0] == 1  # sadd 返回 1 表示新添加
+            
+            self._pipeline_operations += 1
+            
+            if self.debug and added:
+                self.logger.debug(f"添加新指纹: {fp[:20]}...")
+            
+            return added
+            
         except Exception as e:
-            self.logger.error("添加指纹失败")
-            raise
+            self.logger.error(f"添加指纹失败: {fp[:20]}... - {e}")
+            return False
+    
+    def __contains__(self, item: str) -> bool:
+        """
+        同步版本的包含检查（不推荐在异步环境中使用）
+        
+        :param item: 要检查的指纹
+        :return: 是否已存在
+        """
+        # 这是一个同步方法，不能直接调用异步Redis操作
+        # 建议使用 requested() 方法替代
+        raise NotImplementedError("请使用 requested() 方法进行异步检查")
 
     async def get_stats(self) -> dict:
-        """获取过滤器统计信息"""
+        """获取过滤器详细统计信息"""
         try:
             count = await self.redis.scard(self.redis_key)
+            
+            # 获取TTL信息
+            ttl_info = "TTL未设置"
+            if self.ttl:
+                remaining_ttl = await self.redis.ttl(self.redis_key)
+                if remaining_ttl > 0:
+                    ttl_info = f"剩余 {remaining_ttl} 秒"
+                else:
+                    ttl_info = f"配置 {self.ttl} 秒"
+            
             stats = {
+                'filter_type': 'AioRedisFilter',
                 '指纹总数': count,
                 'Redis键名': self.redis_key,
-                'TTL配置': f"{self.ttl}秒" if self.ttl else "持久化"
+                'TTL配置': ttl_info,
+                'Redis操作数': self._redis_operations,
+                'Pipeline操作数': self._pipeline_operations,
+                '性能优化率': f"{self._pipeline_operations / max(1, self._redis_operations) * 100:.1f}%"
             }
-            stats.update(self.stats)
+            
+            # 合并基类统计
+            base_stats = super().get_stats()
+            stats.update(base_stats)
+            
             return stats
+            
         except Exception as e:
-            self.logger.error("获取统计信息失败")
-            return self.stats
+            self.logger.error(f"获取统计信息失败: {e}")
+            return super().get_stats()
 
     async def clear_all(self) -> int:
         """清空所有指纹数据"""

crawlo/filters/memory_filter.py

@@ -1,18 +1,39 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+"""
+内存过滤器实现
+================
+提供基于内存的高效请求去重功能。
+
+支持的过滤器:
+- MemoryFilter: 纯内存去重，性能最佳
+- MemoryFileFilter: 内存+文件持久化，支持重启恢复
+"""
 import os
 import threading
 from weakref import WeakSet
 from typing import Set, TextIO, Optional
 
-from crawlo import Request
 from crawlo.filters import BaseFilter
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import request_fingerprint
 
 
 class MemoryFilter(BaseFilter):
-    """基于内存的高效请求去重过滤器，适用于单机爬虫"""
+    """
+    基于内存的高效请求去重过滤器
+    
+    特点:
+    - 高性能: 基于 Python set() 的 O(1) 查找效率
+    - 内存优化: 支持弱引用临时存储
+    - 统计信息: 提供详细的性能统计
+    - 线程安全: 支持多线程并发访问
+    
+    适用场景:
+    - 单机爬虫
+    - 中小规模数据集
+    - 对性能要求较高的场景
+    """
 
     def __init__(self, crawler):
         """
@@ -21,11 +42,13 @@ class MemoryFilter(BaseFilter):
         :param crawler: 爬虫实例，用于获取配置
         """
         self.fingerprints: Set[str] = set()  # 主指纹存储
-        self._temp_weak_refs = WeakSet()  # 弱引用临时存储（可选）
+        self._temp_weak_refs = WeakSet()     # 弱引用临时存储
+        self._lock = threading.RLock()       # 线程安全锁
 
+        # 初始化日志和统计
         debug = crawler.settings.get_bool('FILTER_DEBUG', False)
         logger = get_logger(
-            self.__class__.__name__,  # 使用类名替代字符串
+            self.__class__.__name__,
             crawler.settings.get('LOG_LEVEL', 'INFO')
         )
         super().__init__(logger, crawler.stats, debug)
@@ -33,10 +56,14 @@ class MemoryFilter(BaseFilter):
         # 性能计数器
         self._dupe_count = 0
         self._unique_count = 0
+        
+        # 内存优化配置
+        self._max_capacity = crawler.settings.get_int('MEMORY_FILTER_MAX_CAPACITY', 1000000)
+        self._cleanup_threshold = crawler.settings.get_float('MEMORY_FILTER_CLEANUP_THRESHOLD', 0.8)
 
     def add_fingerprint(self, fp: str) -> None:
         """
-        添加请求指纹
+        线程安全地添加请求指纹
 
         :param fp: 请求指纹字符串
         :raises TypeError: 如果指纹不是字符串类型
@@ -44,56 +71,96 @@ class MemoryFilter(BaseFilter):
         if not isinstance(fp, str):
             raise TypeError(f"指纹必须是字符串类型，得到 {type(fp)}")
 
-        self.fingerprints.add(fp)
-        self._unique_count += 1
-        # self.logger.debug(f"添加指纹: {fp[:10]}...")  # 日志截断防止过长
-
-    def requested(self, request: Request) -> bool:
+        with self._lock:
+            if fp not in self.fingerprints:
+                # 检查容量限制
+                if len(self.fingerprints) >= self._max_capacity:
+                    self._cleanup_old_fingerprints()
+                
+                self.fingerprints.add(fp)
+                self._unique_count += 1
+                
+                if self.debug:
+                    self.logger.debug(f"添加指纹: {fp[:20]}...")
+    
+    def _cleanup_old_fingerprints(self) -> None:
+        """清理老旧指纹释放内存空间"""
+        cleanup_count = int(len(self.fingerprints) * (1 - self._cleanup_threshold))
+        if cleanup_count > 0:
+            # 随机清理一部分指纹（简单策略）
+            fingerprints_list = list(self.fingerprints)
+            import random
+            to_remove = random.sample(fingerprints_list, cleanup_count)
+            self.fingerprints.difference_update(to_remove)
+            self.logger.info(f"清理了 {cleanup_count} 个老旧指纹")
+
+    def requested(self, request) -> bool:
         """
-        检查请求是否重复（主要接口）
+        线程安全地检查请求是否重复（主要接口）
 
         :param request: 请求对象
         :return: 是否重复
         """
-        fp = request_fingerprint(request)
-        if fp in self:
-            self._dupe_count += 1
-            # self.logger.debug(f"发现重复请求: {fp[:10]}...")
-            return True
+        with self._lock:
+            fp = request_fingerprint(request)
+            if fp in self.fingerprints:
+                self._dupe_count += 1
+                if self.debug:
+                    self.logger.debug(f"发现重复请求: {fp[:20]}...")
+                return True
 
-        self.add_fingerprint(fp)
-        return False
+            self.add_fingerprint(fp)
+            return False
 
     def __contains__(self, item: str) -> bool:
         """
-        支持 in 操作符检查
+        线程安全地支持 in 操作符检查
 
         :param item: 要检查的指纹
         :return: 是否已存在
         """
-        return item in self.fingerprints
+        with self._lock:
+            return item in self.fingerprints
 
     @property
     def stats_summary(self) -> dict:
         """获取过滤器统计信息"""
-        return {
-            'capacity': len(self.fingerprints),
-            'duplicates': self._dupe_count,
-            'uniques': self._unique_count,
-            'memory_usage': self._estimate_memory()
-        }
+        with self._lock:
+            return {
+                'filter_type': 'MemoryFilter',
+                'capacity': len(self.fingerprints),
+                'max_capacity': self._max_capacity,
+                'duplicates': self._dupe_count,
+                'uniques': self._unique_count,
+                'total_processed': self._dupe_count + self._unique_count,
+                'duplicate_rate': f"{self._dupe_count / max(1, self._dupe_count + self._unique_count) * 100:.2f}%",
+                'memory_usage': self._estimate_memory(),
+                'capacity_usage': f"{len(self.fingerprints) / self._max_capacity * 100:.2f}%"
+            }
 
     def _estimate_memory(self) -> str:
         """估算内存使用量（近似值）"""
-        avg_item_size = sum(len(x) for x in self.fingerprints) / max(1, len(self.fingerprints))
+        if not self.fingerprints:
+            return "0 MB"
+        
+        avg_item_size = sum(len(x) for x in self.fingerprints) / len(self.fingerprints)
         total = len(self.fingerprints) * (avg_item_size + 50)  # 50字节额外开销
-        return f"{total / (1024 * 1024):.2f} MB"
+        
+        if total < 1024:
+            return f"{total:.1f} B"
+        elif total < 1024 * 1024:
+            return f"{total / 1024:.1f} KB" 
+        else:
+            return f"{total / (1024 * 1024):.2f} MB"
 
     def clear(self) -> None:
-        """清空所有指纹数据"""
-        self.fingerprints.clear()
-        self._dupe_count = 0
-        self._unique_count = 0
+        """线程安全地清空所有指纹数据"""
+        with self._lock:
+            self.fingerprints.clear()
+            self._dupe_count = 0
+            self._unique_count = 0
+            if self.debug:
+                self.logger.debug("已清空所有指纹")
 
     def close(self) -> None:
         """关闭过滤器（清理资源）"""

crawlo/middleware/proxy.py

@@ -2,16 +2,18 @@
 # -*- coding: UTF-8 -*-
 import asyncio
 import socket
-from typing import Optional, Dict, Any, Callable, Union
+from typing import Optional, Dict, Any, Callable, Union, TYPE_CHECKING
 from urllib.parse import urlparse
 
 from crawlo import Request, Response
 from crawlo.exceptions import NotConfiguredError
 from crawlo.utils.log import get_logger
 
+if TYPE_CHECKING:
+    import aiohttp
+
 try:
     import httpx
-
     HTTPX_EXCEPTIONS = (httpx.NetworkError, httpx.TimeoutException, httpx.ReadError, httpx.ConnectError)
 except ImportError:
     HTTPX_EXCEPTIONS = ()
@@ -19,17 +21,15 @@ except ImportError:
 
 try:
     import aiohttp
-
     AIOHTTP_EXCEPTIONS = (
-    aiohttp.ClientError, aiohttp.ClientConnectorError, aiohttp.ClientResponseError, aiohttp.ServerTimeoutError,
-    aiohttp.ServerDisconnectedError)
+        aiohttp.ClientError, aiohttp.ClientConnectorError, aiohttp.ClientResponseError, aiohttp.ServerTimeoutError,
+        aiohttp.ServerDisconnectedError)
 except ImportError:
     AIOHTTP_EXCEPTIONS = ()
     aiohttp = None
 
 try:
     from curl_cffi import requests as cffi_requests
-
     CURL_CFFI_EXCEPTIONS = (cffi_requests.RequestsError,)
 except (ImportError, AttributeError):
     CURL_CFFI_EXCEPTIONS = ()
@@ -49,7 +49,7 @@ class ProxyMiddleware:
     def __init__(self, settings, log_level):
         self.logger = get_logger(self.__class__.__name__, log_level)
 
-        self._session: Optional[aiohttp.ClientSession] = None
+        self._session: Optional[Any] = None  # aiohttp.ClientSession when aiohttp is available
         self._current_proxy: Optional[Union[str, Dict[str, str]]] = None
         self._last_fetch_time: float = 0
 
@@ -104,7 +104,10 @@ class ProxyMiddleware:
             finally:
                 self._session = None
 
-    async def _get_session(self) -> aiohttp.ClientSession:
+    async def _get_session(self) -> Any:  # returns aiohttp.ClientSession when aiohttp is available
+        if aiohttp is None:
+            raise RuntimeError("aiohttp 未安装，无法使用 ProxyMiddleware")
+            
         if self._session is None or self._session.closed:
             if self._session and self._session.closed:
                 self.logger.debug("现有 session 已关闭，正在创建新 session...")

crawlo/middleware/retry.py

@@ -1,12 +1,47 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 from typing import List
-from anyio import EndOfStream
-from httpcore import ReadError
 from asyncio.exceptions import TimeoutError
-from httpx import RemoteProtocolError, ConnectError, ReadTimeout
-from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
-from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
+
+try:
+    from anyio import EndOfStream
+except ImportError:
+    # 如果 anyio 不可用或者 EndOfStream 不存在，创建一个占位符
+    class EndOfStream(Exception):
+        pass
+
+try:
+    from httpcore import ReadError
+except ImportError:
+    class ReadError(Exception):
+        pass
+
+try:
+    from httpx import RemoteProtocolError, ConnectError, ReadTimeout
+except ImportError:
+    class RemoteProtocolError(Exception):
+        pass
+    class ConnectError(Exception):
+        pass
+    class ReadTimeout(Exception):
+        pass
+
+try:
+    from aiohttp.client_exceptions import ClientConnectionError, ClientPayloadError
+    from aiohttp import ClientConnectorError, ClientTimeout, ClientConnectorSSLError, ClientResponseError
+except ImportError:
+    class ClientConnectionError(Exception):
+        pass
+    class ClientPayloadError(Exception):
+        pass
+    class ClientConnectorError(Exception):
+        pass
+    class ClientTimeout(Exception):
+        pass
+    class ClientConnectorSSLError(Exception):
+        pass
+    class ClientResponseError(Exception):
+        pass
 
 from crawlo.utils.log import get_logger
 from crawlo.stats_collector import StatsCollector