crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/filters/aioredis_filter.py

@@ -1,10 +1,19 @@
-from typing import Optional
+from typing import Optional, Dict, Any, Union, Awaitable, Literal
 import redis.asyncio as aioredis
+import asyncio
+from inspect import iscoroutinefunction
+
+# 尝试导入Redis集群支持
+try:
+    from redis.asyncio.cluster import RedisCluster
+    REDIS_CLUSTER_AVAILABLE = True
+except ImportError:
+    RedisCluster = None
+    REDIS_CLUSTER_AVAILABLE = False
 
 from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
-from crawlo.utils.redis_connection_pool import get_redis_pool
+from crawlo.logging import get_logger
+from crawlo.utils.redis_connection_pool import get_redis_pool, RedisConnectionPool
 
 
 class AioRedisFilter(BaseFilter):
@@ -16,20 +25,16 @@ class AioRedisFilter(BaseFilter):
     - TTL 自动过期清理机制
     - Pipeline 批量操作优化性能
     - 容错设计和连接池管理
-    
-    适用场景:
-    - 分布式爬虫系统
-    - 大规模数据处理
-    - 需要持久化去重的场景
+    - Redis集群支持
     """
 
     def __init__(
             self,
             redis_key: str,
-            client: aioredis.Redis,
-            stats: dict,
+            client: Optional[aioredis.Redis] = None,
+            stats: Optional[Dict[str, Any]] = None,
             debug: bool = False,
-            log_level: str = 'INFO',
+            log_level: int = 20,  # logging.INFO
             cleanup_fp: bool = False,
             ttl: Optional[int] = None
     ):
@@ -44,7 +49,7 @@ class AioRedisFilter(BaseFilter):
         :param cleanup_fp: 关闭时是否清理指纹
         :param ttl: 指纹过期时间（秒）
         """
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         super().__init__(self.logger, stats, debug)
 
         self.redis_key = redis_key
@@ -53,7 +58,7 @@ class AioRedisFilter(BaseFilter):
         self.ttl = ttl
         
         # 保存连接池引用（用于延迟初始化）
-        self._redis_pool = None
+        self._redis_pool: Optional[RedisConnectionPool] = None
         
         # 性能计数器
         self._redis_operations = 0
@@ -105,7 +110,7 @@ class AioRedisFilter(BaseFilter):
             cleanup_fp=crawler.settings.get_bool('CLEANUP_FP', False),
             ttl=ttl,
             debug=crawler.settings.get_bool('FILTER_DEBUG', False),
-            log_level=crawler.settings.get('LOG_LEVEL', 'INFO')
+            log_level=getattr(crawler.settings, 'LOG_LEVEL_NUM', 20)  # 默认INFO级别
         )
         
         # 保存连接池引用，以便在需要时获取连接
@@ -120,16 +125,41 @@ class AioRedisFilter(BaseFilter):
             
         if self.redis is None and self._redis_pool is not None:
             try:
-                self.redis = await self._redis_pool.get_connection()
+                connection = await self._redis_pool.get_connection()
+                # 确保返回的是Redis客户端而不是连接池本身
+                if hasattr(connection, 'ping'):
+                    self.redis = connection
+                else:
+                    self.redis = connection
             except Exception as e:
                 self._connection_failed = True
                 self.logger.error(f"Redis连接失败，将使用本地去重: {e}")
                 return None
         return self.redis
 
-    async def requested(self, request) -> bool:
+    def _is_cluster_mode(self) -> bool:
+        """检查是否为集群模式"""
+        if REDIS_CLUSTER_AVAILABLE and RedisCluster is not None:
+            # 检查 redis 是否为 RedisCluster 实例
+            if self.redis is not None and isinstance(self.redis, RedisCluster):
+                return True
+        return False
+
+    def requested(self, request) -> bool:
+        """
+        检查请求是否已存在（同步方法）
+        
+        :param request: 请求对象
+        :return: True 表示重复，False 表示新请求
+        """
+        # 这个方法需要同步实现，但Redis操作是异步的
+        # 在实际使用中，应该通过异步方式调用 _requested_async
+        # 由于BaseFilter要求同步方法，我们在这里返回False表示不重复
+        return False
+
+    async def requested_async(self, request) -> bool:
         """
-        检查请求是否已存在（优化版本）
+        异步检查请求是否已存在
         
         :param request: 请求对象
         :return: True 表示重复，False 表示新请求
@@ -142,32 +172,38 @@ class AioRedisFilter(BaseFilter):
             if redis_client is None:
                 return False
             
-            # 使用统一的指纹生成器
-            from crawlo.utils.fingerprint import FingerprintGenerator
-            fp = str(FingerprintGenerator.request_fingerprint(
-                request.method, 
-                request.url, 
-                request.body or b'', 
-                dict(request.headers) if hasattr(request, 'headers') else None
-            ))
+            # 使用基类的指纹生成方法
+            fp = str(self._get_fingerprint(request))
             self._redis_operations += 1
 
-            # 使用 pipeline 优化性能
-            pipe = redis_client.pipeline()
-            pipe.sismember(self.redis_key, fp)
-            
-            results = await pipe.execute()
-            exists = results[0]
+            # 检查指纹是否存在
+            if self._is_cluster_mode():
+                # 集群模式下使用哈希标签确保键在同一个slot
+                hash_tag = "{filter}"
+                redis_key_with_tag = f"{self.redis_key}{hash_tag}"
+                # 直接调用异步方法
+                result = redis_client.sismember(redis_key_with_tag, fp)
+                if asyncio.iscoroutine(result):
+                    exists = await result
+                else:
+                    exists = result
+            else:
+                # 直接调用异步方法
+                result = redis_client.sismember(self.redis_key, fp)
+                if asyncio.iscoroutine(result):
+                    exists = await result
+                else:
+                    exists = result
             
             self._pipeline_operations += 1
 
             if exists:
                 if self.debug:
                     self.logger.debug(f"发现重复请求: {fp[:20]}...")
-                return True
+                return bool(exists)
 
             # 如果不存在，添加指纹并设置TTL
-            await self.add_fingerprint(fp)
+            await self._add_fingerprint_async(fp)
             return False
 
         except Exception as e:
@@ -175,9 +211,19 @@ class AioRedisFilter(BaseFilter):
             # 在网络异常时返回False，避免丢失请求
             return False
 
-    async def add_fingerprint(self, fp: str) -> bool:
+    def add_fingerprint(self, fp: str) -> None:
         """
-        添加新指纹到Redis集合（优化版本）
+        添加新指纹到Redis集合（同步方法）
+        
+        :param fp: 请求指纹字符串
+        """
+        # 这个方法需要同步实现，但Redis操作是异步的
+        # 在实际使用中，应该通过异步方式调用 _add_fingerprint_async
+        pass
+
+    async def _add_fingerprint_async(self, fp: str) -> bool:
+        """
+        异步添加新指纹到Redis集合
         
         :param fp: 请求指纹字符串
         :return: 是否成功添加（True 表示新添加，False 表示已存在）
@@ -192,22 +238,44 @@ class AioRedisFilter(BaseFilter):
             
             fp = str(fp)
             
-            # 使用 pipeline 优化性能
-            pipe = redis_client.pipeline()
-            pipe.sadd(self.redis_key, fp)
-            
-            if self.ttl and self.ttl > 0:
-                pipe.expire(self.redis_key, self.ttl)
-            
-            results = await pipe.execute()
-            added = results[0] == 1  # sadd 返回 1 表示新添加
+            # 添加指纹
+            if self._is_cluster_mode():
+                # 集群模式下使用哈希标签确保键在同一个slot
+                hash_tag = "{filter}"
+                redis_key_with_tag = f"{self.redis_key}{hash_tag}"
+                # 直接调用异步方法
+                result = redis_client.sadd(redis_key_with_tag, fp)
+                if asyncio.iscoroutine(result):
+                    added = await result
+                else:
+                    added = result
+                if self.ttl and self.ttl > 0:
+                    expire_result = redis_client.expire(redis_key_with_tag, self.ttl)
+                    if asyncio.iscoroutine(expire_result):
+                        await expire_result
+                    else:
+                        expire_result  # 不需要等待同步结果
+                added = added == 1  # sadd 返回 1 表示新添加
+            else:
+                # 直接调用异步方法
+                result = redis_client.sadd(self.redis_key, fp)
+                if asyncio.iscoroutine(result):
+                    added = await result
+                else:
+                    added = result
+                if self.ttl and self.ttl > 0:
+                    expire_result = redis_client.expire(self.redis_key, self.ttl)
+                    if asyncio.iscoroutine(expire_result):
+                        await expire_result
+                    else:
+                        expire_result  # 不需要等待同步结果
             
             self._pipeline_operations += 1
             
             if self.debug and added:
                 self.logger.debug(f"添加新指纹: {fp[:20]}...")
             
-            return added
+            return bool(added)
             
         except Exception as e:
             self.logger.error(f"添加指纹失败: {fp[:20]}... - {e}")
@@ -252,8 +320,24 @@ class AioRedisFilter(BaseFilter):
                 return False
             
             # 检查指纹是否存在
-            exists = await redis_client.sismember(self.redis_key, str(fp))
-            return exists
+            if self._is_cluster_mode():
+                # 集群模式下使用哈希标签确保键在同一个slot
+                hash_tag = "{filter}"
+                redis_key_with_tag = f"{self.redis_key}{hash_tag}"
+                # 直接调用异步方法
+                result = redis_client.sismember(redis_key_with_tag, str(fp))
+                if asyncio.iscoroutine(result):
+                    exists = await result
+                else:
+                    exists = result
+            else:
+                # 直接调用异步方法
+                result = redis_client.sismember(self.redis_key, str(fp))
+                if asyncio.iscoroutine(result):
+                    exists = await result
+                else:
+                    exists = result
+            return bool(exists)
         except Exception as e:
             self.logger.error(f"检查指纹存在性失败: {fp[:20]}... - {e}")
             # 在网络异常时返回False，避免丢失请求
@@ -261,4 +345,4 @@ class AioRedisFilter(BaseFilter):
 
 
 # 为了兼容性，确保导出类
-__all__ = ['AioRedisFilter']
+__all__ = ['AioRedisFilter']

crawlo/filters/memory_filter.py

@@ -15,8 +15,7 @@ from weakref import WeakSet
 from typing import Set, TextIO, Optional
 
 from crawlo.filters import BaseFilter
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import request_fingerprint
+from crawlo.logging import get_logger
 
 
 class MemoryFilter(BaseFilter):
@@ -47,10 +46,7 @@ class MemoryFilter(BaseFilter):
 
         # 初始化日志和统计
         debug = crawler.settings.get_bool('FILTER_DEBUG', False)
-        logger = get_logger(
-            self.__class__.__name__,
-            crawler.settings.get('LOG_LEVEL', 'INFO')
-        )
+        logger = get_logger(self.__class__.__name__)
         super().__init__(logger, crawler.stats, debug)
 
         # 性能计数器
@@ -102,18 +98,10 @@ class MemoryFilter(BaseFilter):
         :return: 是否重复
         """
         with self._lock:
-            # 使用统一的指纹生成器
-            from crawlo.utils.fingerprint import FingerprintGenerator
-            fp = FingerprintGenerator.request_fingerprint(
-                request.method, 
-                request.url, 
-                request.body or b'', 
-                dict(request.headers) if hasattr(request, 'headers') else None
-            )
+            # 使用基类的指纹生成方法
+            fp = self._get_fingerprint(request)
             if fp in self.fingerprints:
                 self._dupe_count += 1
-                # if self.debug:
-                #     self.logger.debug(f"发现重复请求: {fp[:20]}...")  # 注释掉重复的日志
                 return True
 
             self.add_fingerprint(fp)
@@ -185,17 +173,14 @@ class MemoryFileFilter(BaseFilter):
     def __init__(self, crawler):
         """
         初始化过滤器
-        :param crawler: Scrapy Crawler对象，用于获取配置
+        :param crawler: 爬虫框架Crawler对象，用于获取配置
         """
         self.fingerprints: Set[str] = set()  # 主存储集合
         self._lock = threading.RLock()  # 线程安全锁
         self._file: Optional[TextIO] = None  # 文件句柄
 
         debug = crawler.settings.get_bool("FILTER_DEBUG", False)
-        logger = get_logger(
-            self.__class__.__name__,  # 使用类名作为日志标识
-            crawler.settings.get("LOG_LEVEL", "INFO")
-        )
+        logger = get_logger(self.__class__.__name__)
         super().__init__(logger, crawler.stats, debug)
 
         # 初始化文件存储

crawlo/framework.py

@@ -11,10 +11,10 @@ import os
 import sys
 from typing import Type, Optional, List, Union
 
-from .crawler import ModernCrawler, CrawlerProcess
+from .crawler import Crawler, CrawlerProcess
 from .initialization import initialize_framework
 from .logging import get_logger
-from .utils.env_config import get_version
+from .utils.config_manager import EnvConfigManager
 
 
 class CrawloFramework:
@@ -50,7 +50,7 @@ class CrawloFramework:
         self._logger = get_logger('crawlo.framework')
 
         # 获取版本号
-        version = get_version()
+        version = EnvConfigManager.get_version()
 
         # 创建进程管理器
         self._process = CrawlerProcess(self._settings)
@@ -195,9 +195,13 @@ class CrawloFramework:
         
         self._logger.info(f"Starting spiders: {', '.join(spider_names)}")
         
-        return await self._process.crawl_multiple(spider_classes_or_names, settings)
+        try:
+            return await self._process.crawl_multiple(spider_classes_or_names, settings)
+        finally:
+            # 清理全局Redis连接池
+            await self._cleanup_global_resources()
 
-    def create_crawler(self, spider_cls: Type, settings=None) -> ModernCrawler:
+    def create_crawler(self, spider_cls: Type, settings=None) -> Crawler:
         """
         创建Crawler实例
         
@@ -206,10 +210,10 @@ class CrawloFramework:
             settings: 额外配置
             
         Returns:
-            ModernCrawler实例
+            Crawler实例
         """
         merged_settings = self._merge_settings(settings)
-        return ModernCrawler(spider_cls, merged_settings)
+        return Crawler(spider_cls, merged_settings)
 
     def _merge_settings(self, additional_settings):
         """合并配置"""
@@ -234,6 +238,16 @@ class CrawloFramework:
     def get_metrics(self) -> dict:
         """获取框架指标"""
         return self._process.get_metrics()
+    
+    async def _cleanup_global_resources(self):
+        """清理全局资源（Redis连接池等）"""
+        try:
+            # 清理全局Redis连接池
+            from crawlo.utils.redis_connection_pool import close_all_pools
+            await close_all_pools()
+            self._logger.debug("Global resources cleaned up")
+        except Exception as e:
+            self._logger.warning(f"Failed to cleanup global resources: {e}")
 
 
 # 全局框架实例
@@ -279,7 +293,7 @@ async def run_spiders(spider_classes_or_names: List[Union[Type, str]],
     return await framework.run_multiple(spider_classes_or_names)
 
 
-def create_crawler(spider_cls: Type, settings=None, **kwargs) -> ModernCrawler:
+def create_crawler(spider_cls: Type, settings=None, **kwargs) -> Crawler:
     """创建Crawler的便捷函数"""
     framework = get_framework(settings, **kwargs)
     return framework.create_crawler(spider_cls)

crawlo/initialization/built_in.py

@@ -5,10 +5,15 @@
 """
 
 import time
+from typing import TYPE_CHECKING
+
 from .registry import BaseInitializer, register_initializer
 from .phases import InitializationPhase, PhaseResult
 from .context import InitializationContext
 
+if TYPE_CHECKING:
+    from crawlo.logging import LogConfig
+
 
 class LoggingInitializer(BaseInitializer):
     """日志系统初始化器"""
@@ -28,7 +33,7 @@ class LoggingInitializer(BaseInitializer):
             log_config = self._get_log_config(context)
             
             # 确保日志目录存在
-            if log_config.file_path and log_config.file_enabled:
+            if log_config and log_config.file_path and log_config.file_enabled:
                 import os
                 log_dir = os.path.dirname(log_config.file_path)
                 if log_dir and not os.path.exists(log_dir):
@@ -58,7 +63,7 @@ class LoggingInitializer(BaseInitializer):
                 error=e
             )
     
-    def _get_log_config(self, context: InitializationContext) -> 'LogConfig':
+    def _get_log_config(self, context: InitializationContext) -> 'LogConfig | None':
         """
         获取日志配置
         
@@ -70,6 +75,7 @@ class LoggingInitializer(BaseInitializer):
         """
         # 导入日志配置类
         from crawlo.logging import LogConfig
+        from crawlo.utils.config_manager import ConfigUtils
         
         # 按优先级获取配置：自定义配置 > 上下文配置 > 项目配置 > 默认配置
         config_sources = [
@@ -80,7 +86,7 @@ class LoggingInitializer(BaseInitializer):
         
         # 遍历配置源
         for config_source in config_sources:
-            if config_source:
+            if config_source and ConfigUtils.has_config_prefix(config_source, 'LOG_'):
                 log_config = self._create_log_config_from_source(config_source)
                 if log_config:
                     return log_config
@@ -88,7 +94,7 @@ class LoggingInitializer(BaseInitializer):
         # 使用默认配置
         return LogConfig()
     
-    def _create_log_config_from_source(self, config_source) -> 'LogConfig':
+    def _create_log_config_from_source(self, config_source) -> 'LogConfig | None':
         """
         从配置源创建日志配置
         
@@ -100,30 +106,25 @@ class LoggingInitializer(BaseInitializer):
         """
         # 导入日志配置类
         from crawlo.logging import LogConfig
+        from crawlo.utils.config_manager import ConfigUtils
         
         # 检查配置源是否有效
         if not config_source:
             return None
             
         # 检查是否有日志相关配置
-        has_keys_method = hasattr(config_source, 'keys')
-        if has_keys_method:
-            has_log_config = any(key.startswith('LOG_') for key in config_source.keys())
-        else:
-            has_log_config = any(key.startswith('LOG_') for key in dir(config_source))
-            
-        if not has_log_config:
+        if not ConfigUtils.has_config_prefix(config_source, 'LOG_'):
             return None
             
         # 从配置源获取日志配置
-        log_level = config_source.get('LOG_LEVEL', 'INFO')
-        log_file = config_source.get('LOG_FILE')
-        log_format = config_source.get('LOG_FORMAT', '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s')
-        log_encoding = config_source.get('LOG_ENCODING', 'utf-8')
-        log_max_bytes = config_source.get('LOG_MAX_BYTES', 10 * 1024 * 1024)
-        log_backup_count = config_source.get('LOG_BACKUP_COUNT', 5)
-        log_console_enabled = config_source.get('LOG_CONSOLE_ENABLED', True)
-        log_file_enabled = config_source.get('LOG_FILE_ENABLED', True)
+        log_level = ConfigUtils.get_config_value([config_source], 'LOG_LEVEL', 'INFO')
+        log_file = ConfigUtils.get_config_value([config_source], 'LOG_FILE')
+        log_format = ConfigUtils.get_config_value([config_source], 'LOG_FORMAT', '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s')
+        log_encoding = ConfigUtils.get_config_value([config_source], 'LOG_ENCODING', 'utf-8')
+        log_max_bytes = ConfigUtils.get_config_value([config_source], 'LOG_MAX_BYTES', 10 * 1024 * 1024, int)
+        log_backup_count = ConfigUtils.get_config_value([config_source], 'LOG_BACKUP_COUNT', 5, int)
+        log_console_enabled = ConfigUtils.get_config_value([config_source], 'LOG_CONSOLE_ENABLED', True, bool)
+        log_file_enabled = ConfigUtils.get_config_value([config_source], 'LOG_FILE_ENABLED', True, bool)
         
         # 创建日志配置
         return LogConfig(
@@ -176,10 +177,8 @@ class LoggingInitializer(BaseInitializer):
                         settings_module = importlib.import_module(settings_module_path)
                         
                         # 创建配置字典
-                        project_config = {}
-                        for key in dir(settings_module):
-                            if key.isupper():
-                                project_config[key] = getattr(settings_module, key)
+                        from crawlo.utils.config_manager import ConfigUtils
+                        project_config = ConfigUtils.merge_config_sources([settings_module])
                         
                         return project_config
                     
@@ -268,50 +267,8 @@ class CoreComponentsInitializer(BaseInitializer):
                 error=e
             )
     
-    def _initialize_engine(self, context: InitializationContext):
-        """初始化引擎"""
-        try:
-            # 注意：Engine需要crawler参数，不能在此阶段初始化
-            pass
-        except Exception as e:
-            context.add_error(f"Failed to initialize engine: {e}")
-            raise
-    
-    def _initialize_scheduler(self, context: InitializationContext):
-        """初始化调度器"""
-        try:
-            # 注意：Scheduler需要很多参数，不能在此阶段初始化
-            pass
-        except Exception as e:
-            context.add_error(f"Failed to initialize scheduler: {e}")
-            raise
-    
-    def _initialize_downloader(self, context: InitializationContext):
-        """初始化下载器"""
-        try:
-            # 注意：下载器类需要crawler参数，不能在此阶段初始化实例
-            pass
-        except Exception as e:
-            context.add_error(f"Failed to initialize downloader: {e}")
-            raise
-    
-    def _initialize_pipeline_manager(self, context: InitializationContext):
-        """初始化管道管理器"""
-        try:
-            # 注意：PipelineManager需要crawler参数，不能在此阶段初始化
-            pass
-        except Exception as e:
-            context.add_error(f"Failed to initialize pipeline manager: {e}")
-            raise
-    
-    def _initialize_middleware_manager(self, context: InitializationContext):
-        """初始化中间件管理器"""
-        try:
-            # 注意：MiddlewareManager需要crawler参数，不能在此阶段初始化
-            pass
-        except Exception as e:
-            context.add_error(f"Failed to initialize middleware manager: {e}")
-            raise
+# 注意：核心组件需要crawler参数，不能在此阶段初始化
+        # 实际初始化将在crawler创建时进行
 
 
 class ExtensionsInitializer(BaseInitializer):