crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/queue/queue_manager.py

@@ -0,0 +1,304 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+统一的队列管理器
+提供简洁、一致的队列接口，自动处理不同队列类型的差异
+"""
+from typing import Optional, Dict, Any, Union
+from enum import Enum
+import asyncio
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.queue.pqueue import SpiderPriorityQueue
+from crawlo import Request
+
+try:
+    from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+    REDIS_AVAILABLE = True
+except ImportError:
+    RedisPriorityQueue = None
+    REDIS_AVAILABLE = False
+
+
+class QueueType(Enum):
+    """队列类型枚举"""
+    MEMORY = "memory"
+    REDIS = "redis"
+    AUTO = "auto"  # 自动选择
+
+
+class QueueConfig:
+    """队列配置类"""
+    
+    def __init__(
+        self,
+        queue_type: Union[QueueType, str] = QueueType.AUTO,
+        redis_url: Optional[str] = None,
+        redis_host: str = "127.0.0.1",
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        redis_db: int = 0,
+        queue_name: str = "crawlo:requests",
+        max_queue_size: int = 1000,
+        max_retries: int = 3,
+        timeout: int = 300,
+        **kwargs
+    ):
+        self.queue_type = QueueType(queue_type) if isinstance(queue_type, str) else queue_type
+        
+        # Redis 配置
+        if redis_url:
+            self.redis_url = redis_url
+        else:
+            if redis_password:
+                self.redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+            else:
+                self.redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
+        
+        self.queue_name = queue_name
+        self.max_queue_size = max_queue_size
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.extra_config = kwargs
+    
+    @classmethod
+    def from_settings(cls, settings) -> 'QueueConfig':
+        """从 settings 创建配置"""
+        return cls(
+            queue_type=settings.get('QUEUE_TYPE', QueueType.AUTO),
+            redis_url=settings.get('REDIS_URL'),
+            redis_host=settings.get('REDIS_HOST', '127.0.0.1'),
+            redis_port=settings.get_int('REDIS_PORT', 6379),
+            redis_password=settings.get('REDIS_PASSWORD'),
+            redis_db=settings.get_int('REDIS_DB', 0),
+            queue_name=settings.get('SCHEDULER_QUEUE_NAME', 'crawlo:requests'),
+            max_queue_size=settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 1000),
+            max_retries=settings.get_int('QUEUE_MAX_RETRIES', 3),
+            timeout=settings.get_int('QUEUE_TIMEOUT', 300)
+        )
+
+
+class QueueManager:
+    """统一的队列管理器"""
+    
+    def __init__(self, config: QueueConfig):
+        self.config = config
+        self.logger = get_logger(self.__class__.__name__)
+        self.request_serializer = RequestSerializer()
+        self._queue = None
+        self._queue_semaphore = None
+        self._queue_type = None
+        self._health_status = "unknown"
+    
+    async def initialize(self) -> bool:
+        """初始化队列"""
+        try:
+            queue_type = await self._determine_queue_type()
+            self._queue = await self._create_queue(queue_type)
+            self._queue_type = queue_type
+            
+            # 测试队列健康状态
+            await self._health_check()
+            
+            self.logger.info(f"✅ 队列初始化成功: {queue_type.value}")
+            self.logger.info(f"📊 队列配置: {self._get_queue_info()}")
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"❌ 队列初始化失败: {e}")
+            self._health_status = "error"
+            return False
+    
+    async def put(self, request: Request, priority: int = 0) -> bool:
+        """统一的入队接口"""
+        if not self._queue:
+            raise RuntimeError("队列未初始化")
+        
+        try:
+            # 序列化处理（仅对 Redis 队列）
+            if self._queue_type == QueueType.REDIS:
+                request = self.request_serializer.prepare_for_serialization(request)
+            
+            # 背压控制（仅对内存队列）
+            if self._queue_semaphore:
+                # 对于大量请求，使用非阻塞式检查
+                if not self._queue_semaphore.locked():
+                    await self._queue_semaphore.acquire()
+                else:
+                    # 如果队列已满，返回 False 而不是阻塞
+                    self.logger.warning("队列已满，跳过当前请求")
+                    return False
+            
+            # 统一的入队操作
+            if hasattr(self._queue, 'put'):
+                if self._queue_type == QueueType.REDIS:
+                    success = await self._queue.put(request, priority)
+                else:
+                    await self._queue.put(request)
+                    success = True
+            else:
+                raise RuntimeError(f"队列类型 {self._queue_type} 不支持 put 操作")
+            
+            if success:
+                self.logger.debug(f"✅ 请求入队成功: {request.url}")
+            
+            return success
+            
+        except Exception as e:
+            self.logger.error(f"❌ 请求入队失败: {e}")
+            if self._queue_semaphore:
+                self._queue_semaphore.release()
+            return False
+    
+    async def get(self, timeout: float = 5.0) -> Optional[Request]:
+        """统一的出队接口"""
+        if not self._queue:
+            raise RuntimeError("队列未初始化")
+        
+        try:
+            request = await self._queue.get(timeout=timeout)
+            
+            # 释放信号量（仅对内存队列）
+            if self._queue_semaphore and request:
+                self._queue_semaphore.release()
+            
+            # 反序列化处理（仅对 Redis 队列）
+            if request and self._queue_type == QueueType.REDIS:
+                # 这里需要 spider 实例，暂时返回原始请求
+                # 实际的 callback 恢复在 scheduler 中处理
+                pass
+            
+            return request
+            
+        except Exception as e:
+            self.logger.error(f"❌ 请求出队失败: {e}")
+            return None
+    
+    async def size(self) -> int:
+        """获取队列大小"""
+        if not self._queue:
+            return 0
+        
+        try:
+            if hasattr(self._queue, 'qsize'):
+                if asyncio.iscoroutinefunction(self._queue.qsize):
+                    return await self._queue.qsize()
+                else:
+                    return self._queue.qsize()
+            return 0
+        except Exception as e:
+            self.logger.warning(f"获取队列大小失败: {e}")
+            return 0
+    
+    def empty(self) -> bool:
+        """检查队列是否为空"""
+        try:
+            # 对于内存队列，可以同步检查
+            if self._queue_type == QueueType.MEMORY:
+                return self._queue.qsize() == 0
+            # 对于 Redis 队列，需要异步操作，这里返回近似值
+            return False
+        except Exception:
+            return True
+    
+    async def close(self) -> None:
+        """关闭队列"""
+        if self._queue and hasattr(self._queue, 'close'):
+            try:
+                await self._queue.close()
+                self.logger.info("✅ 队列已关闭")
+            except Exception as e:
+                self.logger.warning(f"关闭队列时发生错误: {e}")
+    
+    def get_status(self) -> Dict[str, Any]:
+        """获取队列状态信息"""
+        return {
+            "type": self._queue_type.value if self._queue_type else "unknown",
+            "health": self._health_status,
+            "config": self._get_queue_info(),
+            "initialized": self._queue is not None
+        }
+    
+    async def _determine_queue_type(self) -> QueueType:
+        """确定队列类型"""
+        if self.config.queue_type == QueueType.AUTO:
+            # 自动选择：优先使用 Redis（如果可用）
+            if REDIS_AVAILABLE and self.config.redis_url:
+                # 测试 Redis 连接
+                try:
+                    test_queue = RedisPriorityQueue(self.config.redis_url)
+                    await test_queue.connect()
+                    await test_queue.close()
+                    self.logger.info("🔍 自动检测: Redis 可用，使用分布式队列")
+                    return QueueType.REDIS
+                except Exception as e:
+                    self.logger.warning(f"🔍 自动检测: Redis 不可用 ({e})，使用内存队列")
+                    return QueueType.MEMORY
+            else:
+                self.logger.info("🔍 自动检测: Redis 未配置，使用内存队列")
+                return QueueType.MEMORY
+        
+        elif self.config.queue_type == QueueType.REDIS:
+            if not REDIS_AVAILABLE:
+                raise RuntimeError("Redis 队列不可用：未安装 redis 依赖")
+            if not self.config.redis_url:
+                raise RuntimeError("Redis 队列不可用：未配置 REDIS_URL")
+            return QueueType.REDIS
+        
+        elif self.config.queue_type == QueueType.MEMORY:
+            return QueueType.MEMORY
+        
+        else:
+            raise ValueError(f"不支持的队列类型: {self.config.queue_type}")
+    
+    async def _create_queue(self, queue_type: QueueType):
+        """创建队列实例"""
+        if queue_type == QueueType.REDIS:
+            queue = RedisPriorityQueue(
+                redis_url=self.config.redis_url,
+                queue_name=self.config.queue_name,
+                max_retries=self.config.max_retries,
+                timeout=self.config.timeout
+            )
+            # 不需要立即连接，使用 lazy connect
+            return queue
+        
+        elif queue_type == QueueType.MEMORY:
+            queue = SpiderPriorityQueue()
+            # 为内存队列设置背压控制
+            self._queue_semaphore = asyncio.Semaphore(self.config.max_queue_size)
+            return queue
+        
+        else:
+            raise ValueError(f"不支持的队列类型: {queue_type}")
+    
+    async def _health_check(self) -> None:
+        """健康检查"""
+        try:
+            if self._queue_type == QueueType.REDIS:
+                # 测试 Redis 连接
+                await self._queue.connect()
+                self._health_status = "healthy"
+            else:
+                # 内存队列总是健康的
+                self._health_status = "healthy"
+        except Exception as e:
+            self.logger.warning(f"队列健康检查失败: {e}")
+            self._health_status = "unhealthy"
+    
+    def _get_queue_info(self) -> Dict[str, Any]:
+        """获取队列配置信息"""
+        info = {
+            "queue_name": self.config.queue_name,
+            "max_queue_size": self.config.max_queue_size
+        }
+        
+        if self._queue_type == QueueType.REDIS:
+            info.update({
+                "redis_url": self.config.redis_url,
+                "max_retries": self.config.max_retries,
+                "timeout": self.config.timeout
+            })
+        
+        return info

crawlo/queue/redis_priority_queue.py

@@ -0,0 +1,192 @@
+import pickle
+import time
+import asyncio
+from typing import Optional
+import redis.asyncio as aioredis
+
+from crawlo import Request
+from crawlo.utils.log import get_logger
+from crawlo.utils.request_serializer import RequestSerializer
+
+
+logger = get_logger(__name__)
+
+
+class RedisPriorityQueue:
+    """
+    基于 Redis 的分布式异步优先级队列（生产级优化版）
+    """
+
+    def __init__(
+            self,
+            redis_url: str = "redis://localhost:6379/0",
+            queue_name: str = "crawlo:requests",
+            processing_queue: str = "crawlo:processing",
+            failed_queue: str = "crawlo:failed",
+            max_retries: int = 3,
+            timeout: int = 300,  # 任务处理超时时间（秒）
+            max_connections: int = 10,  # 连接池大小
+    ):
+        self.redis_url = redis_url
+        self.queue_name = queue_name
+        self.processing_queue = processing_queue
+        self.failed_queue = failed_queue
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.max_connections = max_connections
+        self._redis = None
+        self._lock = asyncio.Lock()  # 用于连接初始化的锁
+        self.request_serializer = RequestSerializer()  # 处理序列化
+
+    async def connect(self, max_retries=3, delay=1):
+        """异步连接 Redis，支持重试"""
+        async with self._lock:
+            if self._redis is not None:
+                return self._redis
+
+            for attempt in range(max_retries):
+                try:
+                    self._redis = await aioredis.from_url(
+                        self.redis_url,
+                        decode_responses=False,  # pickle 需要 bytes
+                        max_connections=self.max_connections,
+                        socket_connect_timeout=5,
+                        socket_timeout=30,
+                    )
+                    # 测试连接
+                    await self._redis.ping()
+                    logger.info("✅ Redis 连接成功")
+                    return self._redis
+                except Exception as e:
+                    logger.warning(f"⚠️ Redis 连接失败 (尝试 {attempt + 1}/{max_retries}): {e}")
+                    if attempt < max_retries - 1:
+                        await asyncio.sleep(delay)
+                    else:
+                        raise ConnectionError(f"❌ 无法连接 Redis: {e}")
+
+    async def _ensure_connection(self):
+        """确保连接有效"""
+        if self._redis is None:
+            await self.connect()
+        try:
+            await self._redis.ping()
+        except Exception:
+            logger.warning("🔄 Redis 连接失效，尝试重连...")
+            self._redis = None
+            await self.connect()
+
+    async def put(self, request: Request, priority: int = 0) -> bool:
+        """放入请求到队列"""
+        await self._ensure_connection()
+        score = -priority
+        key = self._get_request_key(request)
+        try:
+            # 🔥 使用专用的序列化工具清理 Request
+            clean_request = self.request_serializer.prepare_for_serialization(request)
+            
+            serialized = pickle.dumps(clean_request)
+            pipe = self._redis.pipeline()
+            pipe.zadd(self.queue_name, {key: score})
+            pipe.hset(f"{self.queue_name}:data", key, serialized)
+            result = await pipe.execute()
+            
+            if result[0] > 0:
+                logger.debug(f"✅ 成功入队: {request.url}")
+            return result[0] > 0
+        except Exception as e:
+            logger.error(f"❌ 放入队列失败: {e}")
+            return False
+
+    async def get(self, timeout: float = 5.0) -> Optional[Request]:
+        """
+        获取请求（带超时）
+        :param timeout: 最大等待时间（秒），避免无限轮询
+        """
+        await self._ensure_connection()
+        start_time = asyncio.get_event_loop().time()
+
+        while True:
+            try:
+                # 尝试获取任务
+                result = await self._redis.zpopmin(self.queue_name, count=1)
+                if result:
+                    key, score = result[0]
+                    serialized = await self._redis.hget(f"{self.queue_name}:data", key)
+                    if not serialized:
+                        continue
+
+                    # 移动到 processing
+                    processing_key = f"{key}:{int(time.time())}"
+                    pipe = self._redis.pipeline()
+                    pipe.zadd(self.processing_queue, {processing_key: time.time() + self.timeout})
+                    pipe.hset(f"{self.processing_queue}:data", processing_key, serialized)
+                    pipe.hdel(f"{self.queue_name}:data", key)
+                    await pipe.execute()
+
+                    return pickle.loads(serialized)
+
+                # 检查是否超时
+                if asyncio.get_event_loop().time() - start_time > timeout:
+                    return None
+
+                # 短暂等待，避免空轮询
+                await asyncio.sleep(0.1)
+
+            except Exception as e:
+                logger.error(f"❌ 获取队列任务失败: {e}")
+                return None
+
+    async def ack(self, request: Request):
+        """确认任务完成"""
+        await self._ensure_connection()
+        key = self._get_request_key(request)
+        cursor = 0
+        while True:
+            cursor, keys = await self._redis.zscan(self.processing_queue, cursor, match=f"{key}:*")
+            if keys:
+                pipe = self._redis.pipeline()
+                for k in keys:
+                    pipe.zrem(self.processing_queue, k)
+                    pipe.hdel(f"{self.processing_queue}:data", k)
+                await pipe.execute()
+            if cursor == 0:
+                break
+
+    async def fail(self, request: Request, reason: str = ""):
+        """标记任务失败"""
+        await self._ensure_connection()
+        key = self._get_request_key(request)
+        await self.ack(request)
+
+        retry_key = f"{self.failed_queue}:retries:{key}"
+        retries = await self._redis.incr(retry_key)
+        await self._redis.expire(retry_key, 86400)
+
+        if retries <= self.max_retries:
+            await self.put(request, priority=request.priority + 1)
+            logger.info(f"🔁 任务重试 [{retries}/{self.max_retries}]: {request.url}")
+        else:
+            failed_data = {
+                "url": request.url,
+                "reason": reason,
+                "retries": retries,
+                "failed_at": time.time(),
+                "request_pickle": pickle.dumps(request).hex(),  # 可选：保存完整请求
+            }
+            await self._redis.lpush(self.failed_queue, pickle.dumps(failed_data))
+            logger.error(f"❌ 任务彻底失败 [{retries}次]: {request.url}")
+
+    def _get_request_key(self, request: Request) -> str:
+        """生成请求唯一键"""
+        return f"url:{hash(request.url)}"
+
+    async def qsize(self) -> int:
+        """获取队列大小"""
+        await self._ensure_connection()
+        return await self._redis.zcard(self.queue_name)
+
+    async def close(self):
+        """关闭连接"""
+        if self._redis:
+            await self._redis.close()
+            self._redis = None

crawlo/settings/default_settings.py

@@ -16,10 +16,17 @@ PROJECT_NAME = 'crawlo'
 
 # ============================== 网络请求配置 ==============================
 
-# 下载器选择（三选一）
-# DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
-DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader"  # 支持浏览器指纹
-# DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"
+# 下载器选择（支持三种方式）
+# 方式1: 直接指定类路径
+DOWNLOADER = "crawlo.downloader.aiohttp_downloader.AioHttpDownloader"
+# DOWNLOADER = "crawlo.downloader.cffi_downloader.CurlCffiDownloader"  # 支持浏览器指纹
+# DOWNLOADER = "crawlo.downloader.httpx_downloader.HttpXDownloader"    # 支持HTTP/2
+
+# 方式2: 使用简化名称（推荐）
+# DOWNLOADER_TYPE = 'aiohttp'     # 可选: aiohttp, httpx, curl_cffi, cffi
+
+# 方式3: 在Spider中动态选择
+# 可以在Spider类中设置 custom_settings = {'DOWNLOADER_TYPE': 'httpx'}
 
 # 请求超时与安全
 DOWNLOAD_TIMEOUT = 30          # 下载超时时间（秒）
@@ -40,10 +47,15 @@ ALLOWED_CODES = []             # 允许的状态码（空表示不限制）
 
 # 连接与响应大小限制
 CONNECTION_POOL_LIMIT = 50    # 最大并发连接数（连接池大小）
+CONNECTION_POOL_LIMIT_PER_HOST = 20  # 每个主机的连接池大小
 DOWNLOAD_MAXSIZE = 10 * 1024 * 1024   # 最大响应体大小（10MB）
 DOWNLOAD_WARN_SIZE = 1024 * 1024      # 响应体警告阈值（1MB）
 DOWNLOAD_RETRY_TIMES = MAX_RETRY_TIMES  # 下载器内部重试次数（复用全局）
 
+# 下载统计配置
+DOWNLOADER_STATS = True        # 是否启用下载器统计功能
+DOWNLOAD_STATS = True          # 是否记录下载时间和大小统计
+
 # ============================== 并发与调度 ==============================
 
 CONCURRENCY = 8                # 单个爬虫的并发请求数
@@ -51,6 +63,23 @@ INTERVAL = 5                   # 日志统计输出间隔（秒）
 DEPTH_PRIORITY = 1             # 深度优先策略优先级
 MAX_RUNNING_SPIDERS = 3        # 最大同时运行的爬虫数
 
+# ============================== 队列配置 ==============================
+
+# 🎯 运行模式选择：'standalone'(单机), 'distributed'(分布式), 'auto'(自动检测)
+RUN_MODE = 'standalone'         # 默认单机模式，简单易用
+
+# 队列类型选择：'memory'(内存), 'redis'(分布式), 'auto'(自动选择)
+QUEUE_TYPE = 'memory'           # 默认内存队列，无需外部依赖
+SCHEDULER_MAX_QUEUE_SIZE = 2000  # 调度器队列最大容量
+SCHEDULER_QUEUE_NAME = 'crawlo:requests'  # Redis 队列名称
+QUEUE_MAX_RETRIES = 3          # 队列操作最大重试次数
+QUEUE_TIMEOUT = 300            # 队列操作超时时间（秒）
+
+# 大规模爬取优化配置
+LARGE_SCALE_BATCH_SIZE = 1000   # 批处理大小
+LARGE_SCALE_CHECKPOINT_INTERVAL = 5000  # 进度保存间隔
+LARGE_SCALE_MAX_MEMORY_USAGE = 500  # 最大内存使用量（MB）
+
 # ============================== 数据存储配置 ==============================
 
 # --- MySQL 配置 ---
@@ -82,13 +111,17 @@ REQUEST_DIR = '.'
 
 # 去重过滤器类（二选一）
 FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
-# FILTER_CLASS = 'crawlo.filters.redis_filter.AioRedisFilter'  # 分布式去重
+# FILTER_CLASS = 'crawlo.filters.aioredis_filter.AioRedisFilter' # 分布式去重
 
 # --- Redis 过滤器配置 ---
 REDIS_HOST = os.getenv('REDIS_HOST', '127.0.0.1')
 REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
-REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', 'oscar&0503')
-REDIS_URL = f'redis://:{REDIS_PASSWORD or ""}@{REDIS_HOST}:{REDIS_PORT}/0'
+REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')  # 默认无密码
+# 🔧 根据是否有密码生成不同的 URL 格式
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/0'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/0'
 REDIS_KEY = 'request_fingerprint'        # Redis 中存储指纹的键名
 REDIS_TTL = 0                            # 指纹过期时间（0 表示永不过期）
 CLEANUP_FP = 0                           # 程序结束时是否清理指纹（0=不清理）
@@ -153,15 +186,41 @@ CURL_BROWSER_TYPE = "chrome"               # 可选: chrome, edge, safari, firef
 # 自定义浏览器版本映射（可覆盖默认行为）
 CURL_BROWSER_VERSION_MAP = {
     "chrome": "chrome136",
-    "edge": "edge101",
+    "edge": "edge101", 
     "safari": "safari184",
     "firefox": "firefox135",
     # 示例：旧版本测试
     # "chrome_legacy": "chrome110",
 }
 
+# Curl-Cffi 优化配置
+CURL_RANDOMIZE_DELAY = False     # 是否启用随机延迟
+CURL_RETRY_BACKOFF = True        # 是否启用指数退避重试
+
 # 默认请求头（可被 Spider 覆盖）
 DEFAULT_REQUEST_HEADERS = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
-}
+}
+
+# ============================== 下载器优化配置 ==============================
+
+# 下载器健康检查
+DOWNLOADER_HEALTH_CHECK = True   # 是否启用下载器健康检查
+HEALTH_CHECK_INTERVAL = 60       # 健康检查间隔（秒）
+
+# 请求统计配置
+REQUEST_STATS_ENABLED = True     # 是否启用请求统计
+STATS_RESET_ON_START = False     # 启动时是否重置统计
+
+# HttpX 下载器专用配置
+HTTPX_HTTP2 = True               # 是否启用HTTP/2支持
+HTTPX_FOLLOW_REDIRECTS = True    # 是否自动跟随重定向
+
+# AioHttp 下载器专用配置
+AIOHTTP_AUTO_DECOMPRESS = True   # 是否自动解压响应
+AIOHTTP_FORCE_CLOSE = False      # 是否强制关闭连接
+
+# 通用优化配置
+CONNECTION_TTL_DNS_CACHE = 300   # DNS缓存TTL（秒）
+CONNECTION_KEEPALIVE_TIMEOUT = 15 # Keep-Alive超时（秒）