crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/queue/queue_manager.py

@@ -7,15 +7,20 @@
 import asyncio
 import traceback
 from enum import Enum
-from typing import Optional, Dict, Any, Union
+from typing import Optional, Dict, Any, Union, TYPE_CHECKING
+import time
+import random
+
+if TYPE_CHECKING:
+    from crawlo import Request
 
-from crawlo import Request
 from crawlo.queue.pqueue import SpiderPriorityQueue
 from crawlo.utils.error_handler import ErrorHandler
 from crawlo.utils.log import get_logger
 from crawlo.utils.request_serializer import RequestSerializer
 
 try:
+    # 使用完整版Redis队列
     from crawlo.queue.redis_priority_queue import RedisPriorityQueue
 
     REDIS_AVAILABLE = True
@@ -31,6 +36,79 @@ class QueueType(Enum):
     AUTO = "auto"  # 自动选择
 
 
+class IntelligentScheduler:
+    """智能调度器"""
+
+    def __init__(self):
+        self.domain_stats = {}  # 域名统计信息
+        self.url_stats = {}  # URL统计信息
+        self.last_request_time = {}  # 最后请求时间
+
+    def calculate_priority(self, request: "Request") -> int:
+        """计算请求的智能优先级"""
+        priority = getattr(request, 'priority', 0)
+
+        # 获取域名
+        domain = self._extract_domain(request.url)
+
+        # 基于域名访问频率调整优先级
+        if domain in self.domain_stats:
+            domain_access_count = self.domain_stats[domain]['count']
+            last_access_time = self.domain_stats[domain]['last_time']
+
+            # 如果最近访问过该域名，降低优先级（避免过度集中访问同一域名）
+            time_since_last = time.time() - last_access_time
+            if time_since_last < 5:  # 5秒内访问过
+                priority -= 2
+            elif time_since_last < 30:  # 30秒内访问过
+                priority -= 1
+
+            # 如果该域名访问次数过多，进一步降低优先级
+            if domain_access_count > 10:
+                priority -= 1
+
+        # 基于URL访问历史调整优先级
+        if request.url in self.url_stats:
+            url_access_count = self.url_stats[request.url]
+            if url_access_count > 1:
+                # 重复URL降低优先级
+                priority -= url_access_count
+
+        # 基于深度调整优先级
+        depth = getattr(request, 'meta', {}).get('depth', 0)
+        priority -= depth  # 深度越大，优先级越低
+
+        return priority
+
+    def update_stats(self, request: "Request"):
+        """更新统计信息"""
+        domain = self._extract_domain(request.url)
+
+        # 更新域名统计
+        if domain not in self.domain_stats:
+            self.domain_stats[domain] = {'count': 0, 'last_time': 0}
+
+        self.domain_stats[domain]['count'] += 1
+        self.domain_stats[domain]['last_time'] = time.time()
+
+        # 更新URL统计
+        if request.url not in self.url_stats:
+            self.url_stats[request.url] = 0
+        self.url_stats[request.url] += 1
+
+        # 更新最后请求时间
+        self.last_request_time[domain] = time.time()
+
+    def _extract_domain(self, url: str) -> str:
+        """提取域名"""
+        try:
+            from urllib.parse import urlparse
+            parsed = urlparse(url)
+            return parsed.netloc
+        except:
+            return "unknown"
+
+
 class QueueConfig:
     """Queue configuration class"""
 
@@ -87,13 +165,27 @@ class QueueManager:
 
     def __init__(self, config: QueueConfig):
         self.config = config
-        self.logger = get_logger(self.__class__.__name__)
-        self.error_handler = ErrorHandler(self.__class__.__name__)
+        # 延迟初始化logger和error_handler避免循环依赖
+        self._logger = None
+        self._error_handler = None
         self.request_serializer = RequestSerializer()
         self._queue = None
         self._queue_semaphore = None
         self._queue_type = None
         self._health_status = "unknown"
+        self._intelligent_scheduler = IntelligentScheduler()  # 智能调度器
+
+    @property
+    def logger(self):
+        if self._logger is None:
+            self._logger = get_logger(self.__class__.__name__)
+        return self._logger
+
+    @property
+    def error_handler(self):
+        if self._error_handler is None:
+            self._error_handler = ErrorHandler(self.__class__.__name__)
+        return self._error_handler
 
     async def initialize(self) -> bool:
         """初始化队列"""
@@ -129,38 +221,44 @@ class QueueManager:
             self._health_status = "error"
             return False
 
-    async def put(self, request: Request, priority: int = 0) -> bool:
+    async def put(self, request: "Request", priority: int = 0) -> bool:
         """Unified enqueue interface"""
         if not self._queue:
             raise RuntimeError("队列未初始化")
 
         try:
+            # 应用智能调度算法计算优先级
+            intelligent_priority = self._intelligent_scheduler.calculate_priority(request)
+            # 结合原始优先级和智能优先级
+            final_priority = priority + intelligent_priority
+
+            # 更新统计信息
+            self._intelligent_scheduler.update_stats(request)
+
             # 序列化处理（仅对 Redis 队列）
             if self._queue_type == QueueType.REDIS:
                 request = self.request_serializer.prepare_for_serialization(request)
 
             # 背压控制（仅对内存队列）
             if self._queue_semaphore:
-                # 对于大量请求，使用非阻塞式检查
-                if not self._queue_semaphore.locked():
-                    await self._queue_semaphore.acquire()
-                else:
-                    # 如果队列已满，返回 False 而不是阻塞
-                    self.logger.warning("Queue is full, skipping current request")
-                    return False
+                # 对于大量请求，使用阻塞式等待而不是跳过
+                # 这样可以确保不会丢失任何请求
+                await self._queue_semaphore.acquire()
 
             # 统一的入队操作
             if hasattr(self._queue, 'put'):
                 if self._queue_type == QueueType.REDIS:
-                    success = await self._queue.put(request, priority)
+                    success = await self._queue.put(request, final_priority)
                 else:
-                    await self._queue.put(request)
+                    # 对于内存队列，我们需要手动处理优先级
+                    # 在SpiderPriorityQueue中，元素应该是(priority, item)的元组
+                    await self._queue.put((final_priority, request))
                     success = True
             else:
                 raise RuntimeError(f"队列类型 {self._queue_type} 不支持 put 操作")
 
             if success:
-                self.logger.debug(f"Request enqueued successfully: {request.url}")
+                self.logger.debug(f"Request enqueued successfully: {request.url} with priority {final_priority}")
 
             return success
 
@@ -170,7 +268,7 @@ class QueueManager:
                 self._queue_semaphore.release()
             return False
 
-    async def get(self, timeout: float = 5.0) -> Optional[Request]:
+    async def get(self, timeout: float = 5.0) -> Optional["Request"]:
         """Unified dequeue interface"""
         if not self._queue:
             raise RuntimeError("队列未初始化")
@@ -188,6 +286,11 @@ class QueueManager:
                 # 实际的 callback 恢复在 scheduler 中处理
                 pass
 
+            # 如果是内存队列，需要解包(priority, request)元组
+            if request and self._queue_type == QueueType.MEMORY:
+                if isinstance(request, tuple) and len(request) == 2:
+                    request = request[1]  # 取元组中的请求对象
+
             return request
 
         except Exception as e:
@@ -215,7 +318,12 @@ class QueueManager:
         try:
             # 对于内存队列，可以同步检查
             if self._queue_type == QueueType.MEMORY:
-                return self._queue.qsize() == 0
+                # 确保正确检查队列大小
+                if hasattr(self._queue, 'qsize'):
+                    return self._queue.qsize() == 0
+                else:
+                    # 如果没有qsize方法，假设队列为空
+                    return True
             # 对于 Redis 队列，由于需要异步操作，这里返回近似值
             # 为了确保程序能正常退出，我们返回True，让上层通过更精确的异步检查来判断
             return True
@@ -227,7 +335,16 @@ class QueueManager:
         try:
             # 对于内存队列
             if self._queue_type == QueueType.MEMORY:
-                return self._queue.qsize() == 0
+                # 确保正确检查队列大小
+                if hasattr(self._queue, 'qsize'):
+                    if asyncio.iscoroutinefunction(self._queue.qsize):
+                        size = await self._queue.qsize()
+                    else:
+                        size = self._queue.qsize()
+                    return size == 0
+                else:
+                    # 如果没有qsize方法，假设队列为空
+                    return True
             # 对于 Redis 队列，使用异步检查
             elif self._queue_type == QueueType.REDIS:
                 size = await self.size()
@@ -262,10 +379,10 @@ class QueueManager:
             if REDIS_AVAILABLE and self.config.redis_url:
                 # 测试 Redis 连接
                 try:
+                    from crawlo.queue.redis_priority_queue import RedisPriorityQueue
                     test_queue = RedisPriorityQueue(self.config.redis_url)
                     await test_queue.connect()
                     await test_queue.close()
-                    # Change INFO level log to DEBUG level to avoid redundant output
                     self.logger.debug("Auto-detection: Redis available, using distributed queue")
                     return QueueType.REDIS
                 except Exception as e:
@@ -282,6 +399,7 @@ class QueueManager:
                 raise RuntimeError("Redis 队列不可用：未配置 REDIS_URL")
             # 测试 Redis 连接
             try:
+                from crawlo.queue.redis_priority_queue import RedisPriorityQueue
                 test_queue = RedisPriorityQueue(self.config.redis_url)
                 await test_queue.connect()
                 await test_queue.close()
@@ -299,6 +417,12 @@ class QueueManager:
     async def _create_queue(self, queue_type: QueueType):
         """Create queue instance"""
         if queue_type == QueueType.REDIS:
+            # 延迟导入Redis队列
+            try:
+                from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+            except ImportError as e:
+                raise RuntimeError(f"Redis队列不可用：未能导入RedisPriorityQueue ({e})")
+
             # 简化项目名称提取逻辑
             project_name = "default"
             if ':' in self.config.queue_name:

crawlo/queue/redis_priority_queue.py

@@ -1,19 +1,38 @@
 import asyncio
+import asyncio
 import pickle
 import time
 import traceback
-from typing import Optional
+from typing import Optional, TYPE_CHECKING
 
 import redis.asyncio as aioredis
 
-from crawlo import Request
+# 使用 TYPE_CHECKING 避免运行时循环导入
+if TYPE_CHECKING:
+    from crawlo import Request
+
 from crawlo.utils.error_handler import ErrorHandler
 from crawlo.utils.log import get_logger
 from crawlo.utils.redis_connection_pool import get_redis_pool, OptimizedRedisConnectionPool
 from crawlo.utils.request_serializer import RequestSerializer
 
-logger = get_logger(__name__)
-error_handler = ErrorHandler(__name__)
+# 延迟初始化避免循环依赖
+_logger = None
+_error_handler = None
+
+
+def get_module_logger():
+    global _logger
+    if _logger is None:
+        _logger = get_logger(__name__)
+    return _logger
+
+
+def get_module_error_handler():
+    global _error_handler
+    if _error_handler is None:
+        _error_handler = ErrorHandler(__name__)
+    return _error_handler
 
 
 class RedisPriorityQueue:
@@ -39,14 +58,14 @@ class RedisPriorityQueue:
 
         self.redis_url = redis_url
         self.module_name = module_name  # 保存 module_name
-        
+
         # 如果未提供 queue_name，则根据 module_name 自动生成
         if queue_name is None:
             self.queue_name = f"crawlo:{module_name}:queue:requests"
         else:
             # 保持用户提供的队列名称不变，不做修改
             self.queue_name = queue_name
-        
+
         # 如果未提供 processing_queue，则根据 queue_name 自动生成
         if processing_queue is None:
             if ":queue:requests" in self.queue_name:
@@ -55,7 +74,7 @@ class RedisPriorityQueue:
                 self.processing_queue = f"{self.queue_name}:processing"
         else:
             self.processing_queue = processing_queue
-            
+
         # 如果未提供 failed_queue，则根据 queue_name 自动生成
         if failed_queue is None:
             if ":queue:requests" in self.queue_name:
@@ -64,7 +83,7 @@ class RedisPriorityQueue:
                 self.failed_queue = f"{self.queue_name}:failed"
         else:
             self.failed_queue = failed_queue
-        
+
         self.max_retries = max_retries
         self.timeout = timeout
         self.max_connections = max_connections
@@ -98,18 +117,18 @@ class RedisPriorityQueue:
                         decode_responses=False,  # 确保不自动解码响应
                         encoding='utf-8'
                     )
-                    
+
                     self._redis = await self._redis_pool.get_connection()
-                    
+
                     # 测试连接
                     await self._redis.ping()
                     # 只在调试模式下输出详细连接信息
-                    # logger.debug(f"Redis 连接成功 (Module: {self.module_name})")  # 注释掉重复的日志
+                    # get_module_logger().debug(f"Redis 连接成功 (Module: {self.module_name})")  # 注释掉重复的日志
                     return self._redis
                 except Exception as e:
                     error_msg = f"Redis 连接失败 (尝试 {attempt + 1}/{max_retries}, Module: {self.module_name}): {e}"
-                    logger.warning(error_msg)
-                    logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
+                    get_module_logger().warning(error_msg)
+                    get_module_logger().debug(f"详细错误信息:\n{traceback.format_exc()}")
                     if attempt < max_retries - 1:
                         await asyncio.sleep(delay)
                     else:
@@ -122,46 +141,46 @@ class RedisPriorityQueue:
         try:
             await self._redis.ping()
         except Exception as e:
-            logger.warning(f"Redis 连接失效 (Module: {self.module_name})，尝试重连...: {e}")
+            get_module_logger().warning(f"Redis 连接失效 (Module: {self.module_name})，尝试重连...: {e}")
             self._redis = None
             await self.connect()
 
-    async def put(self, request: Request, priority: int = 0) -> bool:
+    async def put(self, request, priority: int = 0) -> bool:
         """放入请求到队列"""
         try:
             await self._ensure_connection()
             score = -priority
             key = self._get_request_key(request)
-            
+
             # 🔥 使用专用的序列化工具清理 Request
             clean_request = self.request_serializer.prepare_for_serialization(request)
-            
+
             # 确保序列化后的数据可以被正确反序列化
             try:
                 serialized = pickle.dumps(clean_request)
                 # 验证序列化数据可以被反序列化
                 pickle.loads(serialized)
             except Exception as serialize_error:
-                logger.error(f"请求序列化验证失败 (Module: {self.module_name}): {serialize_error}")
+                get_module_logger().error(f"请求序列化验证失败 (Module: {self.module_name}): {serialize_error}")
                 return False
-            
+
             pipe = self._redis.pipeline()
             pipe.zadd(self.queue_name, {key: score})
             pipe.hset(f"{self.queue_name}:data", key, serialized)
             result = await pipe.execute()
-            
+
             if result[0] > 0:
-                logger.debug(f"成功入队 (Module: {self.module_name}): {request.url}")  # 注释掉重复的日志
+                get_module_logger().debug(f"成功入队 (Module: {self.module_name}): {request.url}")  # 注释掉重复的日志
             return result[0] > 0
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"放入队列失败 (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"放入队列失败 (Module: {self.module_name})",
                 raise_error=False
             )
             return False
 
-    async def get(self, timeout: float = 5.0) -> Optional[Request]:
+    async def get(self, timeout: float = 5.0):
         """
         获取请求（带超时）
         :param timeout: 最大等待时间（秒），避免无限轮询
@@ -198,7 +217,7 @@ class RedisPriorityQueue:
                         return request
                     except Exception as pickle_error:
                         # 如果pickle反序列化失败，记录错误并跳过这个任务
-                        logger.error(f"无法反序列化请求数据 (Module: {self.module_name}): {pickle_error}")
+                        get_module_logger().error(f"无法反序列化请求数据 (Module: {self.module_name}): {pickle_error}")
                         # 从processing队列中移除这个无效的任务
                         await self._redis.zrem(self.processing_queue, processing_key)
                         await self._redis.hdel(f"{self.processing_queue}:data", processing_key)
@@ -209,18 +228,18 @@ class RedisPriorityQueue:
                 if asyncio.get_event_loop().time() - start_time > timeout:
                     return None
 
-                # 短暂等待，避免空轮询
-                await asyncio.sleep(0.1)
+                # 短暂等待，避免空轮询，但减少等待时间以提高响应速度
+                await asyncio.sleep(0.001)  # 从0.01减少到0.001
 
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"获取队列任务失败 (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"获取队列任务失败 (Module: {self.module_name})",
                 raise_error=False
             )
             return None
 
-    async def ack(self, request: Request):
+    async def ack(self, request: "Request"):
         """确认任务完成"""
         try:
             await self._ensure_connection()
@@ -237,13 +256,13 @@ class RedisPriorityQueue:
                 if cursor == 0:
                     break
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"确认任务完成失败 (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"确认任务完成失败 (Module: {self.module_name})",
                 raise_error=False
             )
 
-    async def fail(self, request: Request, reason: str = ""):
+    async def fail(self, request: "Request", reason: str = ""):
         """标记任务失败"""
         try:
             await self._ensure_connection()
@@ -256,7 +275,8 @@ class RedisPriorityQueue:
 
             if retries <= self.max_retries:
                 await self.put(request, priority=request.priority + 1)
-                logger.info(f"任务重试 [{retries}/{self.max_retries}] (Module: {self.module_name}): {request.url}")
+                get_module_logger().info(
+                    f"任务重试 [{retries}/{self.max_retries}] (Module: {self.module_name}): {request.url}")
             else:
                 failed_data = {
                     "url": request.url,
@@ -266,15 +286,15 @@ class RedisPriorityQueue:
                     "request_pickle": pickle.dumps(request).hex(),  # 可选：保存完整请求
                 }
                 await self._redis.lpush(self.failed_queue, pickle.dumps(failed_data))
-                logger.error(f"任务彻底失败 [{retries}次] (Module: {self.module_name}): {request.url}")
+                get_module_logger().error(f"任务彻底失败 [{retries}次] (Module: {self.module_name}): {request.url}")
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"标记任务失败失败 (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"标记任务失败失败 (Module: {self.module_name})",
                 raise_error=False
             )
 
-    def _get_request_key(self, request: Request) -> str:
+    def _get_request_key(self, request) -> str:
         """生成请求唯一键"""
         return f"{self.module_name}:url:{hash(request.url) & 0x7FFFFFFF}"  # 确保正数
 
@@ -284,9 +304,9 @@ class RedisPriorityQueue:
             await self._ensure_connection()
             return await self._redis.zcard(self.queue_name)
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"Failed to get queue size (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"Failed to get queue size (Module: {self.module_name})",
                 raise_error=False
             )
             return 0
@@ -297,10 +317,10 @@ class RedisPriorityQueue:
             # 连接池会自动管理连接，这里不需要显式关闭单个连接
             self._redis = None
             self._redis_pool = None
-            logger.debug(f"Redis 连接已释放 (Module: {self.module_name})")
+            get_module_logger().debug(f"Redis 连接已释放 (Module: {self.module_name})")
         except Exception as e:
-            error_handler.handle_error(
-                e, 
-                context=f"释放 Redis 连接失败 (Module: {self.module_name})", 
+            get_module_error_handler().handle_error(
+                e,
+                context=f"释放 Redis 连接失败 (Module: {self.module_name})",
                 raise_error=False
             )