crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/pipelines/mysql_pipeline.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 import asyncio
 import aiomysql
-from typing import Optional
+from typing import Optional, List, Dict
 from asyncmy import create_pool
 from crawlo.utils.log import get_logger
 from crawlo.exceptions import ItemDiscard
-from crawlo.utils.db_helper import make_insert_sql, logger
+from crawlo.utils.db_helper import make_insert_sql, make_batch_sql, logger
 
 
 class AsyncmyMySQLPipeline:
@@ -24,6 +24,11 @@ class AsyncmyMySQLPipeline:
                 f"{crawler.spider.name}_items"
         )
 
+        # 批量插入配置
+        self.batch_size = self.settings.get_int('MYSQL_BATCH_SIZE', 100)
+        self.use_batch = self.settings.get_bool('MYSQL_USE_BATCH', False)
+        self.batch_buffer: List[Dict] = []  # 批量缓冲区
+
         # 注册关闭事件
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
@@ -59,30 +64,45 @@ class AsyncmyMySQLPipeline:
         """处理item的核心方法"""
         kwargs = kwargs or {}
         spider_name = getattr(spider, 'name', 'unknown')  # 获取爬虫名称
-        try:
-            await self._ensure_pool()
-            item_dict = dict(item)
-            sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
-
-            rowcount = await self._execute_sql(sql=sql)
-            if rowcount > 1:
-                self.logger.info(
-                    f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
-                )
-            elif rowcount == 1:
-                self.logger.debug(
-                    f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
-                )
-            else:
-                self.logger.warning(
-                    f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
-                )
-
+        
+        # 如果启用批量插入，将item添加到缓冲区
+        if self.use_batch:
+            self.batch_buffer.append(dict(item))
+            
+            # 如果缓冲区达到批量大小，执行批量插入
+            if len(self.batch_buffer) >= self.batch_size:
+                await self._flush_batch(spider_name)
+                
             return item
+        else:
+            # 单条插入逻辑
+            try:
+                await self._ensure_pool()
+                item_dict = dict(item)
+                sql = make_insert_sql(table=self.table_name, data=item_dict, **kwargs)
 
-        except Exception as e:
-            self.logger.error(f"处理item时发生错误: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
+                rowcount = await self._execute_sql(sql=sql)
+                if rowcount > 1:
+                    self.logger.info(
+                        f"爬虫 {spider_name} 成功插入 {rowcount} 条记录到表 {self.table_name}"
+                    )
+                elif rowcount == 1:
+                    self.logger.debug(
+                        f"爬虫 {spider_name} 成功插入单条记录到表 {self.table_name}"
+                    )
+                else:
+                    self.logger.warning(
+                        f"爬虫 {spider_name}: SQL执行成功但未插入新记录 - {sql[:100]}..."
+                    )
+
+                # 统计计数移到这里，与AiomysqlMySQLPipeline保持一致
+                self.crawler.stats.inc_value('mysql/insert_success')
+                return item
+
+            except Exception as e:
+                self.logger.error(f"处理item时发生错误: {e}")
+                self.crawler.stats.inc_value('mysql/insert_failed')
+                raise ItemDiscard(f"处理失败: {e}")
 
     async def _execute_sql(self, sql: str, values: list = None) -> int:
         """执行SQL语句并处理结果"""
@@ -96,15 +116,59 @@ class AsyncmyMySQLPipeline:
                         rowcount = await cursor.execute(sql)
 
                     await conn.commit()
-                    self.crawler.stats.inc_value('mysql/insert_success')
+                    # 移除这里的统计计数
                     return rowcount
                 except Exception as e:
                     await conn.rollback()
-                    self.crawler.stats.inc_value('mysql/insert_failed')
+                    # 移除这里的统计计数
                     raise ItemDiscard(f"MySQL插入失败: {e}")
 
+    async def _flush_batch(self, spider_name: str):
+        """刷新批量缓冲区并执行批量插入"""
+        if not self.batch_buffer:
+            return
+
+        try:
+            await self._ensure_pool()
+            
+            # 使用批量SQL生成函数
+            batch_result = make_batch_sql(table=self.table_name, datas=self.batch_buffer)
+            if batch_result is None:
+                self.logger.warning("批量插入数据为空")
+                self.batch_buffer.clear()
+                return
+
+            sql, values_list = batch_result
+
+            async with self.pool.acquire() as conn:
+                async with conn.cursor() as cursor:
+                    try:
+                        # 执行批量插入
+                        rowcount = await cursor.executemany(sql, values_list)
+                        await conn.commit()
+                        
+                        self.logger.info(
+                            f"爬虫 {spider_name} 批量插入 {rowcount} 条记录到表 {self.table_name}"
+                        )
+                        # 更新统计计数
+                        self.crawler.stats.inc_value('mysql/insert_success', rowcount)
+                        self.batch_buffer.clear()
+                    except Exception as e:
+                        await conn.rollback()
+                        self.crawler.stats.inc_value('mysql/insert_failed', len(self.batch_buffer))
+                        self.logger.error(f"批量插入失败: {e}")
+                        raise ItemDiscard(f"批量插入失败: {e}")
+        except Exception as e:
+            self.logger.error(f"批量插入过程中发生错误: {e}")
+            raise ItemDiscard(f"批量插入处理失败: {e}")
+
     async def spider_closed(self):
         """关闭爬虫时清理资源"""
+        # 在关闭前刷新剩余的批量数据
+        if self.use_batch and self.batch_buffer:
+            spider_name = getattr(self.crawler.spider, 'name', 'unknown')
+            await self._flush_batch(spider_name)
+            
         if self.pool:
             self.pool.close()
             await self.pool.wait_closed()
@@ -127,6 +191,11 @@ class AiomysqlMySQLPipeline:
                 f"{crawler.spider.name}_items"
         )
 
+        # 批量插入配置
+        self.batch_size = self.settings.get_int('MYSQL_BATCH_SIZE', 100)
+        self.use_batch = self.settings.get_bool('MYSQL_USE_BATCH', False)
+        self.batch_buffer: List[Dict] = []  # 批量缓冲区
+
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
     @classmethod
@@ -160,36 +229,89 @@ class AiomysqlMySQLPipeline:
 
     async def process_item(self, item, spider) -> Optional[dict]:
         """处理item方法"""
+        # 如果启用批量插入，将item添加到缓冲区
+        if self.use_batch:
+            self.batch_buffer.append(dict(item))
+            
+            # 如果缓冲区达到批量大小，执行批量插入
+            if len(self.batch_buffer) >= self.batch_size:
+                spider_name = getattr(spider, 'name', 'unknown')
+                await self._flush_batch(spider_name)
+                
+            return item
+        else:
+            # 单条插入逻辑
+            try:
+                await self._init_pool()
+
+                item_dict = dict(item)
+                # 使用make_insert_sql工具函数生成SQL
+                sql = make_insert_sql(table=self.table_name, data=item_dict)
+
+                async with self.pool.acquire() as conn:
+                    async with conn.cursor() as cursor:
+                        try:
+                            await cursor.execute(sql)
+                            await conn.commit()
+                            self.crawler.stats.inc_value('mysql/insert_success')
+                        except aiomysql.Error as e:
+                            await conn.rollback()
+                            self.crawler.stats.inc_value('mysql/insert_failed')
+                            raise ItemDiscard(f"MySQL错误: {e.args[1]}")
+
+                return item
+
+            except Exception as e:
+                self.logger.error(f"Pipeline处理异常: {e}")
+                raise ItemDiscard(f"处理失败: {e}")
+
+    async def _flush_batch(self, spider_name: str):
+        """刷新批量缓冲区并执行批量插入"""
+        if not self.batch_buffer:
+            return
+
         try:
             await self._init_pool()
+            
+            # 使用批量SQL生成函数
+            batch_result = make_batch_sql(table=self.table_name, datas=self.batch_buffer)
+            if batch_result is None:
+                self.logger.warning("批量插入数据为空")
+                self.batch_buffer.clear()
+                return
 
-            item_dict = dict(item)
-            sql = f"""
-            INSERT INTO `{self.table_name}` 
-            ({', '.join([f'`{k}`' for k in item_dict.keys()])})
-            VALUES ({', '.join(['%s'] * len(item_dict))})
-            """
+            sql, values_list = batch_result
 
             async with self.pool.acquire() as conn:
                 async with conn.cursor() as cursor:
                     try:
-                        await cursor.execute(sql, list(item_dict.values()))
+                        # 执行批量插入
+                        rowcount = await cursor.executemany(sql, values_list)
                         await conn.commit()
-                        self.crawler.stats.inc_value('mysql/insert_success')
-                    except aiomysql.Error as e:
+                        
+                        self.logger.info(
+                            f"爬虫 {spider_name} 批量插入 {rowcount} 条记录到表 {self.table_name}"
+                        )
+                        # 更新统计计数
+                        self.crawler.stats.inc_value('mysql/insert_success', rowcount)
+                        self.batch_buffer.clear()
+                    except Exception as e:
                         await conn.rollback()
-                        self.crawler.stats.inc_value('mysql/insert_failed')
-                        raise ItemDiscard(f"MySQL错误: {e.args[1]}")
-
-            return item
-
+                        self.crawler.stats.inc_value('mysql/insert_failed', len(self.batch_buffer))
+                        self.logger.error(f"批量插入失败: {e}")
+                        raise ItemDiscard(f"批量插入失败: {e}")
         except Exception as e:
-            self.logger.error(f"Pipeline处理异常: {e}")
-            raise ItemDiscard(f"处理失败: {e}")
+            self.logger.error(f"批量插入过程中发生错误: {e}")
+            raise ItemDiscard(f"批量插入处理失败: {e}")
 
     async def spider_closed(self):
         """资源清理"""
+        # 在关闭前刷新剩余的批量数据
+        if self.use_batch and self.batch_buffer:
+            spider_name = getattr(self.crawler.spider, 'name', 'unknown')
+            await self._flush_batch(spider_name)
+            
         if self.pool:
             self.pool.close()
             await self.pool.wait_closed()
-            self.logger.info("aiomysql连接池已释放")
+            self.logger.info("aiomysql连接池已释放")

crawlo/pipelines/redis_dedup_pipeline.py

@@ -72,12 +72,16 @@ class RedisDedupPipeline:
         """从爬虫配置创建管道实例"""
         settings = crawler.settings
         
+        # 使用统一的Redis key命名规范: crawlo:{project_name}:item:fingerprint
+        project_name = settings.get('PROJECT_NAME', 'default')
+        redis_key = f"crawlo:{project_name}:item:fingerprint"
+        
         return cls(
             redis_host=settings.get('REDIS_HOST', 'localhost'),
-            redis_port=settings.getint('REDIS_PORT', 6379),
-            redis_db=settings.getint('REDIS_DB', 0),
+            redis_port=settings.get_int('REDIS_PORT', 6379),
+            redis_db=settings.get_int('REDIS_DB', 0),
             redis_password=settings.get('REDIS_PASSWORD') or None,
-            redis_key=settings.get('REDIS_DEDUP_KEY', 'crawlo:item_fingerprints'),
+            redis_key=redis_key,
             log_level=settings.get('LOG_LEVEL', 'INFO')
         )
 

crawlo/queue/queue_manager.py

@@ -12,6 +12,7 @@ import os
 
 from crawlo.utils.log import get_logger
 from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.utils.error_handler import ErrorHandler
 from crawlo.queue.pqueue import SpiderPriorityQueue
 from crawlo import Request
 
@@ -87,6 +88,7 @@ class QueueManager:
     def __init__(self, config: QueueConfig):
         self.config = config
         self.logger = get_logger(self.__class__.__name__)
+        self.error_handler = ErrorHandler(self.__class__.__name__)
         self.request_serializer = RequestSerializer()
         self._queue = None
         self._queue_semaphore = None
@@ -110,7 +112,7 @@ class QueueManager:
         except Exception as e:
             # 记录详细的错误信息和堆栈跟踪
             self.logger.error(f"❌ 队列初始化失败: {e}")
-            self.logger.error(f"详细错误信息:\n{traceback.format_exc()}")
+            self.logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
             self._health_status = "error"
             return False
     
@@ -259,11 +261,22 @@ class QueueManager:
     async def _create_queue(self, queue_type: QueueType):
         """创建队列实例"""
         if queue_type == QueueType.REDIS:
+            # 从队列名称中提取项目名称，用于module_name
+            # 例如：crawlo:books_distributed:queue:requests -> books_distributed
+            project_name = "default"
+            if ':' in self.config.queue_name:
+                parts = self.config.queue_name.split(':')
+                if len(parts) >= 2:
+                    project_name = parts[1]  # 取第二个部分作为项目名称
+            else:
+                project_name = self.config.queue_name or "default"
+            
             queue = RedisPriorityQueue(
                 redis_url=self.config.redis_url,
                 queue_name=self.config.queue_name,
                 max_retries=self.config.max_retries,
-                timeout=self.config.timeout
+                timeout=self.config.timeout,
+                module_name=project_name  # 传递项目名称作为module_name
             )
             # 不需要立即连接，使用 lazy connect
             return queue

crawlo/queue/redis_priority_queue.py

@@ -9,46 +9,71 @@ import os
 from crawlo import Request
 from crawlo.utils.log import get_logger
 from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.redis_connection_pool import get_redis_pool, OptimizedRedisConnectionPool
 
 
 logger = get_logger(__name__)
+error_handler = ErrorHandler(__name__)
 
 
 class RedisPriorityQueue:
     """
-    基于 Redis 的分布式异步优先级队列（生产级优化版）
+    基于 Redis 的分布式异步优先级队列
     """
 
     def __init__(
             self,
             redis_url: str = None,
-            queue_name: str = "crawlo:requests",
-            processing_queue: str = "crawlo:processing",
-            failed_queue: str = "crawlo:failed",
+            queue_name: str = None,  # 修改默认值为 None
+            processing_queue: str = None,  # 修改默认值为 None
+            failed_queue: str = None,  # 修改默认值为 None
             max_retries: int = 3,
             timeout: int = 300,  # 任务处理超时时间（秒）
             max_connections: int = 10,  # 连接池大小
+            module_name: str = "default"  # 添加 module_name 参数
     ):
-        # 如果没有提供 redis_url，则从环境变量构造
+        # 移除直接使用 os.getenv()，要求通过参数传递 redis_url
         if redis_url is None:
-            redis_host = os.getenv('REDIS_HOST', 'localhost')
-            redis_port = os.getenv('REDIS_PORT', '6379')
-            redis_db = os.getenv('REDIS_DB', '0')
-            redis_password = os.getenv('REDIS_PASSWORD', '')
+            # 如果没有提供 redis_url，则抛出异常，要求在 settings 中配置
+            raise ValueError("redis_url must be provided. Configure it in settings instead of using os.getenv()")
+
+        self.redis_url = redis_url
+        self.module_name = module_name  # 保存 module_name
+        
+        # 如果未提供 queue_name，则根据 module_name 自动生成
+        if queue_name is None:
+            self.queue_name = f"crawlo:{module_name}:queue:requests"
+        else:
+            # 如果提供了 queue_name，但不符合规范格式，则转换为规范格式
+            if not queue_name.startswith("crawlo:"):
+                self.queue_name = f"crawlo:{module_name}:queue:requests"
+            else:
+                self.queue_name = queue_name
+        
+        # 如果未提供 processing_queue，则根据 queue_name 自动生成
+        if processing_queue is None:
+            if ":queue:requests" in self.queue_name:
+                self.processing_queue = self.queue_name.replace(":queue:requests", ":queue:processing")
+            else:
+                self.processing_queue = f"{self.queue_name}:processing"
+        else:
+            self.processing_queue = processing_queue
             
-            if redis_password:
-                redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+        # 如果未提供 failed_queue，则根据 queue_name 自动生成
+        if failed_queue is None:
+            if ":queue:requests" in self.queue_name:
+                self.failed_queue = self.queue_name.replace(":queue:requests", ":queue:failed")
             else:
-                redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
+                self.failed_queue = f"{self.queue_name}:failed"
+        else:
+            self.failed_queue = failed_queue
         
-        self.redis_url = redis_url
-        self.queue_name = queue_name
-        self.processing_queue = processing_queue
-        self.failed_queue = failed_queue
         self.max_retries = max_retries
         self.timeout = timeout
         self.max_connections = max_connections
-        self._redis = None
+        self._redis_pool: Optional[OptimizedRedisConnectionPool] = None
+        self._redis: Optional[aioredis.Redis] = None
         self._lock = asyncio.Lock()  # 用于连接初始化的锁
         self.request_serializer = RequestSerializer()  # 处理序列化
 
@@ -60,24 +85,30 @@ class RedisPriorityQueue:
 
             for attempt in range(max_retries):
                 try:
-                    self._redis = await aioredis.from_url(
+                    # 使用优化的连接池
+                    self._redis_pool = get_redis_pool(
                         self.redis_url,
-                        decode_responses=False,  # pickle 需要 bytes
                         max_connections=self.max_connections,
                         socket_connect_timeout=5,
                         socket_timeout=30,
+                        health_check_interval=30,
+                        retry_on_timeout=True
                     )
+                    
+                    self._redis = await self._redis_pool.get_connection()
+                    
                     # 测试连接
                     await self._redis.ping()
-                    logger.info("✅ Redis 连接成功")
+                    logger.info(f"✅ Redis 连接成功 (Module: {self.module_name})")
                     return self._redis
                 except Exception as e:
-                    logger.warning(f"⚠️ Redis 连接失败 (尝试 {attempt + 1}/{max_retries}): {e}")
-                    logger.warning(f"详细错误信息:\n{traceback.format_exc()}")
+                    error_msg = f"⚠️ Redis 连接失败 (尝试 {attempt + 1}/{max_retries}, Module: {self.module_name}): {e}"
+                    logger.warning(error_msg)
+                    logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
                     if attempt < max_retries - 1:
                         await asyncio.sleep(delay)
                     else:
-                        raise ConnectionError(f"❌ 无法连接 Redis: {e}")
+                        raise ConnectionError(f"❌ 无法连接 Redis (Module: {self.module_name}): {e}")
 
     async def _ensure_connection(self):
         """确保连接有效"""
@@ -86,16 +117,17 @@ class RedisPriorityQueue:
         try:
             await self._redis.ping()
         except Exception as e:
-            logger.warning(f"🔄 Redis 连接失效，尝试重连...: {e}")
+            logger.warning(f"🔄 Redis 连接失效 (Module: {self.module_name})，尝试重连...: {e}")
             self._redis = None
             await self.connect()
 
     async def put(self, request: Request, priority: int = 0) -> bool:
         """放入请求到队列"""
-        await self._ensure_connection()
-        score = -priority
-        key = self._get_request_key(request)
         try:
+            await self._ensure_connection()
+            score = -priority
+            key = self._get_request_key(request)
+            
             # 🔥 使用专用的序列化工具清理 Request
             clean_request = self.request_serializer.prepare_for_serialization(request)
             
@@ -106,11 +138,14 @@ class RedisPriorityQueue:
             result = await pipe.execute()
             
             if result[0] > 0:
-                logger.debug(f"✅ 成功入队: {request.url}")
+                logger.debug(f"✅ 成功入队 (Module: {self.module_name}): {request.url}")
             return result[0] > 0
         except Exception as e:
-            logger.error(f"❌ 放入队列失败: {e}")
-            logger.error(f"详细错误信息:\n{traceback.format_exc()}")
+            error_handler.handle_error(
+                e, 
+                context=f"放入队列失败 (Module: {self.module_name})", 
+                raise_error=False
+            )
             return False
 
     async def get(self, timeout: float = 5.0) -> Optional[Request]:
@@ -118,11 +153,11 @@ class RedisPriorityQueue:
         获取请求（带超时）
         :param timeout: 最大等待时间（秒），避免无限轮询
         """
-        await self._ensure_connection()
-        start_time = asyncio.get_event_loop().time()
+        try:
+            await self._ensure_connection()
+            start_time = asyncio.get_event_loop().time()
 
-        while True:
-            try:
+            while True:
                 # 尝试获取任务
                 result = await self._redis.zpopmin(self.queue_name, count=1)
                 if result:
@@ -148,62 +183,95 @@ class RedisPriorityQueue:
                 # 短暂等待，避免空轮询
                 await asyncio.sleep(0.1)
 
-            except Exception as e:
-                logger.error(f"❌ 获取队列任务失败: {e}")
-                logger.error(f"详细错误信息:\n{traceback.format_exc()}")
-                return None
+        except Exception as e:
+            error_handler.handle_error(
+                e, 
+                context=f"获取队列任务失败 (Module: {self.module_name})", 
+                raise_error=False
+            )
+            return None
 
     async def ack(self, request: Request):
         """确认任务完成"""
-        await self._ensure_connection()
-        key = self._get_request_key(request)
-        cursor = 0
-        while True:
-            cursor, keys = await self._redis.zscan(self.processing_queue, cursor, match=f"{key}:*")
-            if keys:
-                pipe = self._redis.pipeline()
-                for k in keys:
-                    pipe.zrem(self.processing_queue, k)
-                    pipe.hdel(f"{self.processing_queue}:data", k)
-                await pipe.execute()
-            if cursor == 0:
-                break
+        try:
+            await self._ensure_connection()
+            key = self._get_request_key(request)
+            cursor = 0
+            while True:
+                cursor, keys = await self._redis.zscan(self.processing_queue, cursor, match=f"{key}:*")
+                if keys:
+                    pipe = self._redis.pipeline()
+                    for k in keys:
+                        pipe.zrem(self.processing_queue, k)
+                        pipe.hdel(f"{self.processing_queue}:data", k)
+                    await pipe.execute()
+                if cursor == 0:
+                    break
+        except Exception as e:
+            error_handler.handle_error(
+                e, 
+                context=f"确认任务完成失败 (Module: {self.module_name})", 
+                raise_error=False
+            )
 
     async def fail(self, request: Request, reason: str = ""):
         """标记任务失败"""
-        await self._ensure_connection()
-        key = self._get_request_key(request)
-        await self.ack(request)
+        try:
+            await self._ensure_connection()
+            key = self._get_request_key(request)
+            await self.ack(request)
 
-        retry_key = f"{self.failed_queue}:retries:{key}"
-        retries = await self._redis.incr(retry_key)
-        await self._redis.expire(retry_key, 86400)
+            retry_key = f"{self.failed_queue}:retries:{key}"
+            retries = await self._redis.incr(retry_key)
+            await self._redis.expire(retry_key, 86400)
 
-        if retries <= self.max_retries:
-            await self.put(request, priority=request.priority + 1)
-            logger.info(f"🔁 任务重试 [{retries}/{self.max_retries}]: {request.url}")
-        else:
-            failed_data = {
-                "url": request.url,
-                "reason": reason,
-                "retries": retries,
-                "failed_at": time.time(),
-                "request_pickle": pickle.dumps(request).hex(),  # 可选：保存完整请求
-            }
-            await self._redis.lpush(self.failed_queue, pickle.dumps(failed_data))
-            logger.error(f"❌ 任务彻底失败 [{retries}次]: {request.url}")
+            if retries <= self.max_retries:
+                await self.put(request, priority=request.priority + 1)
+                logger.info(f"🔁 任务重试 [{retries}/{self.max_retries}] (Module: {self.module_name}): {request.url}")
+            else:
+                failed_data = {
+                    "url": request.url,
+                    "reason": reason,
+                    "retries": retries,
+                    "failed_at": time.time(),
+                    "request_pickle": pickle.dumps(request).hex(),  # 可选：保存完整请求
+                }
+                await self._redis.lpush(self.failed_queue, pickle.dumps(failed_data))
+                logger.error(f"❌ 任务彻底失败 [{retries}次] (Module: {self.module_name}): {request.url}")
+        except Exception as e:
+            error_handler.handle_error(
+                e, 
+                context=f"标记任务失败失败 (Module: {self.module_name})", 
+                raise_error=False
+            )
 
     def _get_request_key(self, request: Request) -> str:
         """生成请求唯一键"""
-        return f"url:{hash(request.url)}"
+        return f"{self.module_name}:url:{hash(request.url) & 0x7FFFFFFF}"  # 确保正数
 
     async def qsize(self) -> int:
         """获取队列大小"""
-        await self._ensure_connection()
-        return await self._redis.zcard(self.queue_name)
+        try:
+            await self._ensure_connection()
+            return await self._redis.zcard(self.queue_name)
+        except Exception as e:
+            error_handler.handle_error(
+                e, 
+                context=f"获取队列大小失败 (Module: {self.module_name})", 
+                raise_error=False
+            )
+            return 0
 
     async def close(self):
         """关闭连接"""
-        if self._redis:
-            await self._redis.close()
-            self._redis = None
+        try:
+            # 连接池会自动管理连接，这里不需要显式关闭单个连接
+            self._redis = None
+            self._redis_pool = None
+            logger.info(f"✅ Redis 连接已释放 (Module: {self.module_name})")
+        except Exception as e:
+            error_handler.handle_error(
+                e, 
+                context=f"释放 Redis 连接失败 (Module: {self.module_name})", 
+                raise_error=False
+            )