crawlo 1.4.6__py3-none-any.whl → 1.4.8__py3-none-any.whl

crawlo/pipelines/mongo_pipeline.py

@@ -1,16 +1,18 @@
 # -*- coding: utf-8 -*-
 from typing import Optional, List, Dict
-from motor.motor_asyncio import AsyncIOMotorClient
+
 from pymongo.errors import PyMongoError
-from crawlo.utils.log import get_logger
+
 from crawlo.exceptions import ItemDiscard
+from crawlo.logging import get_logger
+from crawlo.utils.mongo_connection_pool import MongoConnectionPoolManager
 
 
 class MongoPipeline:
     def __init__(self, crawler):
         self.crawler = crawler
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
 
         # 初始化连接参数
         self.client = None
@@ -43,17 +45,21 @@ class MongoPipeline:
     async def _ensure_connection(self):
         """确保连接已建立"""
         if self.client is None:
-            # 使用连接池配置创建客户端
-            self.client = AsyncIOMotorClient(
-                self.mongo_uri,
-                maxPoolSize=self.max_pool_size,
-                minPoolSize=self.min_pool_size,
-                connectTimeoutMS=self.connect_timeout_ms,
-                socketTimeoutMS=self.socket_timeout_ms
+            # 使用单例连接池管理器
+            self.client = await MongoConnectionPoolManager.get_client(
+                mongo_uri=self.mongo_uri,
+                db_name=self.db_name,
+                max_pool_size=self.max_pool_size,
+                min_pool_size=self.min_pool_size,
+                connect_timeout_ms=self.connect_timeout_ms,
+                socket_timeout_ms=self.socket_timeout_ms
             )
             self.db = self.client[self.db_name]
             self.collection = self.db[self.collection_name]
-            self.logger.info(f"MongoDB连接建立 (集合: {self.collection_name})")
+            self.logger.info(
+                f"MongoDB连接建立 (集合: {self.collection_name}, "
+                f"使用全局共享连接池)"
+            )
 
     async def process_item(self, item, spider) -> Optional[dict]:
         """处理item的核心方法（带重试机制）"""
@@ -126,7 +132,10 @@ class MongoPipeline:
         # 在关闭前刷新剩余的批量数据
         if self.use_batch and self.batch_buffer:
             await self._flush_batch(self.crawler.spider)
-            
+        
+        # 注意：不再关闭客户端，因为客户端是全局共享的
+        # 客户端的关闭由 MongoConnectionPoolManager.close_all_clients() 统一管理
         if self.client:
-            self.client.close()
-            self.logger.info("MongoDB连接已关闭")
+            self.logger.info(
+                f"MongoDB Pipeline 关闭，但保留全局共享连接池以供其他爬虫使用"
+            )

crawlo/pipelines/mysql_pipeline.py

@@ -1,15 +1,14 @@
 # -*- coding: utf-8 -*-
 import asyncio
-import aiomysql
-from asyncmy import create_pool
-from typing import Optional, List, Dict, Any
-from abc import ABC, abstractmethod
 import async_timeout
+from abc import ABC, abstractmethod
+from typing import List, Dict, Any
 
-from crawlo.items import Item
 from crawlo.exceptions import ItemDiscard
+from crawlo.items import Item
 from crawlo.utils.db_helper import SQLBuilder
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
+from crawlo.utils.mysql_connection_pool import MySQLConnectionPoolManager
 from . import BasePipeline
 
 
@@ -19,7 +18,7 @@ class BaseMySQLPipeline(BasePipeline, ABC):
     def __init__(self, crawler):
         self.crawler = crawler
         self.settings = crawler.settings
-        self.logger = get_logger(self.__class__.__name__, self.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
 
         # 记录管道初始化
         self.logger.info(f"MySQL pipeline initialized: {self.__class__.__name__}")
@@ -203,20 +202,13 @@ class BaseMySQLPipeline(BasePipeline, ABC):
                 await self._flush_batch(spider_name)
             except Exception as e:
                 self.logger.error(f"关闭爬虫时刷新批量数据失败: {e}")
-            
+        
+        # 注意：不再关闭连接池，因为连接池是全局共享的
+        # 连接池的关闭由 MySQLConnectionPoolManager.close_all_pools() 统一管理
         if self.pool:
-            try:
-                pool_stats = {
-                    'size': getattr(self.pool, 'size', 'unknown'),
-                    'minsize': getattr(self.pool, 'minsize', 'unknown'),
-                    'maxsize': getattr(self.pool, 'maxsize', 'unknown')
-                }
-                self.logger.info(f"正在关闭MySQL连接池，当前状态: {pool_stats}")
-                self.pool.close()
-                await self.pool.wait_closed()
-                self.logger.info("MySQL连接池已关闭")
-            except Exception as e:
-                self.logger.error(f"关闭MySQL连接池时发生错误: {e}")
+            self.logger.info(
+                f"MySQL Pipeline 关闭，但保留全局共享连接池以供其他爬虫使用"
+            )
             
     async def _make_insert_sql(self, item_dict: Dict, **kwargs) -> str:
         """生成插入SQL语句，子类可以重写此方法"""
@@ -253,9 +245,9 @@ class AsyncmyMySQLPipeline(BaseMySQLPipeline):
 
     async def _ensure_pool(self):
         """确保连接池已初始化（线程安全）"""
-        if self._pool_initialized:
+        if self._pool_initialized and self.pool:
             # 检查连接池是否仍然有效
-            if self.pool and hasattr(self.pool, 'closed') and not self.pool.closed:
+            if hasattr(self.pool, 'closed') and not self.pool.closed:
                 return
             else:
                 self.logger.warning("连接池已初始化但无效，重新初始化")
@@ -263,7 +255,9 @@ class AsyncmyMySQLPipeline(BaseMySQLPipeline):
         async with self._pool_lock:
             if not self._pool_initialized:  # 双重检查避免竞争条件
                 try:
-                    self.pool = await create_pool(
+                    # 使用单例连接池管理器
+                    self.pool = await MySQLConnectionPoolManager.get_pool(
+                        pool_type='asyncmy',
                         host=self.settings.get('MYSQL_HOST', 'localhost'),
                         port=self.settings.get_int('MYSQL_PORT', 3306),
                         user=self.settings.get('MYSQL_USER', 'root'),
@@ -274,11 +268,10 @@ class AsyncmyMySQLPipeline(BaseMySQLPipeline):
                         echo=self.settings.get_bool('MYSQL_ECHO', False)
                     )
                     self._pool_initialized = True
-                    pool_stats = {
-                        'minsize': getattr(self.pool, 'minsize', 'unknown'),
-                        'maxsize': getattr(self.pool, 'maxsize', 'unknown')
-                    }
-                    self.logger.info(f"MySQL连接池初始化完成（表: {self.table_name}, 配置: {pool_stats}）")
+                    self.logger.info(
+                        f"MySQL连接池初始化完成（表: {self.table_name}, "
+                        f"使用全局共享连接池）"
+                    )
                 except Exception as e:
                     self.logger.error(f"MySQL连接池初始化失败: {e}")
                     # 重置状态以便重试
@@ -391,9 +384,9 @@ class AiomysqlMySQLPipeline(BaseMySQLPipeline):
 
     async def _ensure_pool(self):
         """延迟初始化连接池（线程安全）"""
-        if self._pool_initialized:
+        if self._pool_initialized and self.pool:
             # 检查连接池是否仍然有效
-            if self.pool and hasattr(self.pool, 'closed') and not self.pool.closed:
+            if hasattr(self.pool, 'closed') and not self.pool.closed:
                 return
             else:
                 self.logger.warning("连接池已初始化但无效，重新初始化")
@@ -401,23 +394,22 @@ class AiomysqlMySQLPipeline(BaseMySQLPipeline):
         async with self._pool_lock:
             if not self._pool_initialized:
                 try:
-                    self.pool = await aiomysql.create_pool(
+                    # 使用单例连接池管理器
+                    self.pool = await MySQLConnectionPoolManager.get_pool(
+                        pool_type='aiomysql',
                         host=self.settings.get('MYSQL_HOST', 'localhost'),
                         port=self.settings.get_int('MYSQL_PORT', 3306),
                         user=self.settings.get('MYSQL_USER', 'root'),
                         password=self.settings.get('MYSQL_PASSWORD', ''),
                         db=self.settings.get('MYSQL_DB', 'scrapy_db'),
                         minsize=self.settings.get_int('MYSQL_POOL_MIN', 2),
-                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 5),
-                        cursorclass=aiomysql.DictCursor,
-                        autocommit=False
+                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 5)
                     )
                     self._pool_initialized = True
-                    pool_stats = {
-                        'minsize': getattr(self.pool, 'minsize', 'unknown'),
-                        'maxsize': getattr(self.pool, 'maxsize', 'unknown')
-                    }
-                    self.logger.info(f"aiomysql连接池已初始化（表: {self.table_name}, 配置: {pool_stats}）")
+                    self.logger.info(
+                        f"aiomysql连接池已初始化（表: {self.table_name}, "
+                        f"使用全局共享连接池）"
+                    )
                 except Exception as e:
                     self.logger.error(f"aiomysql连接池初始化失败: {e}")
                     # 重置状态以便重试

crawlo/pipelines/pipeline_manager.py

@@ -4,11 +4,11 @@ from typing import List
 from pprint import pformat
 from asyncio import create_task
 
-from crawlo.utils.log import get_logger
-from crawlo.event import item_successful, item_discard
+from crawlo.logging import get_logger
+from crawlo.event import CrawlerEvent
 from crawlo.utils.misc import load_object
 from crawlo.project import common_call
-from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError, DropItem
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
 
 
 def get_dedup_pipeline_classes():
@@ -34,7 +34,7 @@ class PipelineManager:
         self.pipelines: List = []
         self.methods: List = []
 
-        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
+        self.logger = get_logger(self.__class__.__name__)
         pipelines = self.crawler.settings.get_list('PIPELINES')
         dedup_pipeline = self.crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
 
@@ -88,13 +88,13 @@ class PipelineManager:
                     item = await common_call(method, item, self.crawler.spider)
                     if item is None:
                         raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-                except (ItemDiscard, DropItem) as exc:  # 同时捕获两种异常类型
+                except ItemDiscard as exc:
                     self.logger.debug(f"Item discarded by pipeline: {exc}")
-                    create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+                    create_task(self.crawler.subscriber.notify(CrawlerEvent.ITEM_DISCARD, item, exc, self.crawler.spider))
                     # 重新抛出异常，确保上层调用者也能捕获到，并停止执行后续管道
                     raise
-        except (ItemDiscard, DropItem):
+        except ItemDiscard:
             # 异常已经被处理和通知，这里只需要重新抛出
             raise
         else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+            create_task(self.crawler.subscriber.notify(CrawlerEvent.ITEM_SUCCESSFUL, item, self.crawler.spider))

crawlo/pipelines/redis_dedup_pipeline.py

@@ -17,9 +17,9 @@ from typing import Optional
 
 from crawlo import Item
 from crawlo.spider import Spider
-from crawlo.exceptions import DropItem, ItemDiscard
+from crawlo.exceptions import ItemDiscard
 from crawlo.utils.fingerprint import FingerprintGenerator
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 
 class RedisDedupPipeline:
@@ -31,8 +31,7 @@ class RedisDedupPipeline:
             redis_port: int = 6379,
             redis_db: int = 0,
             redis_password: Optional[str] = None,
-            redis_key: str = 'crawlo:item_fingerprints',
-            log_level: str = "INFO"
+            redis_key: str = 'crawlo:item_fingerprints'
     ):
         """
         初始化 Redis 去重管道
@@ -42,9 +41,8 @@ class RedisDedupPipeline:
         :param redis_db: Redis 数据库编号
         :param redis_password: Redis 密码
         :param redis_key: 存储指纹的 Redis 键名
-        :param log_level: 日志级别
         """
-        self.logger = get_logger(self.__class__.__name__, log_level)
+        self.logger = get_logger(self.__class__.__name__)
         
         # 初始化 Redis 连接
         try:
@@ -59,8 +57,6 @@ class RedisDedupPipeline:
             )
             # 测试连接
             self.redis_client.ping()
-            # Change INFO level log to DEBUG level to avoid redundant output
-            # self.logger.debug(f"Redis connection successful: {redis_host}:{redis_port}/{redis_db}")  # 注释掉重复的日志
         except Exception as e:
             self.logger.error(f"Redis connection failed: {e}")
             raise RuntimeError(f"Redis 连接失败: {e}")
@@ -82,8 +78,7 @@ class RedisDedupPipeline:
             redis_port=settings.get_int('REDIS_PORT', 6379),
             redis_db=settings.get_int('REDIS_DB', 0),
             redis_password=settings.get('REDIS_PASSWORD') or None,
-            redis_key=redis_key,
-            log_level=settings.get('LOG_LEVEL', 'INFO')
+            redis_key=redis_key
         )
 
     def process_item(self, item: Item, spider: Spider) -> Item:
@@ -92,7 +87,7 @@ class RedisDedupPipeline:
         
         :param item: 要处理的数据项
         :param spider: 爬虫实例
-        :return: 处理后的数据项或抛出 DropItem 异常
+        :return: 处理后的数据项或抛出 ItemDiscard 异常
         """
         try:
             # 生成数据项指纹
@@ -150,8 +145,12 @@ class RedisDedupPipeline:
             
             # 注意：默认情况下不清理 Redis 中的指纹
             # 如果需要清理，可以在设置中配置
-            if spider.crawler.settings.getbool('REDIS_DEDUP_CLEANUP', False):
-                deleted = self.redis_client.delete(self.redis_key)
-                self.logger.info(f"  - Cleaned fingerprints: {deleted}")
+            # 安全访问crawler和settings
+            crawler = getattr(spider, 'crawler', None)
+            if crawler and hasattr(crawler, 'settings'):
+                settings = crawler.settings
+                if settings.getbool('REDIS_DEDUP_CLEANUP', False):
+                    deleted = self.redis_client.delete(self.redis_key)
+                    self.logger.info(f"  - Cleaned fingerprints: {deleted}")
         except Exception as e:
             self.logger.error(f"Error closing spider: {e}")

crawlo/project.py

@@ -5,7 +5,7 @@ from inspect import iscoroutinefunction
 from typing import Callable, Optional, Any
 
 from crawlo.settings.setting_manager import SettingManager
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 
 # 使用全局logger，避免每个模块都创建自己的延迟初始化函数
 # 延迟获取logger，确保在日志系统配置之后获取

crawlo/queue/__init__.py

@@ -0,0 +1,10 @@
+"""队列管理模块"""
+from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
+from crawlo.queue.pqueue import SpiderPriorityQueue
+
+__all__ = [
+    'QueueManager',
+    'QueueConfig',
+    'QueueType',
+    'SpiderPriorityQueue',
+]

crawlo/queue/queue_manager.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
 
 from crawlo.queue.pqueue import SpiderPriorityQueue
 from crawlo.utils.error_handler import ErrorHandler
-from crawlo.utils.log import get_logger
+from crawlo.logging import get_logger
 from crawlo.utils.request_serializer import RequestSerializer
 
 try:
@@ -123,9 +123,11 @@ class QueueConfig:
             max_queue_size: int = 1000,
             max_retries: int = 3,
             timeout: int = 300,
+            run_mode: Optional[str] = None,  # 新增：运行模式
             **kwargs
     ):
         self.queue_type = QueueType(queue_type) if isinstance(queue_type, str) else queue_type
+        self.run_mode = run_mode  # 保存运行模式
 
         # Redis 配置
         if redis_url:
@@ -166,7 +168,8 @@ class QueueConfig:
             queue_name=queue_name,
             max_queue_size=settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 1000),
             max_retries=settings.get_int('QUEUE_MAX_RETRIES', 3),
-            timeout=settings.get_int('QUEUE_TIMEOUT', 300)
+            timeout=settings.get_int('QUEUE_TIMEOUT', 300),
+            run_mode=settings.get('RUN_MODE')  # 传递运行模式
         )
 
 
@@ -224,6 +227,17 @@ class QueueManager:
 
             return False  # 默认不需要更新配置
 
+        except RuntimeError as e:
+            # Distributed 模式下的 RuntimeError 必须重新抛出
+            if self.config.run_mode == 'distributed':
+                self.logger.error(f"Queue initialization failed: {e}")
+                self._health_status = "error"
+                raise  # 重新抛出异常
+            # 其他模式记录错误但不抛出
+            self.logger.error(f"Queue initialization failed: {e}")
+            self.logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
+            self._health_status = "error"
+            return False
         except Exception as e:
             # 记录详细的错误信息和堆栈跟踪
             self.logger.error(f"Queue initialization failed: {e}")
@@ -403,23 +417,63 @@ class QueueManager:
                 return QueueType.MEMORY
 
         elif self.config.queue_type == QueueType.REDIS:
-            # 当 QUEUE_TYPE = 'redis' 时，行为等同于 'auto' 模式
-            # 优先使用 Redis（如果可用），如果不可用则回退到内存队列
-            if REDIS_AVAILABLE and self.config.redis_url:
+            # Distributed 模式：必须使用 Redis，不允许降级
+            if self.config.run_mode == 'distributed':
+                # 分布式模式必须确保 Redis 可用
+                if not REDIS_AVAILABLE:
+                    error_msg = (
+                        "Distributed 模式要求 Redis 可用，但 Redis 客户端库未安装。\n"
+                        "请安装 Redis 支持: pip install redis"
+                    )
+                    self.logger.error(error_msg)
+                    raise RuntimeError(error_msg)
+                
+                if not self.config.redis_url:
+                    error_msg = (
+                        "Distributed 模式要求配置 Redis 连接信息。\n"
+                        "请在 settings.py 中配置 REDIS_HOST、REDIS_PORT 等参数"
+                    )
+                    self.logger.error(error_msg)
+                    raise RuntimeError(error_msg)
+                
                 # 测试 Redis 连接
                 try:
                     from crawlo.queue.redis_priority_queue import RedisPriorityQueue
                     test_queue = RedisPriorityQueue(self.config.redis_url)
                     await test_queue.connect()
                     await test_queue.close()
-                    self.logger.debug("Redis mode: Redis available, using distributed queue")
+                    self.logger.debug("Distributed mode: Redis connection verified")
                     return QueueType.REDIS
                 except Exception as e:
-                    self.logger.debug(f"Redis mode: Redis unavailable ({e}), falling back to memory queue")
-                    return QueueType.MEMORY
+                    error_msg = (
+                        f"Distributed 模式要求 Redis 可用，但无法连接到 Redis 服务器。\n"
+                        f"错误信息: {e}\n"
+                        f"Redis URL: {self.config.redis_url}\n"
+                        f"请检查：\n"
+                        f"  1. Redis 服务是否正在运行\n"
+                        f"  2. Redis 连接配置是否正确\n"
+                        f"  3. 网络连接是否正常"
+                    )
+                    self.logger.error(error_msg)
+                    raise RuntimeError(error_msg) from e
             else:
-                self.logger.debug("Redis mode: Redis not configured, falling back to memory queue")
-                return QueueType.MEMORY
+                # 非 distributed 模式：QUEUE_TYPE='redis' 时允许降级到 memory
+                # 这提供了向后兼容性和更好的容错性
+                if REDIS_AVAILABLE and self.config.redis_url:
+                    # 测试 Redis 连接
+                    try:
+                        from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+                        test_queue = RedisPriorityQueue(self.config.redis_url)
+                        await test_queue.connect()
+                        await test_queue.close()
+                        self.logger.debug("Redis mode: Redis available, using distributed queue")
+                        return QueueType.REDIS
+                    except Exception as e:
+                        self.logger.warning(f"Redis mode: Redis unavailable ({e}), falling back to memory queue")
+                        return QueueType.MEMORY
+                else:
+                    self.logger.warning("Redis mode: Redis not configured, falling back to memory queue")
+                    return QueueType.MEMORY
 
         elif self.config.queue_type == QueueType.MEMORY:
             return QueueType.MEMORY
@@ -489,9 +543,21 @@ class QueueManager:
         except Exception as e:
             self.logger.warning(f"Queue health check failed: {e}")
             self._health_status = "unhealthy"
-            # 如果是Redis队列且健康检查失败，尝试切换到内存队列
-            # 对于 AUTO 和 REDIS 模式都允许回退
-            if self._queue_type == QueueType.REDIS and self.config.queue_type in [QueueType.AUTO, QueueType.REDIS]:
+            
+            # Distributed 模式下 Redis 健康检查失败应该报错
+            if self.config.run_mode == 'distributed':
+                error_msg = (
+                    f"Distributed 模式下 Redis 健康检查失败。\n"
+                    f"错误信息: {e}\n"
+                    f"Redis URL: {self.config.redis_url}\n"
+                    f"分布式模式不允许降级到内存队列，请修复 Redis 连接问题。"
+                )
+                self.logger.error(error_msg)
+                raise RuntimeError(error_msg) from e
+            
+            # 非 Distributed 模式：如果是Redis队列且健康检查失败，尝试切换到内存队列
+            # 对于 AUTO 模式允许回退
+            if self._queue_type == QueueType.REDIS and self.config.queue_type == QueueType.AUTO:
                 self.logger.info("Redis queue unavailable, attempting to switch to memory queue...")
                 try:
                     await self._queue.close()