crawlo 1.2.9__py3-none-any.whl → 1.3.1__py3-none-any.whl

crawlo/middleware/offsite.py

@@ -45,7 +45,8 @@ class OffsiteMiddleware:
         # 编译域名正则表达式以提高性能
         o._compile_domains()
         
-        crawler.logger.info(f"OffsiteMiddleware已启用，允许的域名: {allowed_domains}")
+        # 使用中间件自己的logger而不是crawler.logger
+        o.logger.info(f"OffsiteMiddleware已启用，允许的域名: {allowed_domains}")
         return o
 
     def _compile_domains(self):

crawlo/mode_manager.py

@@ -14,102 +14,56 @@ import os
 from enum import Enum
 from typing import Dict, Any, Optional
 
-from crawlo.utils.log import get_logger
-
 
 class RunMode(Enum):
     """运行模式枚举"""
-    STANDALONE = "standalone"    # 单机模式
+    STANDALONE = "standalone"  # 单机模式
     DISTRIBUTED = "distributed"  # 分布式模式
-    AUTO = "auto"               # 自动检测模式
+    AUTO = "auto"  # 自动检测模式
 
 
 class ModeManager:
     """运行模式管理器"""
-    
+
     def __init__(self):
-        self.logger = get_logger(self.__class__.__name__)
-    
+        pass
+
     @staticmethod
     def get_standalone_settings() -> Dict[str, Any]:
         """获取单机模式配置"""
         return {
             'QUEUE_TYPE': 'memory',
             'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
             'CONCURRENCY': 8,
             'MAX_RUNNING_SPIDERS': 1,
             'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
-        }
-    
-    @staticmethod
-    def get_distributed_settings(
-        redis_host: str = '127.0.0.1',
-        redis_port: int = 6379,
-        redis_password: Optional[str] = None,
-        redis_db: int = 0,  # 添加 redis_db 参数
-        project_name: str = 'crawlo'
-    ) -> Dict[str, Any]:
-        """获取分布式模式配置"""
-        # 构建 Redis URL，使用传入的 redis_db 参数
-        if redis_password:
-            redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
-        else:
-            redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
-        
-        return {
-            'PROJECT_NAME': project_name,  # 添加项目名称到配置中
-            'QUEUE_TYPE': 'redis',
-            'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
-            'REDIS_HOST': redis_host,
-            'REDIS_PORT': redis_port,
-            'REDIS_PASSWORD': redis_password,
-            'REDIS_DB': redis_db,  # 添加 Redis 数据库编号到配置中
-            'REDIS_URL': redis_url,
-            'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',  # 使用统一命名规范
-            # Redis key配置已移至各组件中，使用统一的命名规范
-            # crawlo:{project_name}:filter:fingerprint (请求去重)
-            'CONCURRENCY': 16,
-            'MAX_RUNNING_SPIDERS': 1,
-            'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
-        }
-    
-    @staticmethod
-    def get_auto_settings() -> Dict[str, Any]:
-        """获取自动检测模式配置"""
-        return {
-            'QUEUE_TYPE': 'auto',
-            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',  # 默认内存过滤器
-            'CONCURRENCY': 12,
-            'MAX_RUNNING_SPIDERS': 1,
-            'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
         }
-    
+
     def resolve_mode_settings(
-        self, 
-        mode: str = 'standalone',
-        **kwargs
+            self,
+            mode: str = 'standalone',
+            **kwargs
     ) -> Dict[str, Any]:
         """
         解析运行模式并返回对应配置
-        
+
         Args:
             mode: 运行模式 ('standalone', 'distributed', 'auto')
             **kwargs: 额外配置参数
-            
+
         Returns:
             Dict[str, Any]: 配置字典
         """
         mode = RunMode(mode.lower())
-        
+        mode_info = None
+
         if mode == RunMode.STANDALONE:
-            self.logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+            mode_info = "使用单机模式 - 简单快速，适合开发和中小规模爬取"
             settings = self.get_standalone_settings()
-            
+
         elif mode == RunMode.DISTRIBUTED:
-            self.logger.info("使用分布式模式 - 支持多节点扩展，适合大规模爬取")
+            mode_info = "使用分布式模式 - 支持多节点扩展，适合大规模爬取"
             settings = self.get_distributed_settings(
                 redis_host=kwargs.get('redis_host', '127.0.0.1'),
                 redis_port=kwargs.get('redis_port', 6379),
@@ -117,25 +71,28 @@ class ModeManager:
                 redis_db=kwargs.get('redis_db', 0),  # 添加 redis_db 参数
                 project_name=kwargs.get('project_name', 'crawlo')
             )
-            
+
         elif mode == RunMode.AUTO:
-            self.logger.info("使用自动检测模式 - 智能选择最佳运行方式")
+            mode_info = "使用自动检测模式 - 智能选择最佳运行方式"
             settings = self.get_auto_settings()
-            
+
         else:
             raise ValueError(f"不支持的运行模式: {mode}")
-        
+
         # 合并用户自定义配置
-        user_settings = {k: v for k, v in kwargs.items() 
-                        if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
+        user_settings = {k: v for k, v in kwargs.items()
+                         if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
         settings.update(user_settings)
-        
+
+        # 将模式信息添加到配置中，供后续使用
+        settings['_mode_info'] = mode_info
+
         return settings
-    
+
     def from_environment(self) -> Dict[str, Any]:
         """从环境变量构建配置"""
         config = {}
-        
+
         # 扫描 CRAWLO_ 前缀的环境变量
         for key, value in os.environ.items():
             if key.startswith('CRAWLO_'):
@@ -150,7 +107,7 @@ class ModeManager:
                         config[config_key] = float(value)
                     except ValueError:
                         config[config_key] = value
-        
+
         return config
 
 
@@ -161,12 +118,12 @@ def standalone_mode(**kwargs) -> Dict[str, Any]:
 
 
 def distributed_mode(
-    redis_host: str = '127.0.0.1',
-    redis_port: int = 6379,
-    redis_password: Optional[str] = None,
-    redis_db: int = 0,  # 添加 redis_db 参数
-    project_name: str = 'crawlo',
-    **kwargs
+        redis_host: str = '127.0.0.1',
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
+        project_name: str = 'crawlo',
+        **kwargs
 ) -> Dict[str, Any]:
     """快速创建分布式模式配置"""
     return ModeManager().resolve_mode_settings(
@@ -189,24 +146,4 @@ def auto_mode(**kwargs) -> Dict[str, Any]:
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
     # 移除直接使用 os.getenv()，要求通过 settings 配置
-    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
-    
-    # 保留原有代码作为参考
-    # mode = os.getenv('CRAWLO_MODE', default_mode).lower()
-    # 
-    # if mode == 'distributed':
-    #     return distributed_mode(
-    #         redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
-    #         redis_port=int(os.getenv('REDIS_PORT', 6379)),
-    #         redis_password=os.getenv('REDIS_PASSWORD'),
-    #         project_name=os.getenv('PROJECT_NAME', 'crawlo'),
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 16)),
-    #     )
-    # elif mode == 'auto':
-    #     return auto_mode(
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 12)),
-    #     )
-    # else:  # standalone
-    #     return standalone_mode(
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 8)),
-    #     )
+    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")

crawlo/pipelines/mysql_pipeline.py

@@ -7,6 +7,7 @@ from typing import Optional, List, Dict
 from crawlo.exceptions import ItemDiscard
 from crawlo.utils.db_helper import make_insert_sql, make_batch_sql
 from crawlo.utils.log import get_logger
+from . import BasePipeline
 
 
 class AsyncmyMySQLPipeline:
@@ -200,7 +201,7 @@ class AiomysqlMySQLPipeline:
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
     @classmethod
-    def create_instance(cls, crawler):
+    def from_crawler(cls, crawler):
         return cls(crawler)
 
     async def _init_pool(self):
@@ -213,12 +214,12 @@ class AiomysqlMySQLPipeline:
                 try:
                     self.pool = await aiomysql.create_pool(
                         host=self.settings.get('MYSQL_HOST', 'localhost'),
-                        port=self.settings.getint('MYSQL_PORT', 3306),
+                        port=self.settings.get_int('MYSQL_PORT', 3306),
                         user=self.settings.get('MYSQL_USER', 'root'),
                         password=self.settings.get('MYSQL_PASSWORD', ''),
                         db=self.settings.get('MYSQL_DB', 'scrapy_db'),
-                        minsize=self.settings.getint('MYSQL_POOL_MIN', 2),
-                        maxsize=self.settings.getint('MYSQL_POOL_MAX', 5),
+                        minsize=self.settings.get_int('MYSQL_POOL_MIN', 2),
+                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 5),
                         cursorclass=aiomysql.DictCursor,
                         autocommit=False
                     )

crawlo/pipelines/pipeline_manager.py

@@ -4,7 +4,6 @@ from typing import List
 from pprint import pformat
 from asyncio import create_task
 
-
 from crawlo.utils.log import get_logger
 from crawlo.event import item_successful, item_discard
 from crawlo.project import load_class, common_call
@@ -20,6 +19,20 @@ class PipelineManager:
 
         self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
         pipelines = self.crawler.settings.get_list('PIPELINES')
+        dedup_pipeline = self.crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
+
+        # 添加调试信息
+        self.logger.debug(f"PIPELINES from settings: {pipelines}")
+        self.logger.debug(f"DEFAULT_DEDUP_PIPELINE from settings: {dedup_pipeline}")
+
+        # 确保DEFAULT_DEDUP_PIPELINE被添加到管道列表开头
+        if dedup_pipeline:
+            # 移除所有去重管道实例（如果存在）
+            pipelines = [item for item in pipelines if item != dedup_pipeline]
+            # 在开头插入去重管道
+            self.logger.debug(f"{dedup_pipeline} insert successful")
+            pipelines.insert(0, dedup_pipeline)
+
         self._add_pipelines(pipelines)
         self._add_methods()
 
@@ -34,7 +47,7 @@ class PipelineManager:
                 pipeline_cls = load_class(pipeline)
                 if not hasattr(pipeline_cls, 'from_crawler'):
                     raise PipelineInitError(
-                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `from_crawler` method"
                     )
                 self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
             except Exception as e:

crawlo/project.py

@@ -7,18 +7,10 @@ from inspect import iscoroutinefunction
 from typing import Callable, Optional, Any
 
 from crawlo.settings.setting_manager import SettingManager
-from crawlo.utils.log import get_logger
+from crawlo.utils.log import get_logger, LoggerManager
 
-# 延迟初始化logger，在需要时通过get_logger获取
-logger = None
-
-
-def _get_logger():
-    """延迟获取logger实例，确保在配置加载后创建"""
-    global logger
-    if logger is None:
-        logger = get_logger(__name__)
-    return logger
+# 使用全局logger，避免每个模块都创建自己的延迟初始化函数
+logger = get_logger(__name__)
 
 
 def load_class(path: str) -> Any:
@@ -50,7 +42,7 @@ def merge_settings(spider, settings):
     spider_name = getattr(spider, 'name', 'UnknownSpider')
     # 检查 settings 是否为 SettingManager 实例
     if not hasattr(settings, 'update_attributes'):
-        _get_logger().error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
+        logger.error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
         # 如果是字典，创建一个新的 SettingManager 实例
         if isinstance(settings, dict):
             from crawlo.settings.setting_manager import SettingManager
@@ -58,14 +50,14 @@ def merge_settings(spider, settings):
             new_settings.update_attributes(settings)
             settings = new_settings
         else:
-            _get_logger().error("无法处理的 settings 类型")
+            logger.error("无法处理的 settings 类型")
             return
             
     if hasattr(spider, 'custom_settings'):
         custom_settings = getattr(spider, 'custom_settings')
         settings.update_attributes(custom_settings)
     else:
-        _get_logger().debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")
 
 
 async def common_call(func: Callable, *args, **kwargs):
@@ -93,7 +85,7 @@ def _get_settings_module_from_cfg(cfg_path: str) -> str:
         config.read(cfg_path, encoding="utf-8")
         if config.has_section("settings") and config.has_option("settings", "default"):
             module_path = config.get("settings", "default")
-            _get_logger().debug(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
+            logger.debug(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
             return module_path
         else:
             raise RuntimeError(f"配置文件缺少 [settings] 或 default 选项: {cfg_path}")
@@ -114,7 +106,7 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
     for root, dirs, files in os.walk(path):
         if "crawlo.cfg" in files:
             cfg_path = os.path.join(root, "crawlo.cfg")
-            _get_logger().debug(f"✅ 找到项目配置文件: {cfg_path}")
+            logger.debug(f"✅ 找到项目配置文件: {cfg_path}")
             return root
     
     # 向上查找直到找到 crawlo.cfg 或包含 settings.py 和 __init__.py 的目录
@@ -130,20 +122,20 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
         # 检查 crawlo.cfg
         cfg_file = os.path.join(path, "crawlo.cfg")
         if os.path.isfile(cfg_file):
-            _get_logger().debug(f"✅ 找到项目配置文件: {cfg_file}")
+            logger.debug(f"✅ 找到项目配置文件: {cfg_file}")
             return path
 
         # 检查 settings.py 和 __init__.py
         settings_file = os.path.join(path, "settings.py")
         init_file = os.path.join(path, "__init__.py")
         if os.path.isfile(settings_file) and os.path.isfile(init_file):
-            _get_logger().debug(f"✅ 找到项目模块: {path}")
+            logger.debug(f"✅ 找到项目模块: {path}")
             # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
             parent = os.path.dirname(path)
             if parent != path:
                 parent_cfg = os.path.join(parent, "crawlo.cfg")
                 if os.path.isfile(parent_cfg):
-                    _get_logger().debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
+                    logger.debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
                     return parent
             return path
 
@@ -167,19 +159,19 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
                 
                 cfg_file = os.path.join(path, "crawlo.cfg")
                 if os.path.isfile(cfg_file):
-                    _get_logger().debug(f"✅ 找到项目配置文件: {cfg_file}")
+                    logger.debug(f"✅ 找到项目配置文件: {cfg_file}")
                     return path
 
                 settings_file = os.path.join(path, "settings.py")
                 init_file = os.path.join(path, "__init__.py")
                 if os.path.isfile(settings_file) and os.path.isfile(init_file):
-                    _get_logger().debug(f"✅ 找到项目模块: {path}")
+                    logger.debug(f"✅ 找到项目模块: {path}")
                     # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
                     parent = os.path.dirname(path)
                     if parent != path:
                         parent_cfg = os.path.join(parent, "crawlo.cfg")
                         if os.path.isfile(parent_cfg):
-                            _get_logger().debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
+                            logger.debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
                             return parent
                     return path
 
@@ -204,19 +196,19 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
                 
                 cfg_file = os.path.join(path, "crawlo.cfg")
                 if os.path.isfile(cfg_file):
-                    _get_logger().debug(f"找到项目配置文件: {cfg_file}")
+                    logger.debug(f"找到项目配置文件: {cfg_file}")
                     return path
 
                 settings_file = os.path.join(path, "settings.py")
                 init_file = os.path.join(path, "__init__.py")
                 if os.path.isfile(settings_file) and os.path.isfile(init_file):
-                    _get_logger().debug(f"找到项目模块: {path}")
+                    logger.debug(f"找到项目模块: {path}")
                     # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
                     parent = os.path.dirname(path)
                     if parent != path:
                         parent_cfg = os.path.join(parent, "crawlo.cfg")
                         if os.path.isfile(parent_cfg):
-                            _get_logger().debug(f"在上层目录找到项目配置文件: {parent_cfg}")
+                            logger.debug(f"在上层目录找到项目配置文件: {parent_cfg}")
                             return parent
                     return path
 
@@ -227,7 +219,7 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
     except Exception:
         pass
 
-    _get_logger().warning("未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
+    logger.warning("未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
     return None
 
 
@@ -241,8 +233,7 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
     Returns:
         SettingManager: 已加载配置的实例
     """
-    # Change INFO level log to DEBUG level to avoid redundant output
-    _get_logger().debug("🚀 正在初始化 Crawlo 项目配置...")
+    logger.debug("🚀 正在初始化 Crawlo 项目配置...")
 
     # 1. 查找项目根
     project_root = _find_project_root()
@@ -259,32 +250,48 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
         # 推断：项目目录名.settings
         project_name = os.path.basename(project_root)
         settings_module_path = f"{project_name}.settings"
-        _get_logger().warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
+        logger.warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
 
     # 3. 注入 sys.path
     project_root_str = os.path.abspath(project_root)
     if project_root_str not in sys.path:
         sys.path.insert(0, project_root_str)
-        _get_logger().debug(f"📁 项目根目录已加入 sys.path: {project_root_str}")
+        logger.debug(f"📁 项目根目录已加入 sys.path: {project_root_str}")
 
     # 4. 加载 SettingManager
-    _get_logger().debug(f"⚙️ 正在加载配置模块: {settings_module_path}")
+    logger.debug(f"⚙️ 正在加载配置模块: {settings_module_path}")
     settings = SettingManager()
 
     try:
         settings.set_settings(settings_module_path)
-        _get_logger().debug("✅ settings 模块加载成功")
+        logger.debug("✅ settings 模块加载成功")
     except Exception as e:
         raise ImportError(f"加载 settings 模块失败 '{settings_module_path}': {e}")
 
-    # 5. 合并运行时配置
+    # 5. 根据 RUN_MODE 获取相应配置
+    run_mode = settings.get('RUN_MODE', 'standalone')
+    if run_mode:
+        from crawlo.mode_manager import ModeManager
+        mode_manager = ModeManager()
+        mode_settings = mode_manager.resolve_mode_settings(run_mode)
+        # 合并模式配置，但不覆盖用户已设置的配置
+        for key, value in mode_settings.items():
+            # 只有当用户没有设置该配置项时才应用模式配置
+            if key not in settings.attributes:
+                settings.set(key, value)
+        logger.debug(f"🔧 已应用 {run_mode} 模式配置")
+
+    # 6. 合并运行时配置
     if custom_settings:
         settings.update_attributes(custom_settings)
-        _get_logger().debug(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
+        logger.debug(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
 
-    # 6. 显示核心配置摘要（INFO级别）
+    # 7. 显示核心配置摘要（INFO级别）
     # _log_settings_summary(settings)
 
+    # 配置日志系统
+    LoggerManager.configure(settings)
+    
     # 将项目初始化完成的消息改为DEBUG级别
-    _get_logger().debug("🎉 Crawlo 项目配置初始化完成！")
-    return settings
+    logger.debug("🎉 Crawlo 项目配置初始化完成！")
+    return settings

crawlo/settings/default_settings.py

@@ -48,7 +48,18 @@ QUEUE_TYPE = 'auto'
 # 默认使用内存过滤器和去重管道，确保在无Redis环境下也能正常运行
 # 在auto模式下，如果Redis可用，框架会自动更新为Redis实现以提供更好的去重能力
 DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline' 
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'  
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'root'
+MYSQL_PASSWORD = '123456'
+MYSQL_DB = 'crawl_pro'
+MYSQL_TABLE = 'crawlo'
+MYSQL_BATCH_SIZE = 100
+MYSQL_USE_BATCH = False  # 是否启用批量插入
+
 
 # --- Redis 过滤器配置 ---
 # 使用环境变量配置工具获取 Redis 配置
@@ -85,7 +96,6 @@ MIDDLEWARES = [
     'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',  # 1. 忽略无效请求
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',  # 2. 控制请求频率
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',  # 3. 添加默认请求头
-    # 'crawlo.middleware.proxy.ProxyMiddleware',  # 4. 设置代理（默认不启用）
     'crawlo.middleware.offsite.OffsiteMiddleware',  # 5. 站外请求过滤
 
     # === 响应处理阶段 ===
@@ -98,8 +108,7 @@ MIDDLEWARES = [
 
 # 框架数据处理管道列表（框架默认管道 + 用户自定义管道）
 PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',  # 控制台输出
-    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',     # MySQL 存储（可选）
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
 ]
 
 # 明确添加默认去重管道到管道列表开头

crawlo/settings/setting_manager.py

@@ -15,38 +15,67 @@ class SettingManager(MutableMapping):
         self.set_settings(default_settings)
         # 在初始化时合并配置
         self._merge_config(values)
-        
+
     def _merge_config(self, user_config):
         """合并默认配置和用户配置"""
         if not user_config:
             return
-            
+
         # 合并中间件配置
         if 'MIDDLEWARES' in user_config:
             default_middlewares = self.attributes.get('MIDDLEWARES', [])
             user_middlewares = user_config['MIDDLEWARES']
-            self.attributes['MIDDLEWARES'] = default_middlewares + user_middlewares
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_middlewares:
+                # 过滤掉空值和注释
+                user_middlewares = [middleware for middleware in user_middlewares if middleware and not middleware.strip().startswith('#')]
+                # 合并默认中间件和用户中间件，去重但保持顺序
+                merged_middlewares = default_middlewares[:]
+                for middleware in user_middlewares:
+                    if middleware not in merged_middlewares:
+                        merged_middlewares.append(middleware)
+                self.attributes['MIDDLEWARES'] = merged_middlewares
+
         # 合并管道配置
         if 'PIPELINES' in user_config:
             default_pipelines = self.attributes.get('PIPELINES', [])
             user_pipelines = user_config['PIPELINES']
-            merged_pipelines = default_pipelines + user_pipelines
-            # 特殊处理PIPELINES，确保去重管道在最前面
-            dedup_pipeline = self.attributes.get('DEFAULT_DEDUP_PIPELINE')
-            if dedup_pipeline:
-                # 移除所有去重管道实例（如果存在）
-                merged_pipelines = [item for item in merged_pipelines if item != dedup_pipeline]
-                # 在开头插入去重管道
-                merged_pipelines.insert(0, dedup_pipeline)
-            self.attributes['PIPELINES'] = merged_pipelines
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_pipelines:
+                # 过滤掉空值和注释
+                user_pipelines = [pipeline for pipeline in user_pipelines if pipeline and not pipeline.strip().startswith('#')]
+                # 合并默认管道和用户管道，去重但保持顺序
+                merged_pipelines = default_pipelines[:]
+                for pipeline in user_pipelines:
+                    if pipeline not in merged_pipelines:
+                        merged_pipelines.append(pipeline)
+                self.attributes['PIPELINES'] = merged_pipelines
+
+        # 特殊处理PIPELINES，确保去重管道在最前面
+        dedup_pipeline = self.attributes.get('DEFAULT_DEDUP_PIPELINE')
+        if dedup_pipeline:
+            pipelines = self.attributes.get('PIPELINES', [])
+            # 移除所有去重管道实例（如果存在）
+            pipelines = [item for item in pipelines if item != dedup_pipeline]
+            # 在开头插入去重管道
+            pipelines.insert(0, dedup_pipeline)
+            self.attributes['PIPELINES'] = pipelines
+
         # 合并扩展配置
         if 'EXTENSIONS' in user_config:
             default_extensions = self.attributes.get('EXTENSIONS', [])
             user_extensions = user_config['EXTENSIONS']
-            self.attributes['EXTENSIONS'] = default_extensions + user_extensions
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_extensions:
+                # 过滤掉空值和注释
+                user_extensions = [extension for extension in user_extensions if extension and not extension.strip().startswith('#')]
+                # 合并默认扩展和用户扩展，去重但保持顺序
+                merged_extensions = default_extensions[:]
+                for extension in user_extensions:
+                    if extension not in merged_extensions:
+                        merged_extensions.append(extension)
+                self.attributes['EXTENSIONS'] = merged_extensions
+
         # 更新其他用户配置
         for key, value in user_config.items():
             if key not in ['MIDDLEWARES', 'PIPELINES', 'EXTENSIONS']:
@@ -107,9 +136,15 @@ class SettingManager(MutableMapping):
     def set_settings(self, module):
         if isinstance(module, str):
             module = import_module(module)
+        
+        # 收集模块中的所有配置项
+        module_settings = {}
         for key in dir(module):
             if key.isupper():
-                self.set(key, getattr(module, key))
+                module_settings[key] = getattr(module, key)
+        
+        # 使用合并逻辑而不是直接设置
+        self._merge_config(module_settings)
 
     # 实现 MutableMapping 必须的方法
     def __getitem__(self, item):
@@ -147,7 +182,7 @@ class SettingManager(MutableMapping):
         # 创建一个新的实例
         cls = self.__class__
         new_instance = cls.__new__(cls)
-        
+
         # 复制attributes字典，但排除不可pickle的对象
         new_attributes = {}
         for key, value in self.attributes.items():
@@ -157,8 +192,8 @@ class SettingManager(MutableMapping):
             except Exception:
                 # 如果复制失败，保留原始引用（对于logger等对象）
                 new_attributes[key] = value
-        
+
         # 设置新实例的attributes
         new_instance.attributes = new_attributes
-        
+
         return new_instance