crawlo 1.3.0__tar.gz → 1.3.2__tar.gz

{crawlo-1.3.0/crawlo.egg-info → crawlo-1.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.3.0
+Version: 1.3.2
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder
@@ -56,7 +56,7 @@ Requires-Dist: selenium>=3.141.0; extra == "all"
   
   <p align="center">
     <a href="https://www.python.org/downloads/">
-      <img src="https://img.shields.io/badge/python-3.8%2B-blue" alt="Python Version">
+      <img src="https://img.shields.io/badge/python-%3C%3D3.12-blue" alt="Python Version">
     </a>
     <a href="LICENSE">
       <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
@@ -429,13 +429,22 @@ Crawlo框架的中间件、管道和扩展组件采用模块化设计，框架
 
 用户可以通过`CUSTOM_MIDDLEWARES`配置自定义中间件：
 
-```python
+``python
 # settings.py
 CUSTOM_MIDDLEWARES = [
     'myproject.middlewares.CustomMiddleware',
 ]
 ```
 
+> **注意**：DefaultHeaderMiddleware 和 OffsiteMiddleware 需要相应的配置才能启用：
+> - DefaultHeaderMiddleware 需要配置 `DEFAULT_REQUEST_HEADERS` 或 `USER_AGENT` 参数
+> - OffsiteMiddleware 需要配置 `ALLOWED_DOMAINS` 参数
+> 
+> 如果未配置相应参数，这些中间件会因为 NotConfiguredError 而被禁用。
+
+> **注意**：中间件的顺序很重要。SimpleProxyMiddleware 通常放在列表末尾，
+> 这样可以在所有默认中间件处理后再应用代理设置。
+
 #### 管道配置
 
 框架默认加载以下管道：
@@ -930,7 +939,7 @@ Crawlo框架内置了多种中间件，其中代理中间件有两种实现：
 
 如果需要使用简化版代理中间件，可以在配置文件中替换默认的代理中间件：
 
-```python
+``python
 # settings.py
 MIDDLEWARES = [
     # 注释掉复杂版代理中间件

{crawlo-1.3.0 → crawlo-1.3.2}/README.md

@@ -6,7 +6,7 @@
   
   <p align="center">
     <a href="https://www.python.org/downloads/">
-      <img src="https://img.shields.io/badge/python-3.8%2B-blue" alt="Python Version">
+      <img src="https://img.shields.io/badge/python-%3C%3D3.12-blue" alt="Python Version">
     </a>
     <a href="LICENSE">
       <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
@@ -379,13 +379,22 @@ Crawlo框架的中间件、管道和扩展组件采用模块化设计，框架
 
 用户可以通过`CUSTOM_MIDDLEWARES`配置自定义中间件：
 
-```python
+``python
 # settings.py
 CUSTOM_MIDDLEWARES = [
     'myproject.middlewares.CustomMiddleware',
 ]
 ```
 
+> **注意**：DefaultHeaderMiddleware 和 OffsiteMiddleware 需要相应的配置才能启用：
+> - DefaultHeaderMiddleware 需要配置 `DEFAULT_REQUEST_HEADERS` 或 `USER_AGENT` 参数
+> - OffsiteMiddleware 需要配置 `ALLOWED_DOMAINS` 参数
+> 
+> 如果未配置相应参数，这些中间件会因为 NotConfiguredError 而被禁用。
+
+> **注意**：中间件的顺序很重要。SimpleProxyMiddleware 通常放在列表末尾，
+> 这样可以在所有默认中间件处理后再应用代理设置。
+
 #### 管道配置
 
 框架默认加载以下管道：
@@ -880,7 +889,7 @@ Crawlo框架内置了多种中间件，其中代理中间件有两种实现：
 
 如果需要使用简化版代理中间件，可以在配置文件中替换默认的代理中间件：
 
-```python
+``python
 # settings.py
 MIDDLEWARES = [
     # 注释掉复杂版代理中间件

crawlo-1.3.2/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.3.2'

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/commands/utils.py

@@ -133,8 +133,11 @@ def validate_spider_name(spider_name: str) -> bool:
         bool: 是否有效
     """
     import re
+    # 清理爬虫名称中的不可见字符
+    cleaned_name = ''.join(c for c in spider_name if not unicodedata.category(c).startswith('C'))
+    
     # 爬虫名称应该是有效的Python标识符
-    return spider_name.isidentifier() and re.match(r'^[a-z][a-z0-9_]*$', spider_name)
+    return cleaned_name.isidentifier() and re.match(r'^[a-z][a-z0-9_]*$', cleaned_name)
 
 
 def format_file_size(size_bytes: int) -> str:
@@ -181,7 +184,14 @@ def is_valid_domain(domain: str) -> bool:
         bool: 是否有效
     """
     import re
+    # 清理域名中的不可见字符
+    cleaned_domain = ''.join(c for c in domain if not unicodedata.category(c).startswith('C'))
+    
     pattern = re.compile(
         r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)*[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$'
     )
-    return bool(pattern.match(domain))
+    return bool(pattern.match(cleaned_domain))
+
+
+# 添加导入
+import unicodedata

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/crawler.py

@@ -36,13 +36,12 @@ from .spider import Spider, get_global_spider_registry
 from .core.engine import Engine
 from .subscriber import Subscriber
 from .extension import ExtensionManager
+from crawlo.utils.log import get_logger
 from .stats_collector import StatsCollector
 from .event import spider_opened, spider_closed
 from .settings.setting_manager import SettingManager
 from crawlo.project import merge_settings, get_settings
 
-# 使用自定义日志系统
-from crawlo.utils.log import get_logger
 logger = get_logger(__name__)
 
 
@@ -112,7 +111,12 @@ class Crawler:
     - Exception handling and cleanup
     """
 
-    def __init__(self, spider_cls: Type[Spider], settings: SettingManager, context: Optional[CrawlerContext] = None):
+    def __init__(
+            self,
+            spider_cls: Type[Spider],
+            settings: SettingManager,
+            context: Optional[CrawlerContext] = None
+    ):
         self.spider_cls = spider_cls
         self.spider: Optional[Spider] = None
         self.engine: Optional[Engine] = None
@@ -137,6 +141,22 @@ class Crawler:
             'error_count': 0
         }
 
+        # Initialize components
+        self.subscriber = self._create_subscriber()
+        self.spider = self._create_spider()
+        self.engine = self._create_engine()
+        self.stats = self._create_stats()
+        # Note: Do not initialize extension manager here, let it initialize in the engine
+
+        # Validate crawler state
+        self._validate_crawler_state()
+
+        # 打印启动信息，确保在日志系统配置之后打印
+        self._log_startup_info()
+        
+        # 将启动爬虫名称的日志移到这里，确保在日志系统配置之后打印
+        logger.info(f"Starting running {self.spider.name}")
+
     async def crawl(self):
         """
         Start the crawler core process
@@ -233,6 +253,52 @@ class Crawler:
             return self._end_time - self._start_time
         return 0.0
 
+    def _log_startup_info(self):
+        """Print startup information, including run mode and key configuration checks"""
+        # Get run mode
+        run_mode = self.settings.get('RUN_MODE', 'standalone')
+
+        # Get version number
+        version = self.settings.get('VERSION', '1.0.0')
+        if not version or version == 'None':
+            version = '1.0.0'
+
+        # Print framework start info
+        logger.info(f"Crawlo Framework Started {version}")
+
+        # Add mode info if available
+        mode_info = self.settings.get('_mode_info')
+        if mode_info:
+            logger.info(mode_info)
+        else:
+            # 如果没有_mode_info，添加默认信息
+            logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+
+        # Get actual queue type
+        queue_type = self.settings.get('QUEUE_TYPE', 'memory')
+
+        # Display information based on run mode and queue type combination
+        if run_mode == 'distributed':
+            logger.info("Run Mode: distributed")
+            logger.info("Distributed Mode - Multi-node collaboration supported")
+            # Show Redis configuration
+            redis_host = self.settings.get('REDIS_HOST', 'localhost')
+            redis_port = self.settings.get('REDIS_PORT', 6379)
+            logger.info(f"Redis Address: {redis_host}:{redis_port}")
+        elif run_mode == 'standalone':
+            if queue_type == 'redis':
+                logger.info("Run Mode: standalone+redis")
+                # Show Redis configuration
+                redis_host = self.settings.get('REDIS_HOST', 'localhost')
+                redis_port = self.settings.get('REDIS_PORT', 6379)
+                logger.info(f"Redis Address: {redis_host}:{redis_port}")
+            elif queue_type == 'auto':
+                logger.info("Run Mode: standalone+auto")
+            else:  # memory
+                logger.info("Run Mode: standalone")
+        else:
+            logger.info(f"Run Mode: {run_mode}")
+
     async def _ensure_cleanup(self):
         """Ensure resource cleanup"""
         try:
@@ -483,7 +549,10 @@ class CrawlerProcess:
         signal.signal(signal.SIGINT, self._shutdown)
         signal.signal(signal.SIGTERM, self._shutdown)
 
-        self._log_startup_info()
+        # 注意：移除在这里调用_log_startup_info()，因为这时候日志系统可能还没有被正确配置
+        # 日志系统的配置是在project.py的get_settings函数中进行的，而CrawlerProcess的实例化
+        # 是在get_settings函数返回之前进行的，所以这时候调用_log_startup_info()可能会导致
+        # 日志信息没有被正确写入到日志文件中
 
         logger.debug(
             f"CrawlerProcess initialized successfully\n"
@@ -983,39 +1052,41 @@ class CrawlerProcess:
         if not version or version == 'None':
             version = '1.0.0'
 
-        # Build startup info log
-        startup_info = [
-            f"Crawlo Framework Started {version}"
-        ]
+        # Print framework start info
+        logger.info(f"Crawlo Framework Started {version}")
+
+        # Add mode info if available
+        mode_info = self.settings.get('_mode_info')
+        if mode_info:
+            logger.info(mode_info)
+        else:
+            # 如果没有_mode_info，添加默认信息
+            logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
 
         # Get actual queue type
         queue_type = self.settings.get('QUEUE_TYPE', 'memory')
 
         # Display information based on run mode and queue type combination
         if run_mode == 'distributed':
-            startup_info.append("Run Mode: distributed")
-            startup_info.append("Distributed Mode - Multi-node collaboration supported")
+            logger.info("Run Mode: distributed")
+            logger.info("Distributed Mode - Multi-node collaboration supported")
             # Show Redis configuration
             redis_host = self.settings.get('REDIS_HOST', 'localhost')
             redis_port = self.settings.get('REDIS_PORT', 6379)
-            startup_info.append(f"Redis Address: {redis_host}:{redis_port}")
+            logger.info(f"Redis Address: {redis_host}:{redis_port}")
         elif run_mode == 'standalone':
             if queue_type == 'redis':
-                startup_info.append("Run Mode: standalone+redis")
+                logger.info("Run Mode: standalone+redis")
                 # Show Redis configuration
                 redis_host = self.settings.get('REDIS_HOST', 'localhost')
                 redis_port = self.settings.get('REDIS_PORT', 6379)
-                startup_info.append(f"Redis Address: {redis_host}:{redis_port}")
+                logger.info(f"Redis Address: {redis_host}:{redis_port}")
             elif queue_type == 'auto':
-                startup_info.append("Run Mode: standalone+auto")
+                logger.info("Run Mode: standalone+auto")
             else:  # memory
-                startup_info.append("Run Mode: standalone")
+                logger.info("Run Mode: standalone")
         else:
-            startup_info.append(f"Run Mode: {run_mode}")
-
-        # Print startup information at INFO level
-        for info in startup_info:
-            logger.info(info)
+            logger.info(f"Run Mode: {run_mode}")
 
 
 # === Utility functions ===

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/middleware/offsite.py

@@ -28,8 +28,16 @@ class OffsiteMiddleware:
         创建中间件实例
         从爬虫设置中获取允许的域名列表
         """
-        # 从爬虫设置中获取允许的域名
-        allowed_domains = crawler.settings.get_list('ALLOWED_DOMAINS')
+        # 优先使用 Spider 实例的 allowed_domains，回退到全局设置中的 ALLOWED_DOMAINS
+        allowed_domains = []
+        
+        # 检查当前爬虫实例是否有 allowed_domains 属性
+        if hasattr(crawler, 'spider') and crawler.spider and hasattr(crawler.spider, 'allowed_domains'):
+            allowed_domains = getattr(crawler.spider, 'allowed_domains', [])
+        
+        # 如果 Spider 实例没有设置 allowed_domains，则从全局设置中获取
+        if not allowed_domains:
+            allowed_domains = crawler.settings.get_list('ALLOWED_DOMAINS')
         
         # 如果没有配置允许的域名，则禁用此中间件
         if not allowed_domains:
@@ -45,7 +53,8 @@ class OffsiteMiddleware:
         # 编译域名正则表达式以提高性能
         o._compile_domains()
         
-        crawler.logger.info(f"OffsiteMiddleware已启用，允许的域名: {allowed_domains}")
+        # 使用中间件自己的logger而不是crawler.logger
+        o.logger.info(f"OffsiteMiddleware已启用，允许的域名: {allowed_domains}")
         return o
 
     def _compile_domains(self):

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/mode_manager.py

@@ -14,8 +14,6 @@ import os
 from enum import Enum
 from typing import Dict, Any, Optional
 
-from crawlo.utils.log import get_logger
-
 
 class RunMode(Enum):
     """运行模式枚举"""
@@ -28,7 +26,7 @@ class ModeManager:
     """运行模式管理器"""
 
     def __init__(self):
-        self.logger = get_logger(self.__class__.__name__)
+        pass
 
     @staticmethod
     def get_standalone_settings() -> Dict[str, Any]:
@@ -40,7 +38,6 @@ class ModeManager:
             'CONCURRENCY': 8,
             'MAX_RUNNING_SPIDERS': 1,
             'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
         }
 
     @staticmethod
@@ -48,45 +45,39 @@ class ModeManager:
             redis_host: str = '127.0.0.1',
             redis_port: int = 6379,
             redis_password: Optional[str] = None,
-            redis_db: int = 0,  # 添加 redis_db 参数
+            redis_db: int = 0,
             project_name: str = 'crawlo'
     ) -> Dict[str, Any]:
         """获取分布式模式配置"""
-        # 构建 Redis URL，使用传入的 redis_db 参数
+        # 构建 Redis URL
         if redis_password:
             redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
             redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
-
+            
         return {
-            'PROJECT_NAME': project_name,  # 添加项目名称到配置中
             'QUEUE_TYPE': 'redis',
             'FILTER_CLASS': 'crawlo.filters.aioredis_filter.AioRedisFilter',
+            'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline',
             'REDIS_HOST': redis_host,
             'REDIS_PORT': redis_port,
             'REDIS_PASSWORD': redis_password,
-            'REDIS_DB': redis_db,  # 添加 Redis 数据库编号到配置中
+            'REDIS_DB': redis_db,
             'REDIS_URL': redis_url,
-            'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',  # 使用统一命名规范
-            # Redis key配置已移至各组件中，使用统一的命名规范
-            # crawlo:{project_name}:filter:fingerprint (请求去重)
+            'PROJECT_NAME': project_name,
+            'SCHEDULER_QUEUE_NAME': f'crawlo:{project_name}:queue:requests',
             'CONCURRENCY': 16,
-            'MAX_RUNNING_SPIDERS': 1,
+            'MAX_RUNNING_SPIDERS': 10,
             'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
         }
 
     @staticmethod
     def get_auto_settings() -> Dict[str, Any]:
         """获取自动检测模式配置"""
-        return {
-            'QUEUE_TYPE': 'auto',
-            'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',  # 默认内存过滤器
-            'CONCURRENCY': 12,
-            'MAX_RUNNING_SPIDERS': 1,
-            'DOWNLOAD_DELAY': 1.0,
-            'LOG_LEVEL': 'INFO',
-        }
+        # 默认使用内存队列和过滤器
+        settings = ModeManager.get_standalone_settings()
+        settings['QUEUE_TYPE'] = 'auto'
+        return settings
 
     def resolve_mode_settings(
             self,
@@ -104,13 +95,14 @@ class ModeManager:
             Dict[str, Any]: 配置字典
         """
         mode = RunMode(mode.lower())
+        mode_info = None
 
         if mode == RunMode.STANDALONE:
-            self.logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
+            mode_info = "使用单机模式 - 简单快速，适合开发和中小规模爬取"
             settings = self.get_standalone_settings()
 
         elif mode == RunMode.DISTRIBUTED:
-            self.logger.info("使用分布式模式 - 支持多节点扩展，适合大规模爬取")
+            mode_info = "使用分布式模式 - 支持多节点扩展，适合大规模爬取"
             settings = self.get_distributed_settings(
                 redis_host=kwargs.get('redis_host', '127.0.0.1'),
                 redis_port=kwargs.get('redis_port', 6379),
@@ -120,7 +112,7 @@ class ModeManager:
             )
 
         elif mode == RunMode.AUTO:
-            self.logger.info("使用自动检测模式 - 智能选择最佳运行方式")
+            mode_info = "使用自动检测模式 - 智能选择最佳运行方式"
             settings = self.get_auto_settings()
 
         else:
@@ -131,6 +123,9 @@ class ModeManager:
                          if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
         settings.update(user_settings)
 
+        # 将模式信息添加到配置中，供后续使用
+        settings['_mode_info'] = mode_info
+
         return settings
 
     def from_environment(self) -> Dict[str, Any]:
@@ -190,24 +185,4 @@ def auto_mode(**kwargs) -> Dict[str, Any]:
 def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
     # 移除直接使用 os.getenv()，要求通过 settings 配置
-    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
-
-    # 保留原有代码作为参考
-    # mode = os.getenv('CRAWLO_MODE', default_mode).lower()
-    # 
-    # if mode == 'distributed':
-    #     return distributed_mode(
-    #         redis_host=os.getenv('REDIS_HOST', '127.0.0.1'),
-    #         redis_port=int(os.getenv('REDIS_PORT', 6379)),
-    #         redis_password=os.getenv('REDIS_PASSWORD'),
-    #         project_name=os.getenv('PROJECT_NAME', 'crawlo'),
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 16)),
-    #     )
-    # elif mode == 'auto':
-    #     return auto_mode(
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 12)),
-    #     )
-    # else:  # standalone
-    #     return standalone_mode(
-    #         CONCURRENCY=int(os.getenv('CONCURRENCY', 8)),
-    #     )
+    raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/pipelines/mysql_pipeline.py

@@ -7,6 +7,7 @@ from typing import Optional, List, Dict
 from crawlo.exceptions import ItemDiscard
 from crawlo.utils.db_helper import make_insert_sql, make_batch_sql
 from crawlo.utils.log import get_logger
+from . import BasePipeline
 
 
 class AsyncmyMySQLPipeline:
@@ -200,7 +201,7 @@ class AiomysqlMySQLPipeline:
         crawler.subscriber.subscribe(self.spider_closed, event='spider_closed')
 
     @classmethod
-    def create_instance(cls, crawler):
+    def from_crawler(cls, crawler):
         return cls(crawler)
 
     async def _init_pool(self):
@@ -213,12 +214,12 @@ class AiomysqlMySQLPipeline:
                 try:
                     self.pool = await aiomysql.create_pool(
                         host=self.settings.get('MYSQL_HOST', 'localhost'),
-                        port=self.settings.getint('MYSQL_PORT', 3306),
+                        port=self.settings.get_int('MYSQL_PORT', 3306),
                         user=self.settings.get('MYSQL_USER', 'root'),
                         password=self.settings.get('MYSQL_PASSWORD', ''),
                         db=self.settings.get('MYSQL_DB', 'scrapy_db'),
-                        minsize=self.settings.getint('MYSQL_POOL_MIN', 2),
-                        maxsize=self.settings.getint('MYSQL_POOL_MAX', 5),
+                        minsize=self.settings.get_int('MYSQL_POOL_MIN', 2),
+                        maxsize=self.settings.get_int('MYSQL_POOL_MAX', 5),
                         cursorclass=aiomysql.DictCursor,
                         autocommit=False
                     )

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/pipelines/pipeline_manager.py

@@ -30,6 +30,7 @@ class PipelineManager:
             # 移除所有去重管道实例（如果存在）
             pipelines = [item for item in pipelines if item != dedup_pipeline]
             # 在开头插入去重管道
+            self.logger.debug(f"{dedup_pipeline} insert successful")
             pipelines.insert(0, dedup_pipeline)
 
         self._add_pipelines(pipelines)
@@ -46,7 +47,7 @@ class PipelineManager:
                 pipeline_cls = load_class(pipeline)
                 if not hasattr(pipeline_cls, 'from_crawler'):
                     raise PipelineInitError(
-                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                        f"Pipeline init failed, must inherit from `BasePipeline` or have a `from_crawler` method"
                     )
                 self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
             except Exception as e:

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/project.py

@@ -268,12 +268,25 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
     except Exception as e:
         raise ImportError(f"加载 settings 模块失败 '{settings_module_path}': {e}")
 
-    # 5. 合并运行时配置
+    # 5. 根据 RUN_MODE 获取相应配置
+    run_mode = settings.get('RUN_MODE', 'standalone')
+    if run_mode:
+        from crawlo.mode_manager import ModeManager
+        mode_manager = ModeManager()
+        mode_settings = mode_manager.resolve_mode_settings(run_mode)
+        # 合并模式配置，但不覆盖用户已设置的配置
+        for key, value in mode_settings.items():
+            # 只有当用户没有设置该配置项时才应用模式配置
+            if key not in settings.attributes:
+                settings.set(key, value)
+        logger.debug(f"🔧 已应用 {run_mode} 模式配置")
+
+    # 6. 合并运行时配置
     if custom_settings:
         settings.update_attributes(custom_settings)
         logger.debug(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
 
-    # 6. 显示核心配置摘要（INFO级别）
+    # 7. 显示核心配置摘要（INFO级别）
     # _log_settings_summary(settings)
 
     # 配置日志系统
@@ -281,4 +294,4 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
     
     # 将项目初始化完成的消息改为DEBUG级别
     logger.debug("🎉 Crawlo 项目配置初始化完成！")
-    return settings
+    return settings

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/settings/default_settings.py

@@ -48,7 +48,18 @@ QUEUE_TYPE = 'auto'
 # 默认使用内存过滤器和去重管道，确保在无Redis环境下也能正常运行
 # 在auto模式下，如果Redis可用，框架会自动更新为Redis实现以提供更好的去重能力
 DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline' 
-FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'  
+FILTER_CLASS = 'crawlo.filters.memory_filter.MemoryFilter'
+
+
+MYSQL_HOST = '127.0.0.1'
+MYSQL_PORT = 3306
+MYSQL_USER = 'root'
+MYSQL_PASSWORD = '123456'
+MYSQL_DB = 'crawl_pro'
+MYSQL_TABLE = 'crawlo'
+MYSQL_BATCH_SIZE = 100
+MYSQL_USE_BATCH = False  # 是否启用批量插入
+
 
 # --- Redis 过滤器配置 ---
 # 使用环境变量配置工具获取 Redis 配置
@@ -85,7 +96,6 @@ MIDDLEWARES = [
     'crawlo.middleware.request_ignore.RequestIgnoreMiddleware',  # 1. 忽略无效请求
     'crawlo.middleware.download_delay.DownloadDelayMiddleware',  # 2. 控制请求频率
     'crawlo.middleware.default_header.DefaultHeaderMiddleware',  # 3. 添加默认请求头
-    # 'crawlo.middleware.proxy.ProxyMiddleware',  # 4. 设置代理（默认不启用）
     'crawlo.middleware.offsite.OffsiteMiddleware',  # 5. 站外请求过滤
 
     # === 响应处理阶段 ===
@@ -98,8 +108,7 @@ MIDDLEWARES = [
 
 # 框架数据处理管道列表（框架默认管道 + 用户自定义管道）
 PIPELINES = [
-    'crawlo.pipelines.console_pipeline.ConsolePipeline',  # 控制台输出
-    # 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline',     # MySQL 存储（可选）
+    'crawlo.pipelines.console_pipeline.ConsolePipeline',
 ]
 
 # 明确添加默认去重管道到管道列表开头

{crawlo-1.3.0 → crawlo-1.3.2}/crawlo/settings/setting_manager.py

@@ -27,7 +27,14 @@ class SettingManager(MutableMapping):
             user_middlewares = user_config['MIDDLEWARES']
             # 如果用户配置了空列表，则仍然使用默认配置
             if user_middlewares:
-                self.attributes['MIDDLEWARES'] = default_middlewares + user_middlewares
+                # 过滤掉空值和注释
+                user_middlewares = [middleware for middleware in user_middlewares if middleware and not middleware.strip().startswith('#')]
+                # 合并默认中间件和用户中间件，去重但保持顺序
+                merged_middlewares = default_middlewares[:]
+                for middleware in user_middlewares:
+                    if middleware not in merged_middlewares:
+                        merged_middlewares.append(middleware)
+                self.attributes['MIDDLEWARES'] = merged_middlewares
 
         # 合并管道配置
         if 'PIPELINES' in user_config:
@@ -37,8 +44,12 @@ class SettingManager(MutableMapping):
             if user_pipelines:
                 # 过滤掉空值和注释
                 user_pipelines = [pipeline for pipeline in user_pipelines if pipeline and not pipeline.strip().startswith('#')]
-                if user_pipelines:
-                    self.attributes['PIPELINES'] = user_pipelines
+                # 合并默认管道和用户管道，去重但保持顺序
+                merged_pipelines = default_pipelines[:]
+                for pipeline in user_pipelines:
+                    if pipeline not in merged_pipelines:
+                        merged_pipelines.append(pipeline)
+                self.attributes['PIPELINES'] = merged_pipelines
 
         # 特殊处理PIPELINES，确保去重管道在最前面
         dedup_pipeline = self.attributes.get('DEFAULT_DEDUP_PIPELINE')
@@ -56,8 +67,14 @@ class SettingManager(MutableMapping):
             user_extensions = user_config['EXTENSIONS']
             # 如果用户配置了空列表，则仍然使用默认配置
             if user_extensions:
-                self.attributes['EXTENSIONS'] = default_extensions + user_extensions
-            # 如果用户没有配置扩展，则使用默认配置
+                # 过滤掉空值和注释
+                user_extensions = [extension for extension in user_extensions if extension and not extension.strip().startswith('#')]
+                # 合并默认扩展和用户扩展，去重但保持顺序
+                merged_extensions = default_extensions[:]
+                for extension in user_extensions:
+                    if extension not in merged_extensions:
+                        merged_extensions.append(extension)
+                self.attributes['EXTENSIONS'] = merged_extensions
 
         # 更新其他用户配置
         for key, value in user_config.items():
@@ -119,9 +136,15 @@ class SettingManager(MutableMapping):
     def set_settings(self, module):
         if isinstance(module, str):
             module = import_module(module)
+        
+        # 收集模块中的所有配置项
+        module_settings = {}
         for key in dir(module):
             if key.isupper():
-                self.set(key, getattr(module, key))
+                module_settings[key] = getattr(module, key)
+        
+        # 使用合并逻辑而不是直接设置
+        self._merge_config(module_settings)
 
     # 实现 MutableMapping 必须的方法
     def __getitem__(self, item):