crawlo 1.2.9__py3-none-any.whl → 1.3.0__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.2.9'
+__version__ = '1.3.0'

crawlo/commands/run.py

@@ -5,26 +5,27 @@
 # @Author  : crawl-coder
 # @Desc    : 命令行入口：crawlo run <spider_name>|all，用于运行指定爬虫。
 """
+import os
 import sys
 import asyncio
 import configparser
-import os
-from pathlib import Path
 from importlib import import_module
 
+from rich import box
 from rich.console import Console
 from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
 from rich.table import Table
 from rich.text import Text
-from rich import box
-from rich.progress import Progress, SpinnerColumn, TextColumn
 
+from crawlo.commands.stats import record_stats
 from crawlo.crawler import CrawlerProcess
-from crawlo.utils.log import get_logger
 from crawlo.project import get_settings, _find_project_root
-from crawlo.commands.stats import record_stats
+# 使用自定义日志系统
+from crawlo.utils.log import get_logger
 
 logger = get_logger(__name__)
+
 console = Console()
 
 
@@ -77,6 +78,9 @@ def main(args):
     用法:
         crawlo run <spider_name>|all [--json] [--no-stats]
     """
+    # 添加调试信息
+    logger.debug("DEBUG: 进入main函数")
+    
     if len(args) < 1:
         console.print("[bold red]用法:[/bold red] [blue]crawlo run[/blue] <爬虫名称>|all [bold yellow][--json] [--no-stats][/bold yellow]")
         console.print("示例:")
@@ -187,21 +191,7 @@ def main(args):
                     return 1
 
             # 显示即将运行的爬虫列表
-            table = Table(
-                title=f"启动全部 {len(spider_names)} 个爬虫",
-                box=box.ROUNDED,
-                show_header=True,
-                header_style="bold magenta"
-            )
-            table.add_column("名称", style="cyan")
-            table.add_column("类名", style="green")
-
-            for name in sorted(spider_names):
-                cls = process.get_spider_class(name)
-                table.add_row(name, cls.__name__)
-
-            console.print(table)
-            console.print()
+            # 根据用户要求，不再显示详细的爬虫列表信息
 
             # 注册 stats 记录（除非 --no-stats）
             if not no_stats:
@@ -260,20 +250,21 @@ def main(args):
         spider_class = process.get_spider_class(spider_name)
 
         # 显示启动信息
-        if not show_json:
-            info_table = Table(
-                title=f"启动爬虫: [bold cyan]{spider_name}[/bold cyan]",
-                box=box.SIMPLE,
-                show_header=False,
-                title_style="bold green"
-            )
-            info_table.add_column("Key", style="yellow")
-            info_table.add_column("Value", style="cyan")
-            info_table.add_row("Project", project_package)
-            info_table.add_row("Class", spider_class.__name__)
-            info_table.add_row("Module", spider_class.__module__)
-            console.print(info_table)
-            console.print()
+        # 根据用户要求，不再显示项目启动信息
+        # if not show_json:
+        #     info_table = Table(
+        #         title=f"启动爬虫: [bold cyan]{spider_name}[/bold cyan]",
+        #         box=box.SIMPLE,
+        #         show_header=False,
+        #         title_style="bold green"
+        #     )
+        #     info_table.add_column("Key", style="yellow")
+        #     info_table.add_column("Value", style="cyan")
+        #     info_table.add_row("Project", project_package)
+        #     info_table.add_row("Class", spider_class.__name__)
+        #     info_table.add_row("Module", spider_class.__module__)
+        #     console.print(info_table)
+        #     console.print()
 
         # 注册 stats 记录
         if not no_stats:

crawlo/core/engine.py

@@ -75,8 +75,7 @@ class Engine(object):
             version = '1.0.0'
         # Change INFO level log to DEBUG level to avoid duplication with CrawlerProcess startup log
         self.logger.debug(
-            f"Crawlo Started version {version} . "
-            # f"(project name : {self.settings.get('PROJECT_NAME')})"
+            f"Crawlo Started version {version}"
         )
 
     async def start_spider(self, spider):

crawlo/crawler.py

@@ -21,7 +21,7 @@ Example Usage:
     # Single crawler run
     crawler = Crawler(MySpider, settings)
     await crawler.crawl()
-    
+
     # Multi-crawler concurrent management
     process = CrawlerProcess()
     await process.crawl([Spider1, Spider2])
@@ -34,7 +34,6 @@ import threading
 from typing import Type, Optional, Set, List, Union, Dict, Any
 from .spider import Spider, get_global_spider_registry
 from .core.engine import Engine
-from .utils.log import get_logger
 from .subscriber import Subscriber
 from .extension import ExtensionManager
 from .stats_collector import StatsCollector
@@ -42,16 +41,9 @@ from .event import spider_opened, spider_closed
 from .settings.setting_manager import SettingManager
 from crawlo.project import merge_settings, get_settings
 
-# 延迟初始化logger，在需要时通过get_logger获取
-logger = None
-
-
-def _get_logger():
-    """延迟获取logger实例，确保在配置加载后创建"""
-    global logger
-    if logger is None:
-        logger = get_logger(__name__)
-    return logger
+# 使用自定义日志系统
+from crawlo.utils.log import get_logger
+logger = get_logger(__name__)
 
 
 class CrawlerContext:
@@ -110,7 +102,7 @@ class CrawlerContext:
 class Crawler:
     """
     Single crawler runtime instance, managing Spider and engine lifecycle
-    
+
     Provides functionality:
     - Spider lifecycle management (initialization, running, closing)
     - Engine component coordination management
@@ -148,7 +140,7 @@ class Crawler:
     async def crawl(self):
         """
         Start the crawler core process
-        
+
         Includes the following stages:
         1. Initialization stage: Create all components
         2. Validation stage: Check configuration and state
@@ -190,12 +182,12 @@ class Crawler:
             # Update context status
             self.context.increment_completed()
 
-            _get_logger().info(f"Spider {self.spider.name} completed, took {self._get_total_duration():.2f} seconds")
+            logger.info(f"Spider {self.spider.name} completed, took {self._get_total_duration():.2f} seconds")
 
         except Exception as e:
             self._performance_metrics['error_count'] += 1
             self.context.increment_failed(str(e))
-            _get_logger().error(f"Spider {getattr(self.spider, 'name', 'Unknown')} failed to run: {e}", exc_info=True)
+            logger.error(f"Spider {getattr(self.spider, 'name', 'Unknown')} failed to run: {e}", exc_info=True)
             raise
         finally:
             self.context.decrement_active()
@@ -213,7 +205,7 @@ class Crawler:
         else:
             spider_name = 'Unknown'
 
-        _get_logger().info(f"Starting running {spider_name}")
+        logger.info(f"Starting running {spider_name}")
 
     def _validate_crawler_state(self):
         """
@@ -233,7 +225,7 @@ class Crawler:
         if not self.spider.name:
             raise ValueError("Spider name cannot be empty")
 
-        _get_logger().debug(f"Spider {self.spider.name} state validation passed")
+        logger.debug(f"Spider {self.spider.name} state validation passed")
 
     def _get_total_duration(self) -> float:
         """Get total runtime"""
@@ -247,7 +239,7 @@ class Crawler:
             if not self._closed:
                 await self.close()
         except Exception as e:
-            _get_logger().warning(f"Error cleaning up resources: {e}")
+            logger.warning(f"Error cleaning up resources: {e}")
 
     def get_performance_metrics(self) -> Dict[str, Any]:
         """Get performance metrics"""
@@ -267,7 +259,7 @@ class Crawler:
     def _create_spider(self) -> Spider:
         """
         Create and validate spider instance (enhanced version)
-        
+
         Performs the following validations:
         - Spider name must exist
         - start_requests method must be callable
@@ -300,7 +292,7 @@ class Crawler:
 
         # parse method check (warning instead of error)
         if not callable(getattr(spider, 'parse', None)):
-            _get_logger().warning(
+            logger.warning(
                 f"Spider '{spider.name}' does not define 'parse' method.\n"
                 f"Ensure all Requests specify a callback function, otherwise responses will be ignored."
             )
@@ -308,27 +300,29 @@ class Crawler:
         # Set spider configuration
         self._set_spider(spider)
 
-        _get_logger().debug(f"Spider '{spider.name}' initialized successfully")
+        logger.debug(f"Spider '{spider.name}' initialized successfully")
         return spider
 
     def _create_engine(self) -> Engine:
         """Create and initialize engine"""
         engine = Engine(self)
         engine.engine_start()
-        _get_logger().debug(f"Engine initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(f"Engine initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return engine
 
     def _create_stats(self) -> StatsCollector:
         """Create stats collector"""
         stats = StatsCollector(self)
-        _get_logger().debug(f"Stats collector initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(
+            f"Stats collector initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return stats
 
     def _create_extension(self) -> ExtensionManager:
         """Create extension manager"""
         # Modify extension manager creation method, delay initialization until needed
         extension = ExtensionManager.create_instance(self)
-        _get_logger().debug(f"Extension manager initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(
+            f"Extension manager initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return extension
 
     def _set_spider(self, spider: Spider):
@@ -343,12 +337,12 @@ class Crawler:
         # Merge spider custom configuration
         merge_settings(spider, self.settings)
 
-        _get_logger().debug(f"Spider '{spider.name}' configuration merged successfully")
+        logger.debug(f"Spider '{spider.name}' configuration merged successfully")
 
     async def close(self, reason='finished') -> None:
         """
         Close crawler and clean up resources (enhanced version)
-        
+
         Ensure closing only once and handle all cleanup operations
         """
         async with self._close_lock:
@@ -371,15 +365,15 @@ class Crawler:
                         from crawlo.commands.stats import record_stats
                         record_stats(self)
                     except ImportError:
-                        _get_logger().debug("Statistics recording module does not exist, skipping statistics recording")
+                        logger.debug("Statistics recording module does not exist, skipping statistics recording")
 
-                _get_logger().info(
+                logger.info(
                     f"Spider '{getattr(self.spider, 'name', 'Unknown')}' closed, "
                     f"reason: {reason}, took: {self._get_total_duration():.2f} seconds"
                 )
 
             except Exception as e:
-                _get_logger().error(f"Error closing crawler: {e}", exc_info=True)
+                logger.error(f"Error closing crawler: {e}", exc_info=True)
             finally:
                 # Ensure resource cleanup
                 await self._cleanup_resources()
@@ -413,13 +407,13 @@ class Crawler:
         if cleanup_tasks:
             await asyncio.gather(*cleanup_tasks, return_exceptions=True)
 
-        _get_logger().debug("Resource cleanup completed")
+        logger.debug("Resource cleanup completed")
 
 
 class CrawlerProcess:
     """
     Crawler process manager
-    
+
     Supported features:
     - Multi-crawler concurrent scheduling and resource management
     - Automatic module discovery and spider registration
@@ -428,15 +422,15 @@ class CrawlerProcess:
     - Real-time status monitoring and statistics
     - Error recovery and retry mechanism
     - Large-scale crawler optimization support
-    
+
     Usage example:
         # Basic usage
         process = CrawlerProcess()
         await process.crawl(MySpider)
-        
+
         # Multi-crawler concurrency
         await process.crawl([Spider1, Spider2, 'spider_name'])
-        
+
         # Custom concurrency
         process = CrawlerProcess(max_concurrency=8)
     """
@@ -563,7 +557,7 @@ class CrawlerProcess:
     def auto_discover(modules: List[str]):
         """
         Automatically import modules, trigger Spider class definition and registration (enhanced version)
-        
+
         Supports recursive scanning and error recovery
         """
         import importlib
@@ -617,7 +611,7 @@ class CrawlerProcess:
     async def crawl(self, spiders: Union[Type[Spider], str, List[Union[Type[Spider], str]]]):
         """
         Start one or more crawlers
-        
+
         Enhanced features:
         - Intelligent concurrency control
         - Real-time monitoring and statistics
@@ -639,7 +633,7 @@ class CrawlerProcess:
         await self.start_monitoring()
 
         try:
-            # Phase 3: Sort by class name to ensure predictable startup order
+            # Phase 3: Initialize context and monitoring
             spider_classes_to_run.sort(key=lambda cls: cls.__name__.lower())
 
             logger.debug(
@@ -738,7 +732,7 @@ class CrawlerProcess:
     ) -> List[Type[Spider]]:
         """
         Resolve input to spider class list
-        
+
         Supports various input formats and validates uniqueness
         """
         inputs = self._normalize_inputs(spiders_input)
@@ -762,7 +756,8 @@ class CrawlerProcess:
                 seen_spider_names.add(spider_name)
                 spider_classes.append(spider_cls)
 
-                logger.debug(f"Spider resolved successfully: {item} -> {spider_cls.__name__} (name='{spider_name}')")
+                logger.debug(
+                    f"Spider resolved successfully: {item} -> {spider_cls.__name__} (name='{spider_name}')")
 
             except Exception as e:
                 logger.error(f"Failed to resolve spider: {item} - {e}")
@@ -774,7 +769,7 @@ class CrawlerProcess:
     def _normalize_inputs(spiders_input) -> List[Union[Type[Spider], str]]:
         """
         Normalize input to list
-        
+
         Supports more input types and provides better error information
         """
         if isinstance(spiders_input, (type, str)):
@@ -793,7 +788,7 @@ class CrawlerProcess:
     def _resolve_spider_class(self, item: Union[Type[Spider], str]) -> Type[Spider]:
         """
         Resolve single input item to spider class
-        
+
         Provides better error prompts and debugging information
         """
         if isinstance(item, type) and issubclass(item, Spider):
@@ -820,7 +815,7 @@ class CrawlerProcess:
     async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
         """
         Spider running function limited by semaphore
-        
+
         Includes enhanced error handling and monitoring functionality
         """
         task = asyncio.current_task()
@@ -888,7 +883,7 @@ class CrawlerProcess:
     def _shutdown(self, _signum, _frame):
         """
         Graceful shutdown signal handling
-        
+
         Provides better shutdown experience and resource cleanup
         """
         signal_name = {signal.SIGINT: 'SIGINT', signal.SIGTERM: 'SIGTERM'}.get(_signum, str(_signum))
@@ -913,7 +908,7 @@ class CrawlerProcess:
     async def _wait_for_shutdown(self):
         """
         Wait for all active tasks to complete
-        
+
         Provides better shutdown time control and progress feedback
         """
         try:
@@ -967,15 +962,15 @@ class CrawlerProcess:
     def _get_default_settings(cls) -> SettingManager:
         """
         Load default configuration
-        
+
         Provides better error handling and fallback strategy
         """
         try:
             settings = get_settings()
-            _get_logger().debug("Default configuration loaded successfully")
+            logger.debug("Default configuration loaded successfully")
             return settings
         except Exception as e:
-            _get_logger().warning(f"Unable to load default configuration: {e}, using empty configuration")
+            logger.warning(f"Unable to load default configuration: {e}, using empty configuration")
             return SettingManager()
 
     def _log_startup_info(self):
@@ -990,7 +985,7 @@ class CrawlerProcess:
 
         # Build startup info log
         startup_info = [
-            f"Crawlo Framework Started v{version}"
+            f"Crawlo Framework Started {version}"
         ]
 
         # Get actual queue type
@@ -1018,7 +1013,7 @@ class CrawlerProcess:
         else:
             startup_info.append(f"Run Mode: {run_mode}")
 
-        # Print startup information
+        # Print startup information at INFO level
         for info in startup_info:
             logger.info(info)
 
@@ -1032,7 +1027,7 @@ def create_crawler_with_optimizations(
 ) -> Crawler:
     """
     Create an optimized crawler instance
-    
+
     :param spider_cls: Spider class
     :param settings: Settings manager
     :param optimization_kwargs: Optimization parameters
@@ -1056,7 +1051,7 @@ def create_process_with_large_scale_config(
 ) -> CrawlerProcess:
     """
     Create a process manager that supports large-scale optimization
-    
+
     :param config_type: Configuration type ('conservative', 'balanced', 'aggressive', 'memory_optimized')
     :param concurrency: Concurrency count
     :param kwargs: Other parameters
@@ -1100,4 +1095,4 @@ __all__ = [
     'CrawlerContext',
     'create_crawler_with_optimizations',
     'create_process_with_large_scale_config'
-]
+]

crawlo/extension/logging_extension.py

@@ -1,8 +1,10 @@
 from typing import Any
 from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
 from crawlo.utils.log import LoggerManager
 
+# 使用全局logger，避免每个模块都创建自己的延迟初始化函数
+logger = LoggerManager.get_logger(__name__)
+
 
 class CustomLoggerExtension:
     """
@@ -32,7 +34,7 @@ class CustomLoggerExtension:
         return cls(crawler.settings)
 
     def spider_opened(self, spider: Any) -> None:
-        logger = get_logger(__name__)
+        logger = LoggerManager.get_logger(__name__)
         try:
             logger.info(
                 f"CustomLoggerExtension: Logging initialized. "

crawlo/middleware/middleware_manager.py

@@ -133,4 +133,4 @@ class MiddlewareManager:
     def _validate_middleware_method(method_name, middleware) -> bool:
         method = getattr(type(middleware), method_name)
         base_method = getattr(BaseMiddleware, method_name)
-        return False if method == base_method else True
+        return False if method == base_method else True

crawlo/mode_manager.py

@@ -19,36 +19,37 @@ from crawlo.utils.log import get_logger
 
 class RunMode(Enum):
     """运行模式枚举"""
-    STANDALONE = "standalone"    # 单机模式
+    STANDALONE = "standalone"  # 单机模式
     DISTRIBUTED = "distributed"  # 分布式模式
-    AUTO = "auto"               # 自动检测模式
+    AUTO = "auto"  # 自动检测模式
 
 
 class ModeManager:
     """运行模式管理器"""
-    
+
     def __init__(self):
         self.logger = get_logger(self.__class__.__name__)
-    
+
     @staticmethod
     def get_standalone_settings() -> Dict[str, Any]:
         """获取单机模式配置"""
         return {
             'QUEUE_TYPE': 'memory',
             'FILTER_CLASS': 'crawlo.filters.memory_filter.MemoryFilter',
+            'DEFAULT_DEDUP_PIPELINE': 'crawlo.pipelines.memory_dedup_pipeline.MemoryDedupPipeline',
             'CONCURRENCY': 8,
             'MAX_RUNNING_SPIDERS': 1,
             'DOWNLOAD_DELAY': 1.0,
             'LOG_LEVEL': 'INFO',
         }
-    
+
     @staticmethod
     def get_distributed_settings(
-        redis_host: str = '127.0.0.1',
-        redis_port: int = 6379,
-        redis_password: Optional[str] = None,
-        redis_db: int = 0,  # 添加 redis_db 参数
-        project_name: str = 'crawlo'
+            redis_host: str = '127.0.0.1',
+            redis_port: int = 6379,
+            redis_password: Optional[str] = None,
+            redis_db: int = 0,  # 添加 redis_db 参数
+            project_name: str = 'crawlo'
     ) -> Dict[str, Any]:
         """获取分布式模式配置"""
         # 构建 Redis URL，使用传入的 redis_db 参数
@@ -56,7 +57,7 @@ class ModeManager:
             redis_url = f'redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}'
         else:
             redis_url = f'redis://{redis_host}:{redis_port}/{redis_db}'
-        
+
         return {
             'PROJECT_NAME': project_name,  # 添加项目名称到配置中
             'QUEUE_TYPE': 'redis',
@@ -74,7 +75,7 @@ class ModeManager:
             'DOWNLOAD_DELAY': 1.0,
             'LOG_LEVEL': 'INFO',
         }
-    
+
     @staticmethod
     def get_auto_settings() -> Dict[str, Any]:
         """获取自动检测模式配置"""
@@ -86,28 +87,28 @@ class ModeManager:
             'DOWNLOAD_DELAY': 1.0,
             'LOG_LEVEL': 'INFO',
         }
-    
+
     def resolve_mode_settings(
-        self, 
-        mode: str = 'standalone',
-        **kwargs
+            self,
+            mode: str = 'standalone',
+            **kwargs
     ) -> Dict[str, Any]:
         """
         解析运行模式并返回对应配置
-        
+
         Args:
             mode: 运行模式 ('standalone', 'distributed', 'auto')
             **kwargs: 额外配置参数
-            
+
         Returns:
             Dict[str, Any]: 配置字典
         """
         mode = RunMode(mode.lower())
-        
+
         if mode == RunMode.STANDALONE:
             self.logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
             settings = self.get_standalone_settings()
-            
+
         elif mode == RunMode.DISTRIBUTED:
             self.logger.info("使用分布式模式 - 支持多节点扩展，适合大规模爬取")
             settings = self.get_distributed_settings(
@@ -117,25 +118,25 @@ class ModeManager:
                 redis_db=kwargs.get('redis_db', 0),  # 添加 redis_db 参数
                 project_name=kwargs.get('project_name', 'crawlo')
             )
-            
+
         elif mode == RunMode.AUTO:
             self.logger.info("使用自动检测模式 - 智能选择最佳运行方式")
             settings = self.get_auto_settings()
-            
+
         else:
             raise ValueError(f"不支持的运行模式: {mode}")
-        
+
         # 合并用户自定义配置
-        user_settings = {k: v for k, v in kwargs.items() 
-                        if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
+        user_settings = {k: v for k, v in kwargs.items()
+                         if k not in ['redis_host', 'redis_port', 'redis_password', 'project_name']}
         settings.update(user_settings)
-        
+
         return settings
-    
+
     def from_environment(self) -> Dict[str, Any]:
         """从环境变量构建配置"""
         config = {}
-        
+
         # 扫描 CRAWLO_ 前缀的环境变量
         for key, value in os.environ.items():
             if key.startswith('CRAWLO_'):
@@ -150,7 +151,7 @@ class ModeManager:
                         config[config_key] = float(value)
                     except ValueError:
                         config[config_key] = value
-        
+
         return config
 
 
@@ -161,12 +162,12 @@ def standalone_mode(**kwargs) -> Dict[str, Any]:
 
 
 def distributed_mode(
-    redis_host: str = '127.0.0.1',
-    redis_port: int = 6379,
-    redis_password: Optional[str] = None,
-    redis_db: int = 0,  # 添加 redis_db 参数
-    project_name: str = 'crawlo',
-    **kwargs
+        redis_host: str = '127.0.0.1',
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        redis_db: int = 0,  # 添加 redis_db 参数
+        project_name: str = 'crawlo',
+        **kwargs
 ) -> Dict[str, Any]:
     """快速创建分布式模式配置"""
     return ModeManager().resolve_mode_settings(
@@ -190,7 +191,7 @@ def from_env(default_mode: str = 'standalone') -> Dict[str, Any]:
     """从环境变量创建配置"""
     # 移除直接使用 os.getenv()，要求通过 settings 配置
     raise RuntimeError("环境变量配置已移除，请在 settings 中配置相关参数")
-    
+
     # 保留原有代码作为参考
     # mode = os.getenv('CRAWLO_MODE', default_mode).lower()
     # 

crawlo/pipelines/pipeline_manager.py

@@ -4,7 +4,6 @@ from typing import List
 from pprint import pformat
 from asyncio import create_task
 
-
 from crawlo.utils.log import get_logger
 from crawlo.event import item_successful, item_discard
 from crawlo.project import load_class, common_call
@@ -20,6 +19,19 @@ class PipelineManager:
 
         self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
         pipelines = self.crawler.settings.get_list('PIPELINES')
+        dedup_pipeline = self.crawler.settings.get('DEFAULT_DEDUP_PIPELINE')
+
+        # 添加调试信息
+        self.logger.debug(f"PIPELINES from settings: {pipelines}")
+        self.logger.debug(f"DEFAULT_DEDUP_PIPELINE from settings: {dedup_pipeline}")
+
+        # 确保DEFAULT_DEDUP_PIPELINE被添加到管道列表开头
+        if dedup_pipeline:
+            # 移除所有去重管道实例（如果存在）
+            pipelines = [item for item in pipelines if item != dedup_pipeline]
+            # 在开头插入去重管道
+            pipelines.insert(0, dedup_pipeline)
+
         self._add_pipelines(pipelines)
         self._add_methods()
 

crawlo/project.py

@@ -7,18 +7,10 @@ from inspect import iscoroutinefunction
 from typing import Callable, Optional, Any
 
 from crawlo.settings.setting_manager import SettingManager
-from crawlo.utils.log import get_logger
+from crawlo.utils.log import get_logger, LoggerManager
 
-# 延迟初始化logger，在需要时通过get_logger获取
-logger = None
-
-
-def _get_logger():
-    """延迟获取logger实例，确保在配置加载后创建"""
-    global logger
-    if logger is None:
-        logger = get_logger(__name__)
-    return logger
+# 使用全局logger，避免每个模块都创建自己的延迟初始化函数
+logger = get_logger(__name__)
 
 
 def load_class(path: str) -> Any:
@@ -50,7 +42,7 @@ def merge_settings(spider, settings):
     spider_name = getattr(spider, 'name', 'UnknownSpider')
     # 检查 settings 是否为 SettingManager 实例
     if not hasattr(settings, 'update_attributes'):
-        _get_logger().error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
+        logger.error(f"merge_settings 接收到的 settings 不是 SettingManager 实例: {type(settings)}")
         # 如果是字典，创建一个新的 SettingManager 实例
         if isinstance(settings, dict):
             from crawlo.settings.setting_manager import SettingManager
@@ -58,14 +50,14 @@ def merge_settings(spider, settings):
             new_settings.update_attributes(settings)
             settings = new_settings
         else:
-            _get_logger().error("无法处理的 settings 类型")
+            logger.error("无法处理的 settings 类型")
             return
             
     if hasattr(spider, 'custom_settings'):
         custom_settings = getattr(spider, 'custom_settings')
         settings.update_attributes(custom_settings)
     else:
-        _get_logger().debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")
 
 
 async def common_call(func: Callable, *args, **kwargs):
@@ -93,7 +85,7 @@ def _get_settings_module_from_cfg(cfg_path: str) -> str:
         config.read(cfg_path, encoding="utf-8")
         if config.has_section("settings") and config.has_option("settings", "default"):
             module_path = config.get("settings", "default")
-            _get_logger().debug(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
+            logger.debug(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
             return module_path
         else:
             raise RuntimeError(f"配置文件缺少 [settings] 或 default 选项: {cfg_path}")
@@ -114,7 +106,7 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
     for root, dirs, files in os.walk(path):
         if "crawlo.cfg" in files:
             cfg_path = os.path.join(root, "crawlo.cfg")
-            _get_logger().debug(f"✅ 找到项目配置文件: {cfg_path}")
+            logger.debug(f"✅ 找到项目配置文件: {cfg_path}")
             return root
     
     # 向上查找直到找到 crawlo.cfg 或包含 settings.py 和 __init__.py 的目录
@@ -130,20 +122,20 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
         # 检查 crawlo.cfg
         cfg_file = os.path.join(path, "crawlo.cfg")
         if os.path.isfile(cfg_file):
-            _get_logger().debug(f"✅ 找到项目配置文件: {cfg_file}")
+            logger.debug(f"✅ 找到项目配置文件: {cfg_file}")
             return path
 
         # 检查 settings.py 和 __init__.py
         settings_file = os.path.join(path, "settings.py")
         init_file = os.path.join(path, "__init__.py")
         if os.path.isfile(settings_file) and os.path.isfile(init_file):
-            _get_logger().debug(f"✅ 找到项目模块: {path}")
+            logger.debug(f"✅ 找到项目模块: {path}")
             # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
             parent = os.path.dirname(path)
             if parent != path:
                 parent_cfg = os.path.join(parent, "crawlo.cfg")
                 if os.path.isfile(parent_cfg):
-                    _get_logger().debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
+                    logger.debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
                     return parent
             return path
 
@@ -167,19 +159,19 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
                 
                 cfg_file = os.path.join(path, "crawlo.cfg")
                 if os.path.isfile(cfg_file):
-                    _get_logger().debug(f"✅ 找到项目配置文件: {cfg_file}")
+                    logger.debug(f"✅ 找到项目配置文件: {cfg_file}")
                     return path
 
                 settings_file = os.path.join(path, "settings.py")
                 init_file = os.path.join(path, "__init__.py")
                 if os.path.isfile(settings_file) and os.path.isfile(init_file):
-                    _get_logger().debug(f"✅ 找到项目模块: {path}")
+                    logger.debug(f"✅ 找到项目模块: {path}")
                     # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
                     parent = os.path.dirname(path)
                     if parent != path:
                         parent_cfg = os.path.join(parent, "crawlo.cfg")
                         if os.path.isfile(parent_cfg):
-                            _get_logger().debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
+                            logger.debug(f"✅ 在上层目录找到项目配置文件: {parent_cfg}")
                             return parent
                     return path
 
@@ -204,19 +196,19 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
                 
                 cfg_file = os.path.join(path, "crawlo.cfg")
                 if os.path.isfile(cfg_file):
-                    _get_logger().debug(f"找到项目配置文件: {cfg_file}")
+                    logger.debug(f"找到项目配置文件: {cfg_file}")
                     return path
 
                 settings_file = os.path.join(path, "settings.py")
                 init_file = os.path.join(path, "__init__.py")
                 if os.path.isfile(settings_file) and os.path.isfile(init_file):
-                    _get_logger().debug(f"找到项目模块: {path}")
+                    logger.debug(f"找到项目模块: {path}")
                     # 即使找到了项目模块，也继续向上查找是否有 crawlo.cfg
                     parent = os.path.dirname(path)
                     if parent != path:
                         parent_cfg = os.path.join(parent, "crawlo.cfg")
                         if os.path.isfile(parent_cfg):
-                            _get_logger().debug(f"在上层目录找到项目配置文件: {parent_cfg}")
+                            logger.debug(f"在上层目录找到项目配置文件: {parent_cfg}")
                             return parent
                     return path
 
@@ -227,7 +219,7 @@ def _find_project_root(start_path: str = ".") -> Optional[str]:
     except Exception:
         pass
 
-    _get_logger().warning("未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
+    logger.warning("未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
     return None
 
 
@@ -241,8 +233,7 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
     Returns:
         SettingManager: 已加载配置的实例
     """
-    # Change INFO level log to DEBUG level to avoid redundant output
-    _get_logger().debug("🚀 正在初始化 Crawlo 项目配置...")
+    logger.debug("🚀 正在初始化 Crawlo 项目配置...")
 
     # 1. 查找项目根
     project_root = _find_project_root()
@@ -259,32 +250,35 @@ def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
         # 推断：项目目录名.settings
         project_name = os.path.basename(project_root)
         settings_module_path = f"{project_name}.settings"
-        _get_logger().warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
+        logger.warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
 
     # 3. 注入 sys.path
     project_root_str = os.path.abspath(project_root)
     if project_root_str not in sys.path:
         sys.path.insert(0, project_root_str)
-        _get_logger().debug(f"📁 项目根目录已加入 sys.path: {project_root_str}")
+        logger.debug(f"📁 项目根目录已加入 sys.path: {project_root_str}")
 
     # 4. 加载 SettingManager
-    _get_logger().debug(f"⚙️ 正在加载配置模块: {settings_module_path}")
+    logger.debug(f"⚙️ 正在加载配置模块: {settings_module_path}")
     settings = SettingManager()
 
     try:
         settings.set_settings(settings_module_path)
-        _get_logger().debug("✅ settings 模块加载成功")
+        logger.debug("✅ settings 模块加载成功")
     except Exception as e:
         raise ImportError(f"加载 settings 模块失败 '{settings_module_path}': {e}")
 
     # 5. 合并运行时配置
     if custom_settings:
         settings.update_attributes(custom_settings)
-        _get_logger().debug(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
+        logger.debug(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
 
     # 6. 显示核心配置摘要（INFO级别）
     # _log_settings_summary(settings)
 
+    # 配置日志系统
+    LoggerManager.configure(settings)
+    
     # 将项目初始化完成的消息改为DEBUG级别
-    _get_logger().debug("🎉 Crawlo 项目配置初始化完成！")
+    logger.debug("🎉 Crawlo 项目配置初始化完成！")
     return settings

crawlo/settings/setting_manager.py

@@ -15,38 +15,50 @@ class SettingManager(MutableMapping):
         self.set_settings(default_settings)
         # 在初始化时合并配置
         self._merge_config(values)
-        
+
     def _merge_config(self, user_config):
         """合并默认配置和用户配置"""
         if not user_config:
             return
-            
+
         # 合并中间件配置
         if 'MIDDLEWARES' in user_config:
             default_middlewares = self.attributes.get('MIDDLEWARES', [])
             user_middlewares = user_config['MIDDLEWARES']
-            self.attributes['MIDDLEWARES'] = default_middlewares + user_middlewares
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_middlewares:
+                self.attributes['MIDDLEWARES'] = default_middlewares + user_middlewares
+
         # 合并管道配置
         if 'PIPELINES' in user_config:
             default_pipelines = self.attributes.get('PIPELINES', [])
             user_pipelines = user_config['PIPELINES']
-            merged_pipelines = default_pipelines + user_pipelines
-            # 特殊处理PIPELINES，确保去重管道在最前面
-            dedup_pipeline = self.attributes.get('DEFAULT_DEDUP_PIPELINE')
-            if dedup_pipeline:
-                # 移除所有去重管道实例（如果存在）
-                merged_pipelines = [item for item in merged_pipelines if item != dedup_pipeline]
-                # 在开头插入去重管道
-                merged_pipelines.insert(0, dedup_pipeline)
-            self.attributes['PIPELINES'] = merged_pipelines
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_pipelines:
+                # 过滤掉空值和注释
+                user_pipelines = [pipeline for pipeline in user_pipelines if pipeline and not pipeline.strip().startswith('#')]
+                if user_pipelines:
+                    self.attributes['PIPELINES'] = user_pipelines
+
+        # 特殊处理PIPELINES，确保去重管道在最前面
+        dedup_pipeline = self.attributes.get('DEFAULT_DEDUP_PIPELINE')
+        if dedup_pipeline:
+            pipelines = self.attributes.get('PIPELINES', [])
+            # 移除所有去重管道实例（如果存在）
+            pipelines = [item for item in pipelines if item != dedup_pipeline]
+            # 在开头插入去重管道
+            pipelines.insert(0, dedup_pipeline)
+            self.attributes['PIPELINES'] = pipelines
+
         # 合并扩展配置
         if 'EXTENSIONS' in user_config:
             default_extensions = self.attributes.get('EXTENSIONS', [])
             user_extensions = user_config['EXTENSIONS']
-            self.attributes['EXTENSIONS'] = default_extensions + user_extensions
-            
+            # 如果用户配置了空列表，则仍然使用默认配置
+            if user_extensions:
+                self.attributes['EXTENSIONS'] = default_extensions + user_extensions
+            # 如果用户没有配置扩展，则使用默认配置
+
         # 更新其他用户配置
         for key, value in user_config.items():
             if key not in ['MIDDLEWARES', 'PIPELINES', 'EXTENSIONS']:
@@ -147,7 +159,7 @@ class SettingManager(MutableMapping):
         # 创建一个新的实例
         cls = self.__class__
         new_instance = cls.__new__(cls)
-        
+
         # 复制attributes字典，但排除不可pickle的对象
         new_attributes = {}
         for key, value in self.attributes.items():
@@ -157,8 +169,8 @@ class SettingManager(MutableMapping):
             except Exception:
                 # 如果复制失败，保留原始引用（对于logger等对象）
                 new_attributes[key] = value
-        
+
         # 设置新实例的attributes
         new_instance.attributes = new_attributes
-        
+
         return new_instance

crawlo/utils/log.py

@@ -8,28 +8,20 @@ from logging import (
     INFO,
     getLevelName,
 )
-# 导入日志轮转处理器
-from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler
 
 LOG_FORMAT = '%(asctime)s - [%(name)s] - %(levelname)s: %(message)s'
 
 
 class LoggerManager:
+    """日志管理器，提供统一的日志配置和获取接口"""
     logger_cache = {}
     _default_filename = None
     _default_level = DEBUG  # 设置为最低级别，由handler控制实际输出
-    _default_file_level = INFO
-    _default_console_level = INFO
+    _default_file_level = INFO  # 默认为INFO级别
+    _default_console_level = INFO  # 默认为INFO级别
     _default_log_format = LOG_FORMAT
     _default_encoding = 'utf-8'
-    # 添加日志轮转相关默认配置
-    _default_log_max_bytes = 10 * 1024 * 1024  # 10MB
-    _default_log_backup_count = 5
-    _default_log_when = 'midnight'
-    _default_log_interval = 1
-    _default_log_use_rotation = False
-    _default_log_rotation_type = 'size'  # 'size' or 'time'
-    _default_log_rotation_suffix = None  # 轮转文件后缀格式
+    _configured = False  # 标记是否已配置
 
     @classmethod
     def _to_level(cls, level):
@@ -67,19 +59,12 @@ class LoggerManager:
         get_val = settings.get if hasattr(settings, 'get') else (lambda k, d=None: kwargs.get(k, d))
 
         filename = get_val('LOG_FILE')
-        level = get_val('LOG_LEVEL', 'DEBUG')  # 默认为DEBUG级别
-        file_level = get_val('LOG_FILE_LEVEL', 'INFO')
-        console_level = get_val('LOG_CONSOLE_LEVEL', 'INFO')  # 保留LOG_CONSOLE_LEVEL配置
+        level = get_val('LOG_LEVEL', 'INFO')  # 默认为INFO级别
+        file_level = get_val('LOG_FILE_LEVEL', level)  # 默认继承LOG_LEVEL的值
+        # 根据项目规范，已完全移除LOG_CONSOLE_LEVEL支持，统一使用LOG_LEVEL控制控制台和文件的日志输出级别
+        console_level = level  # 控制台日志级别直接使用LOG_LEVEL的值
         log_format = get_val('LOG_FORMAT', LOG_FORMAT)
         encoding = get_val('LOG_ENCODING', 'utf-8')
-        # 获取日志轮转配置
-        use_rotation = get_val('LOG_USE_ROTATION', False)
-        rotation_type = get_val('LOG_ROTATION_TYPE', 'size')
-        max_bytes = get_val('LOG_MAX_BYTES', cls._default_log_max_bytes)
-        backup_count = get_val('LOG_BACKUP_COUNT', cls._default_log_backup_count)
-        when = get_val('LOG_WHEN', cls._default_log_when)
-        interval = get_val('LOG_INTERVAL', cls._default_log_interval)
-        rotation_suffix = get_val('LOG_ROTATION_SUFFIX', cls._default_log_rotation_suffix)  # 轮转文件后缀
 
         cls._default_filename = filename
         cls._default_level = cls._to_level(level)
@@ -87,21 +72,13 @@ class LoggerManager:
         cls._default_console_level = cls._to_level(console_level)
         cls._default_log_format = log_format
         cls._default_encoding = encoding
-        # 设置日志轮转配置
-        cls._default_log_use_rotation = use_rotation
-        cls._default_log_rotation_type = rotation_type
-        cls._default_log_max_bytes = max_bytes
-        cls._default_log_backup_count = backup_count
-        cls._default_log_when = when
-        cls._default_log_interval = interval
-        cls._default_log_rotation_suffix = rotation_suffix
-        
-        # 移除对根日志记录器级别的修改，避免副作用
+
+        cls._configured = True
 
     @classmethod
     def get_logger(cls, name='default', level=None, filename=None):
         """
-        简化接口，只暴露必要参数
+        获取logger实例
         """
         # 确定最终参数
         # 如果传入了level参数，则使用它，否则使用默认级别
@@ -110,7 +87,7 @@ class LoggerManager:
         else:
             # Logger级别设置为DEBUG（最低级别），由handler控制实际输出
             final_level = DEBUG
-            
+
         final_filename = filename if filename is not None else cls._default_filename
 
         # 安全的字符串化 key，避免任何 unhashable 类型
@@ -146,32 +123,9 @@ class LoggerManager:
                 if log_dir and not os.path.exists(log_dir):
                     os.makedirs(log_dir, exist_ok=True)
 
-                # 检查是否启用日志轮转
-                if cls._default_log_use_rotation:
-                    if cls._default_log_rotation_type == 'size':
-                        # 使用大小轮转
-                        fh = RotatingFileHandler(
-                            final_filename, 
-                            maxBytes=cls._default_log_max_bytes,
-                            backupCount=cls._default_log_backup_count,
-                            encoding=cls._default_encoding
-                        )
-                    else:
-                        # 使用时间轮转
-                        fh = TimedRotatingFileHandler(
-                            final_filename,
-                            when=cls._default_log_when,
-                            interval=cls._default_log_interval,
-                            backupCount=cls._default_log_backup_count,
-                            encoding=cls._default_encoding
-                        )
-                        # 如果提供了自定义后缀格式，则设置
-                        if cls._default_log_rotation_suffix:
-                            fh.suffix = cls._default_log_rotation_suffix
-                else:
-                    # 使用普通文件处理器（默认行为，会追加到文件）
-                    fh = FileHandler(final_filename, mode='a', encoding=cls._default_encoding)
-                    
+                # 使用普通文件处理器（移除日志轮转功能）
+                fh = FileHandler(final_filename, mode='a', encoding=cls._default_encoding)
+
                 fh.setFormatter(formatter)
                 fh.setLevel(cls._default_file_level)
                 _logger.addHandler(fh)
@@ -183,6 +137,11 @@ class LoggerManager:
         cls.logger_cache[key] = _logger
         return _logger
 
+    @classmethod
+    def is_configured(cls):
+        """检查日志系统是否已配置"""
+        return cls._configured
+
 
 # 全局快捷函数
 get_logger = LoggerManager.get_logger

{crawlo-1.2.9.dist-info → crawlo-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.2.9
+Version: 1.3.0
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

{crawlo-1.2.9.dist-info → crawlo-1.3.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 crawlo/__init__.py,sha256=qZzTmb7hw5h_qcP2EYGUZcoSScxlKZFJ76CjSeS7UfA,1381
-crawlo/__version__.py,sha256=S2h29KjM2QfA587KjqKjzd2pK0GLkBmHIq8WDOutUiI,21
+crawlo/__version__.py,sha256=zi_LaUT_OsChAtsPXbOeRpQkCohSsOyeXfavQPM0GoE,22
 crawlo/cli.py,sha256=OXprmcTUbFK02ptw_Gq8Gk4-ZCU-WEMJgzU1ztgP6Bk,2327
 crawlo/config.py,sha256=dNoNyTkXLe2msQ7bZx3YTQItk1m49nIg5-g89FQDNwE,9486
 crawlo/config_validator.py,sha256=gsiLqf5swWd9ISDvoLqCdG7iSXr-ZdBPD4iT6ug1ua4,11239
-crawlo/crawler.py,sha256=cXvzTF3W6LeZNll4hbMLK-E1xDpZL5M9fgtd_8Gk6eI,39863
+crawlo/crawler.py,sha256=rxyjA5pXOd709bujgniqYG9tR3eoNaok6wJaeZOgzmo,39451
 crawlo/event.py,sha256=7-y6HNv_EIJSYQNzsj0mVK-Gg4ON3wdQeMdQjfFJPlw,313
 crawlo/exceptions.py,sha256=sMay0wnWLfc_FXWslqxm60qz6b66LXs3EdN_w8ygE9k,1166
-crawlo/mode_manager.py,sha256=D8LOgqkqN4yglCKtXw56mq41r30FvxCUb5JOitSTp_U,7502
-crawlo/project.py,sha256=vHqDrGFgRThzFuF2RxDtSHxPerruR3liwXuGC3cVda8,10796
+crawlo/mode_manager.py,sha256=soEgZNBt6jA0qtC1WH-MG_2WngDk2RfmQckLsK3NzmQ,7510
+crawlo/project.py,sha256=830PPRUD6ldE8MKPdkFkPiUcecHhlWP3fUXYC96_T0Y,10506
 crawlo/stats_collector.py,sha256=NkO09CB-220qz5rxFcD_dedGfr2VPFrDo4hya0Zh8Qc,1577
 crawlo/subscriber.py,sha256=D3hzE7Pc_zJjc-zR7lct5pt32bz6LsDYeC8uHlS4Hso,4986
 crawlo/task_manager.py,sha256=19InAxS9oJ9EMj20Aw2urN-v6BeC22dkgPuW-B9-4UI,819
@@ -16,12 +16,12 @@ crawlo/commands/check.py,sha256=7pD43s97DD-fSLO9OEOuNcNr7o-2g94rJULL8fUzdaI,2260
 crawlo/commands/genspider.py,sha256=HhtvBLkIuhYtJUzom6PquItiC22vU9LNpOkjDUiqdM4,4937
 crawlo/commands/help.py,sha256=gwfHibRpdYDmZO6waUMOEn8SMJ_ubdjL-prD5fiuVY8,4973
 crawlo/commands/list.py,sha256=BqlPjBa5FLotjAlyZ3-nGmXg5cWcCNbHi8U5znb2_D8,5722
-crawlo/commands/run.py,sha256=b_HxEmaze0jpPNIrPbm5lyPYZ20--eUVKJZpXoATf0M,12088
+crawlo/commands/run.py,sha256=KcJ4h4D7lavB6qQDpYMrbgJMgY5vCSLHaLckos5EUNY,11793
 crawlo/commands/startproject.py,sha256=aqKRJarKqTf5XjJnGXwjRpp0uYF16LreFbwwQLGpK-0,16070
 crawlo/commands/stats.py,sha256=8wTubR1RQ1JPTlpOKArcGcQ39bM-0cuH27lDpndnwPQ,6014
 crawlo/commands/utils.py,sha256=pXiFzwVIVXdSPO2Fty_u19P1lsE8HmuE8gTMamKZZUs,5047
 crawlo/core/__init__.py,sha256=PnFyJdVNHBoPmV1sW0AHQXijeoSTQ8cMYrbNM1JK8kA,41
-crawlo/core/engine.py,sha256=NIWbfRL_Lzjl6yYQ6LSfRJmDHBxjtk6x3nOWrnzaHME,14569
+crawlo/core/engine.py,sha256=Hy0K_g9My6aQ3CPkxAcCiPsumdwh4O8qRhmFlNoErd4,14496
 crawlo/core/processor.py,sha256=qmCqAeqhwYu-UE86evYesaGt9qpuSIfH-ZIZKcXFCZc,1140
 crawlo/core/scheduler.py,sha256=D-YzXVvnP6DEkovmz9hThhzIe2UgRrQLNt9pJCPEPwY,12593
 crawlo/data/__init__.py,sha256=8MgDxcMhx-emFARcLAw_ODOZNz0neYBcx7kEbzothd8,116
@@ -37,7 +37,7 @@ crawlo/extension/__init__.py,sha256=FbOwJ4jh60xCbSh7P9CUGJsGAC-VH4MyOtCftRMlxbk,
 crawlo/extension/health_check.py,sha256=0GveZgUtFwjYEKlm3qbwIvCmb4FR0qrIKc8cEF1yQV8,5516
 crawlo/extension/log_interval.py,sha256=VCIeNqXcWDnxj4m6l77cjqgRzV8LfsPMb22X0Xc1Vwc,2417
 crawlo/extension/log_stats.py,sha256=vrChs3bj_Dvay3kxxkBOp4-w0K-IG-2XZ0PoSUahTPs,2908
-crawlo/extension/logging_extension.py,sha256=ET6VAu1J2qNMz4NnG1G3zQLRhbsvV7l6xRIuQLE6DaE,1626
+crawlo/extension/logging_extension.py,sha256=RfL1wI4nz-1Xtg420Ktp7RXnOPnZSHwO0Zpg1w4fO4M,1726
 crawlo/extension/memory_monitor.py,sha256=4aszl3C0GMQbqFhGZjZq5iQuXQR1sOz06VHjjEHgkyE,4290
 crawlo/extension/performance_profiler.py,sha256=EPiNuXuPPDU0Jtgy8arYHpr_8ASK13cCI2BytdJnu_I,4899
 crawlo/extension/request_recorder.py,sha256=RC23yzXClnVv9j2ljQvjBkUfWznfnDHsrQejKhE9y5E,4074
@@ -51,7 +51,7 @@ crawlo/items/items.py,sha256=e-3nXI9ckD64vcDxxQiAU6ufbtJMs09gbZQcYjxgwHY,3374
 crawlo/middleware/__init__.py,sha256=ldaGFNbiJnK9Fx12Vdf9fDNfzXxoETtShp5r-vodtw0,549
 crawlo/middleware/default_header.py,sha256=wQ7BrUHd-hRosFoKsReV9hwNNr_jwK6V0ZfxL6MOGrk,5032
 crawlo/middleware/download_delay.py,sha256=zt9R5g2HWErWA_MAOnGcw_D8l6HD769Kyaw-Hv-vcTc,3438
-crawlo/middleware/middleware_manager.py,sha256=_JnJ50u3u4zFKkNZDae8M8FHcK-V2LkbcX7DkBdtJ2M,6279
+crawlo/middleware/middleware_manager.py,sha256=9Sj9rrWK6R9NZq9eT38sWRGuBKLKfjSgEAxu-5NCWgU,6278
 crawlo/middleware/offsite.py,sha256=b3BMwNKGR41YGJGHt1S0H7yXujEkztVvonUQGO05hoM,4026
 crawlo/middleware/proxy.py,sha256=NquB6tqHAgHs3-2_1_5220kJYfjNG5JyHRJyo_2j4wo,15636
 crawlo/middleware/request_ignore.py,sha256=xcyZ1c7r_HhbzR3r9pfjsLGW7L7FBVeYvlNt8cpP2wY,2577
@@ -71,7 +71,7 @@ crawlo/pipelines/json_pipeline.py,sha256=vlu1nqbD2mtqtExt9cL5nibx1CwJM1RNqd4WGjZ
 crawlo/pipelines/memory_dedup_pipeline.py,sha256=oIksbIrmSw9s9jMh6JJMfVbv6hzseVMV_g9S8UHQUP4,3837
 crawlo/pipelines/mongo_pipeline.py,sha256=k7gNqAO-g2MtIfArphC6z5ZzkKVRkBKcv-2ImziPFA0,5706
 crawlo/pipelines/mysql_pipeline.py,sha256=G2DMhdh0ihBBolIul4YVTDz2JbrZGJauDtWF-gqRW0w,13473
-crawlo/pipelines/pipeline_manager.py,sha256=DuIRl6FYbrKKUCrtSHnq4pnRImpZPYtg1YRcgUd1xTk,2425
+crawlo/pipelines/pipeline_manager.py,sha256=vCgfbhgsKMLm_7jCnr3cE5GemIYkG9u4oF8u4Ta_7so,3013
 crawlo/pipelines/redis_dedup_pipeline.py,sha256=POYRiWAOp1pqDW9iTPJ8h3VcpLALeLrpw74MvJJqPiM,6342
 crawlo/queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 crawlo/queue/pqueue.py,sha256=yzF900ko2RReVNZtfk_VV3PzSXEUALI6SHf97geuu6s,1058
@@ -79,7 +79,7 @@ crawlo/queue/queue_manager.py,sha256=XqS_oVbNQJWdtokOuDDPK-FzMrVdnZ3UKp1MF_DMJww
 crawlo/queue/redis_priority_queue.py,sha256=k1OChSMRovSMkbbJ9388axfhpYeMevuJTe-3N1oYhbA,13126
 crawlo/settings/__init__.py,sha256=xsukVKn_h2Hopm1Nj-bXkhbfyS62QTTvJi7fhZUwR9M,123
 crawlo/settings/default_settings.py,sha256=98URrj6QBrx_pmJ1yvK-MSAW8VrZ-pl0FfiZEHV0ZnI,9183
-crawlo/settings/setting_manager.py,sha256=55iJJambnFliZxwhNd7BtlgCV6eviuPRBrrIgyrmZgw,5659
+crawlo/settings/setting_manager.py,sha256=V3nVJEPtusadoz5eILXFeNyDXX1u_MgIiKIFIWVDY1s,6189
 crawlo/spider/__init__.py,sha256=ZnSAL9PXLZSIH-Jdv-P6RuWmQUdukr8KPLQK6SXZZaU,20435
 crawlo/templates/crawlo.cfg.tmpl,sha256=9BAmwEibS5Tvy6HIcGXPb0BGeuesmibebmTW0iAEkmo,230
 crawlo/templates/run.py.tmpl,sha256=v_g-LQMYJ6pC8TZgyWj0yB2yTTKrwy9lEJufAYCXyxY,1228
@@ -117,7 +117,7 @@ crawlo/utils/error_handler.py,sha256=q6NqHxjYrKdswfmhshMYMmfBIr0M2YWPYxts4ScHl4Y
 crawlo/utils/func_tools.py,sha256=WUZEGpWMuDDX7g-QySM7iaiC74erW2SSkZoUvDw1NjM,2369
 crawlo/utils/large_scale_config.py,sha256=j7wQ5ty7pQlBRygw2vhRJ7OI19RYBZKPfYMP3WeF2WI,8154
 crawlo/utils/large_scale_helper.py,sha256=Kxdy3WMuqjzQTyCc6z4xEYxXDi4xnYKJzsVwaBYZrrg,12108
-crawlo/utils/log.py,sha256=AcRx6_9U6fY-6GEZP77Il2o3FZZm5Cx0MFubIK9R3oI,7501
+crawlo/utils/log.py,sha256=xZe3UU78yr10lK0hxALBQB0Uv9cXShOPPksoe5n_qKI,5229
 crawlo/utils/performance_monitor.py,sha256=Q9xxuXBIfFoig_U-FQPOUuPAh1axO3MzYgpielDyku0,9547
 crawlo/utils/queue_helper.py,sha256=xpUUTOqlU1xz2Pb9NKAVGo3AfAO-7Xvx8Lm1q65Dgck,4743
 crawlo/utils/redis_connection_pool.py,sha256=amGjhaKpodMrw9X56qxZ6f3OTZhjrI89sSVGqgwAQGU,11050
@@ -212,8 +212,8 @@ tests/test_tools.py,sha256=9t9FXZ61MfdB70nck9NYzCq97yd3SLVlLiMybEAlClk,5345
 tests/test_user_agents.py,sha256=rUotyuE2iJDi2LQBrUh980U-dAMTs4ARPMJxICOoQFY,3231
 tests/tools_example.py,sha256=MtIypR-OFiWwi-skurwmq4fM0cGTt-GUX4hSekYs7BY,7739
 tests/verify_distributed.py,sha256=krnYYA5Qx9xXDMWc9YF5DxPSplGvawDg2n0l-3CAqoM,3928
-crawlo-1.2.9.dist-info/METADATA,sha256=wyjBzrx9hUZzlt0r1JPu33ouyrm_iJtphQAi2THb7LA,26298
-crawlo-1.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-crawlo-1.2.9.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
-crawlo-1.2.9.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
-crawlo-1.2.9.dist-info/RECORD,,
+crawlo-1.3.0.dist-info/METADATA,sha256=5BRT0EE3J1yUtWZ0l_pZqEWxTgGA1p3laxJjTSu7980,26298
+crawlo-1.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+crawlo-1.3.0.dist-info/entry_points.txt,sha256=5HoVoTSPxI8SCa5B7pQYxLSrkOdiunyO9tqNsLMv52g,43
+crawlo-1.3.0.dist-info/top_level.txt,sha256=keG_67pbZ_wZL2dmDRA9RMaNHTaV_x_oxZ9DKNgwvR0,22
+crawlo-1.3.0.dist-info/RECORD,,