crawlo 1.2.9__tar.gz → 1.3.0__tar.gz

{crawlo-1.2.9/crawlo.egg-info → crawlo-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.2.9
+Version: 1.3.0
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

crawlo-1.3.0/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = '1.3.0'

{crawlo-1.2.9 → crawlo-1.3.0}/crawlo/commands/run.py

@@ -5,26 +5,27 @@
 # @Author  : crawl-coder
 # @Desc    : 命令行入口：crawlo run <spider_name>|all，用于运行指定爬虫。
 """
+import os
 import sys
 import asyncio
 import configparser
-import os
-from pathlib import Path
 from importlib import import_module
 
+from rich import box
 from rich.console import Console
 from rich.panel import Panel
+from rich.progress import Progress, SpinnerColumn, TextColumn
 from rich.table import Table
 from rich.text import Text
-from rich import box
-from rich.progress import Progress, SpinnerColumn, TextColumn
 
+from crawlo.commands.stats import record_stats
 from crawlo.crawler import CrawlerProcess
-from crawlo.utils.log import get_logger
 from crawlo.project import get_settings, _find_project_root
-from crawlo.commands.stats import record_stats
+# 使用自定义日志系统
+from crawlo.utils.log import get_logger
 
 logger = get_logger(__name__)
+
 console = Console()
 
 
@@ -77,6 +78,9 @@ def main(args):
     用法:
         crawlo run <spider_name>|all [--json] [--no-stats]
     """
+    # 添加调试信息
+    logger.debug("DEBUG: 进入main函数")
+    
     if len(args) < 1:
         console.print("[bold red]用法:[/bold red] [blue]crawlo run[/blue] <爬虫名称>|all [bold yellow][--json] [--no-stats][/bold yellow]")
         console.print("示例:")
@@ -187,21 +191,7 @@ def main(args):
                     return 1
 
             # 显示即将运行的爬虫列表
-            table = Table(
-                title=f"启动全部 {len(spider_names)} 个爬虫",
-                box=box.ROUNDED,
-                show_header=True,
-                header_style="bold magenta"
-            )
-            table.add_column("名称", style="cyan")
-            table.add_column("类名", style="green")
-
-            for name in sorted(spider_names):
-                cls = process.get_spider_class(name)
-                table.add_row(name, cls.__name__)
-
-            console.print(table)
-            console.print()
+            # 根据用户要求，不再显示详细的爬虫列表信息
 
             # 注册 stats 记录（除非 --no-stats）
             if not no_stats:
@@ -260,20 +250,21 @@ def main(args):
         spider_class = process.get_spider_class(spider_name)
 
         # 显示启动信息
-        if not show_json:
-            info_table = Table(
-                title=f"启动爬虫: [bold cyan]{spider_name}[/bold cyan]",
-                box=box.SIMPLE,
-                show_header=False,
-                title_style="bold green"
-            )
-            info_table.add_column("Key", style="yellow")
-            info_table.add_column("Value", style="cyan")
-            info_table.add_row("Project", project_package)
-            info_table.add_row("Class", spider_class.__name__)
-            info_table.add_row("Module", spider_class.__module__)
-            console.print(info_table)
-            console.print()
+        # 根据用户要求，不再显示项目启动信息
+        # if not show_json:
+        #     info_table = Table(
+        #         title=f"启动爬虫: [bold cyan]{spider_name}[/bold cyan]",
+        #         box=box.SIMPLE,
+        #         show_header=False,
+        #         title_style="bold green"
+        #     )
+        #     info_table.add_column("Key", style="yellow")
+        #     info_table.add_column("Value", style="cyan")
+        #     info_table.add_row("Project", project_package)
+        #     info_table.add_row("Class", spider_class.__name__)
+        #     info_table.add_row("Module", spider_class.__module__)
+        #     console.print(info_table)
+        #     console.print()
 
         # 注册 stats 记录
         if not no_stats:

{crawlo-1.2.9 → crawlo-1.3.0}/crawlo/core/engine.py

@@ -75,8 +75,7 @@ class Engine(object):
             version = '1.0.0'
         # Change INFO level log to DEBUG level to avoid duplication with CrawlerProcess startup log
         self.logger.debug(
-            f"Crawlo Started version {version} . "
-            # f"(project name : {self.settings.get('PROJECT_NAME')})"
+            f"Crawlo Started version {version}"
         )
 
     async def start_spider(self, spider):

{crawlo-1.2.9 → crawlo-1.3.0}/crawlo/crawler.py

@@ -21,7 +21,7 @@ Example Usage:
     # Single crawler run
     crawler = Crawler(MySpider, settings)
     await crawler.crawl()
-    
+
     # Multi-crawler concurrent management
     process = CrawlerProcess()
     await process.crawl([Spider1, Spider2])
@@ -34,7 +34,6 @@ import threading
 from typing import Type, Optional, Set, List, Union, Dict, Any
 from .spider import Spider, get_global_spider_registry
 from .core.engine import Engine
-from .utils.log import get_logger
 from .subscriber import Subscriber
 from .extension import ExtensionManager
 from .stats_collector import StatsCollector
@@ -42,16 +41,9 @@ from .event import spider_opened, spider_closed
 from .settings.setting_manager import SettingManager
 from crawlo.project import merge_settings, get_settings
 
-# 延迟初始化logger，在需要时通过get_logger获取
-logger = None
-
-
-def _get_logger():
-    """延迟获取logger实例，确保在配置加载后创建"""
-    global logger
-    if logger is None:
-        logger = get_logger(__name__)
-    return logger
+# 使用自定义日志系统
+from crawlo.utils.log import get_logger
+logger = get_logger(__name__)
 
 
 class CrawlerContext:
@@ -110,7 +102,7 @@ class CrawlerContext:
 class Crawler:
     """
     Single crawler runtime instance, managing Spider and engine lifecycle
-    
+
     Provides functionality:
     - Spider lifecycle management (initialization, running, closing)
     - Engine component coordination management
@@ -148,7 +140,7 @@ class Crawler:
     async def crawl(self):
         """
         Start the crawler core process
-        
+
         Includes the following stages:
         1. Initialization stage: Create all components
         2. Validation stage: Check configuration and state
@@ -190,12 +182,12 @@ class Crawler:
             # Update context status
             self.context.increment_completed()
 
-            _get_logger().info(f"Spider {self.spider.name} completed, took {self._get_total_duration():.2f} seconds")
+            logger.info(f"Spider {self.spider.name} completed, took {self._get_total_duration():.2f} seconds")
 
         except Exception as e:
             self._performance_metrics['error_count'] += 1
             self.context.increment_failed(str(e))
-            _get_logger().error(f"Spider {getattr(self.spider, 'name', 'Unknown')} failed to run: {e}", exc_info=True)
+            logger.error(f"Spider {getattr(self.spider, 'name', 'Unknown')} failed to run: {e}", exc_info=True)
             raise
         finally:
             self.context.decrement_active()
@@ -213,7 +205,7 @@ class Crawler:
         else:
             spider_name = 'Unknown'
 
-        _get_logger().info(f"Starting running {spider_name}")
+        logger.info(f"Starting running {spider_name}")
 
     def _validate_crawler_state(self):
         """
@@ -233,7 +225,7 @@ class Crawler:
         if not self.spider.name:
             raise ValueError("Spider name cannot be empty")
 
-        _get_logger().debug(f"Spider {self.spider.name} state validation passed")
+        logger.debug(f"Spider {self.spider.name} state validation passed")
 
     def _get_total_duration(self) -> float:
         """Get total runtime"""
@@ -247,7 +239,7 @@ class Crawler:
             if not self._closed:
                 await self.close()
         except Exception as e:
-            _get_logger().warning(f"Error cleaning up resources: {e}")
+            logger.warning(f"Error cleaning up resources: {e}")
 
     def get_performance_metrics(self) -> Dict[str, Any]:
         """Get performance metrics"""
@@ -267,7 +259,7 @@ class Crawler:
     def _create_spider(self) -> Spider:
         """
         Create and validate spider instance (enhanced version)
-        
+
         Performs the following validations:
         - Spider name must exist
         - start_requests method must be callable
@@ -300,7 +292,7 @@ class Crawler:
 
         # parse method check (warning instead of error)
         if not callable(getattr(spider, 'parse', None)):
-            _get_logger().warning(
+            logger.warning(
                 f"Spider '{spider.name}' does not define 'parse' method.\n"
                 f"Ensure all Requests specify a callback function, otherwise responses will be ignored."
             )
@@ -308,27 +300,29 @@ class Crawler:
         # Set spider configuration
         self._set_spider(spider)
 
-        _get_logger().debug(f"Spider '{spider.name}' initialized successfully")
+        logger.debug(f"Spider '{spider.name}' initialized successfully")
         return spider
 
     def _create_engine(self) -> Engine:
         """Create and initialize engine"""
         engine = Engine(self)
         engine.engine_start()
-        _get_logger().debug(f"Engine initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(f"Engine initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return engine
 
     def _create_stats(self) -> StatsCollector:
         """Create stats collector"""
         stats = StatsCollector(self)
-        _get_logger().debug(f"Stats collector initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(
+            f"Stats collector initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return stats
 
     def _create_extension(self) -> ExtensionManager:
         """Create extension manager"""
         # Modify extension manager creation method, delay initialization until needed
         extension = ExtensionManager.create_instance(self)
-        _get_logger().debug(f"Extension manager initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
+        logger.debug(
+            f"Extension manager initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
         return extension
 
     def _set_spider(self, spider: Spider):
@@ -343,12 +337,12 @@ class Crawler:
         # Merge spider custom configuration
         merge_settings(spider, self.settings)
 
-        _get_logger().debug(f"Spider '{spider.name}' configuration merged successfully")
+        logger.debug(f"Spider '{spider.name}' configuration merged successfully")
 
     async def close(self, reason='finished') -> None:
         """
         Close crawler and clean up resources (enhanced version)
-        
+
         Ensure closing only once and handle all cleanup operations
         """
         async with self._close_lock:
@@ -371,15 +365,15 @@ class Crawler:
                         from crawlo.commands.stats import record_stats
                         record_stats(self)
                     except ImportError:
-                        _get_logger().debug("Statistics recording module does not exist, skipping statistics recording")
+                        logger.debug("Statistics recording module does not exist, skipping statistics recording")
 
-                _get_logger().info(
+                logger.info(
                     f"Spider '{getattr(self.spider, 'name', 'Unknown')}' closed, "
                     f"reason: {reason}, took: {self._get_total_duration():.2f} seconds"
                 )
 
             except Exception as e:
-                _get_logger().error(f"Error closing crawler: {e}", exc_info=True)
+                logger.error(f"Error closing crawler: {e}", exc_info=True)
             finally:
                 # Ensure resource cleanup
                 await self._cleanup_resources()
@@ -413,13 +407,13 @@ class Crawler:
         if cleanup_tasks:
             await asyncio.gather(*cleanup_tasks, return_exceptions=True)
 
-        _get_logger().debug("Resource cleanup completed")
+        logger.debug("Resource cleanup completed")
 
 
 class CrawlerProcess:
     """
     Crawler process manager
-    
+
     Supported features:
     - Multi-crawler concurrent scheduling and resource management
     - Automatic module discovery and spider registration
@@ -428,15 +422,15 @@ class CrawlerProcess:
     - Real-time status monitoring and statistics
     - Error recovery and retry mechanism
     - Large-scale crawler optimization support
-    
+
     Usage example:
         # Basic usage
         process = CrawlerProcess()
         await process.crawl(MySpider)
-        
+
         # Multi-crawler concurrency
         await process.crawl([Spider1, Spider2, 'spider_name'])
-        
+
         # Custom concurrency
         process = CrawlerProcess(max_concurrency=8)
     """
@@ -563,7 +557,7 @@ class CrawlerProcess:
     def auto_discover(modules: List[str]):
         """
         Automatically import modules, trigger Spider class definition and registration (enhanced version)
-        
+
         Supports recursive scanning and error recovery
         """
         import importlib
@@ -617,7 +611,7 @@ class CrawlerProcess:
     async def crawl(self, spiders: Union[Type[Spider], str, List[Union[Type[Spider], str]]]):
         """
         Start one or more crawlers
-        
+
         Enhanced features:
         - Intelligent concurrency control
         - Real-time monitoring and statistics
@@ -639,7 +633,7 @@ class CrawlerProcess:
         await self.start_monitoring()
 
         try:
-            # Phase 3: Sort by class name to ensure predictable startup order
+            # Phase 3: Initialize context and monitoring
             spider_classes_to_run.sort(key=lambda cls: cls.__name__.lower())
 
             logger.debug(
@@ -738,7 +732,7 @@ class CrawlerProcess:
     ) -> List[Type[Spider]]:
         """
         Resolve input to spider class list
-        
+
         Supports various input formats and validates uniqueness
         """
         inputs = self._normalize_inputs(spiders_input)
@@ -762,7 +756,8 @@ class CrawlerProcess:
                 seen_spider_names.add(spider_name)
                 spider_classes.append(spider_cls)
 
-                logger.debug(f"Spider resolved successfully: {item} -> {spider_cls.__name__} (name='{spider_name}')")
+                logger.debug(
+                    f"Spider resolved successfully: {item} -> {spider_cls.__name__} (name='{spider_name}')")
 
             except Exception as e:
                 logger.error(f"Failed to resolve spider: {item} - {e}")
@@ -774,7 +769,7 @@ class CrawlerProcess:
     def _normalize_inputs(spiders_input) -> List[Union[Type[Spider], str]]:
         """
         Normalize input to list
-        
+
         Supports more input types and provides better error information
         """
         if isinstance(spiders_input, (type, str)):
@@ -793,7 +788,7 @@ class CrawlerProcess:
     def _resolve_spider_class(self, item: Union[Type[Spider], str]) -> Type[Spider]:
         """
         Resolve single input item to spider class
-        
+
         Provides better error prompts and debugging information
         """
         if isinstance(item, type) and issubclass(item, Spider):
@@ -820,7 +815,7 @@ class CrawlerProcess:
     async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
         """
         Spider running function limited by semaphore
-        
+
         Includes enhanced error handling and monitoring functionality
         """
         task = asyncio.current_task()
@@ -888,7 +883,7 @@ class CrawlerProcess:
     def _shutdown(self, _signum, _frame):
         """
         Graceful shutdown signal handling
-        
+
         Provides better shutdown experience and resource cleanup
         """
         signal_name = {signal.SIGINT: 'SIGINT', signal.SIGTERM: 'SIGTERM'}.get(_signum, str(_signum))
@@ -913,7 +908,7 @@ class CrawlerProcess:
     async def _wait_for_shutdown(self):
         """
         Wait for all active tasks to complete
-        
+
         Provides better shutdown time control and progress feedback
         """
         try:
@@ -967,15 +962,15 @@ class CrawlerProcess:
     def _get_default_settings(cls) -> SettingManager:
         """
         Load default configuration
-        
+
         Provides better error handling and fallback strategy
         """
         try:
             settings = get_settings()
-            _get_logger().debug("Default configuration loaded successfully")
+            logger.debug("Default configuration loaded successfully")
             return settings
         except Exception as e:
-            _get_logger().warning(f"Unable to load default configuration: {e}, using empty configuration")
+            logger.warning(f"Unable to load default configuration: {e}, using empty configuration")
             return SettingManager()
 
     def _log_startup_info(self):
@@ -990,7 +985,7 @@ class CrawlerProcess:
 
         # Build startup info log
         startup_info = [
-            f"Crawlo Framework Started v{version}"
+            f"Crawlo Framework Started {version}"
         ]
 
         # Get actual queue type
@@ -1018,7 +1013,7 @@ class CrawlerProcess:
         else:
             startup_info.append(f"Run Mode: {run_mode}")
 
-        # Print startup information
+        # Print startup information at INFO level
         for info in startup_info:
             logger.info(info)
 
@@ -1032,7 +1027,7 @@ def create_crawler_with_optimizations(
 ) -> Crawler:
     """
     Create an optimized crawler instance
-    
+
     :param spider_cls: Spider class
     :param settings: Settings manager
     :param optimization_kwargs: Optimization parameters
@@ -1056,7 +1051,7 @@ def create_process_with_large_scale_config(
 ) -> CrawlerProcess:
     """
     Create a process manager that supports large-scale optimization
-    
+
     :param config_type: Configuration type ('conservative', 'balanced', 'aggressive', 'memory_optimized')
     :param concurrency: Concurrency count
     :param kwargs: Other parameters
@@ -1100,4 +1095,4 @@ __all__ = [
     'CrawlerContext',
     'create_crawler_with_optimizations',
     'create_process_with_large_scale_config'
-]
+]

{crawlo-1.2.9 → crawlo-1.3.0}/crawlo/extension/logging_extension.py

@@ -1,8 +1,10 @@
 from typing import Any
 from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import get_logger
 from crawlo.utils.log import LoggerManager
 
+# 使用全局logger，避免每个模块都创建自己的延迟初始化函数
+logger = LoggerManager.get_logger(__name__)
+
 
 class CustomLoggerExtension:
     """
@@ -32,7 +34,7 @@ class CustomLoggerExtension:
         return cls(crawler.settings)
 
     def spider_opened(self, spider: Any) -> None:
-        logger = get_logger(__name__)
+        logger = LoggerManager.get_logger(__name__)
         try:
             logger.info(
                 f"CustomLoggerExtension: Logging initialized. "

{crawlo-1.2.9 → crawlo-1.3.0}/crawlo/middleware/middleware_manager.py

@@ -133,4 +133,4 @@ class MiddlewareManager:
     def _validate_middleware_method(method_name, middleware) -> bool:
         method = getattr(type(middleware), method_name)
         base_method = getattr(BaseMiddleware, method_name)
-        return False if method == base_method else True
+        return False if method == base_method else True