crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/crawler.py

@@ -1,1167 +1,639 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-Crawlo Crawler Module
-====================
-Provides crawler process management and runtime core functionality.
-
-Core Components:
-- Crawler: Single crawler runtime instance, managing Spider and engine lifecycle
-- CrawlerProcess: Crawler process manager, supporting multi-crawler concurrent scheduling and resource management
-
-Features:
-- Intelligent concurrency control and resource management
-- Graceful shutdown and signal handling
-- Statistics monitoring and performance tracking
-- Automatic module discovery and registration
-- Error recovery and retry mechanism
-- Large-scale crawler optimization support
-
-Example Usage:
-    # Single crawler run
-    crawler = Crawler(MySpider, settings)
-    await crawler.crawl()
-
-    # Multi-crawler concurrent management
-    process = CrawlerProcess()
-    await process.crawl([Spider1, Spider2])
-"""
-from __future__ import annotations
-import asyncio
-import signal
-import time
-import threading
-from typing import Type, Optional, Set, List, Union, Dict, Any
-from .spider import Spider, get_global_spider_registry
-from .core.engine import Engine
-from .subscriber import Subscriber
-from .extension import ExtensionManager
-from crawlo.utils.log import get_logger
-from .stats_collector import StatsCollector
-from .event import spider_opened, spider_closed
-from .settings.setting_manager import SettingManager
-from crawlo.project import merge_settings, get_settings
-
-logger = get_logger(__name__)
-
-
-class CrawlerContext:
-    """
-    Crawler context manager
-    Provides shared state and resource management
-    """
-
-    def __init__(self):
-        self.start_time = time.time()
-        self.total_crawlers = 0
-        self.active_crawlers = 0
-        self.completed_crawlers = 0
-        self.failed_crawlers = 0
-        self.error_log = []
-        self._lock = threading.RLock()
-
-    def increment_total(self):
-        with self._lock:
-            self.total_crawlers += 1
-
-    def increment_active(self):
-        with self._lock:
-            self.active_crawlers += 1
-
-    def decrement_active(self):
-        with self._lock:
-            self.active_crawlers -= 1
-
-    def increment_completed(self):
-        with self._lock:
-            self.completed_crawlers += 1
-
-    def increment_failed(self, error: str):
-        with self._lock:
-            self.failed_crawlers += 1
-            self.error_log.append({
-                'timestamp': time.time(),
-                'error': error
-            })
-
-    def get_stats(self) -> Dict[str, Any]:
-        with self._lock:
-            duration = time.time() - self.start_time
-            return {
-                'total_crawlers': self.total_crawlers,
-                'active_crawlers': self.active_crawlers,
-                'completed_crawlers': self.completed_crawlers,
-                'failed_crawlers': self.failed_crawlers,
-                'success_rate': (self.completed_crawlers / max(1, self.total_crawlers)) * 100,
-                'duration_seconds': round(duration, 2),
-                'error_count': len(self.error_log)
-            }
-
-
-class Crawler:
-    """
-    Single crawler runtime instance, managing Spider and engine lifecycle
-
-    Provides functionality:
-    - Spider lifecycle management (initialization, running, closing)
-    - Engine component coordination management
-    - Configuration merging and validation
-    - Statistics data collection
-    - Extension management
-    - Exception handling and cleanup
-    """
-
-    def __init__(
-            self,
-            spider_cls: Type[Spider],
-            settings: SettingManager,
-            context: Optional[CrawlerContext] = None
-    ):
-        self.spider_cls = spider_cls
-        self.spider: Optional[Spider] = None
-        self.engine: Optional[Engine] = None
-        self.stats: Optional[StatsCollector] = None
-        self.subscriber: Optional[Subscriber] = None
-        self.extension: Optional[ExtensionManager] = None
-        self.settings: SettingManager = settings.copy()
-        self.context = context or CrawlerContext()
-
-        # State management
-        self._closed = False
-        self._close_lock = asyncio.Lock()
-        self._start_time = None
-        self._end_time = None
-
-        # Performance monitoring
-        self._performance_metrics = {
-            'initialization_time': 0,
-            'crawl_duration': 0,
-            'memory_peak': 0,
-            'request_count': 0,
-            'error_count': 0
-        }
-
-        # Initialize components
-        self.subscriber = self._create_subscriber()
-        self.spider = self._create_spider()
-        self.engine = self._create_engine()
-        self.stats = self._create_stats()
-        # Note: Do not initialize extension manager here, let it initialize in the engine
-
-        # Validate crawler state
-        self._validate_crawler_state()
-
-        # 将启动信息的打印移到crawl方法中，避免在CrawlerProcess中重复打印
-        # self._log_startup_info()
-        
-        # 将启动爬虫名称的日志移到这里，确保在日志系统配置之后打印
-        # logger.info(f"Starting running {self.spider.name}")
-
-    async def crawl(self):
-        """
-        Start the crawler core process
-
-        Includes the following stages:
-        1. Initialization stage: Create all components
-        2. Validation stage: Check configuration and state
-        3. Running stage: Start the crawler engine
-        4. Cleanup stage: Resource release
-        """
-        init_start = time.time()
-        self._start_time = init_start
-
-        try:
-            # Update context status
-            self.context.increment_active()
-
-            # Phase 1: Initialize components
-            # Adjust component initialization order to ensure log output order meets requirements
-            self.subscriber = self._create_subscriber()
-            self.spider = self._create_spider()
-            self.engine = self._create_engine()
-            self.stats = self._create_stats()
-            # Note: Do not initialize extension manager here, let it initialize in the engine
-
-            # Record initialization time
-            self._performance_metrics['initialization_time'] = time.time() - init_start
-
-            # Phase 2: Validate state
-            self._validate_crawler_state()
-
-            # Phase 3: Display runtime configuration summary
-            self._log_runtime_summary()
-
-            # Phase 4: Start crawler
-            crawl_start = time.time()
-            await self.engine.start_spider(self.spider)
-
-            # Record crawl time
-            self._performance_metrics['crawl_duration'] = time.time() - crawl_start
-            self._end_time = time.time()
-
-            # Update context status
-            self.context.increment_completed()
-
-            logger.info(f"Spider {self.spider.name} completed, took {self._get_total_duration():.2f} seconds")
-
-        except Exception as e:
-            self._performance_metrics['error_count'] += 1
-            self.context.increment_failed(str(e))
-            logger.error(f"Spider {getattr(self.spider, 'name', 'Unknown')} failed to run: {e}", exc_info=True)
-            raise
-        finally:
-            self.context.decrement_active()
-            # Ensure resource cleanup
-            await self._ensure_cleanup()
-
-    def _log_runtime_summary(self):
-        """Log runtime configuration summary"""
-        # Get spider name
-        spider_name = getattr(self.spider, 'name', 'Unknown')
-
-        # Ensure spider name is a string and strip leading/trailing whitespace
-        if spider_name:
-            spider_name = str(spider_name).strip()
-        else:
-            spider_name = 'Unknown'
-
-        logger.info(f"Starting running {spider_name}")
-
-    def _validate_crawler_state(self):
-        """
-        Validate crawler state and configuration
-        Ensure all necessary components are properly initialized
-        """
-        if not self.spider:
-            raise RuntimeError("Spider instance not initialized")
-        if not self.engine:
-            raise RuntimeError("Engine not initialized")
-        if not self.stats:
-            raise RuntimeError("Stats collector not initialized")
-        if not self.subscriber:
-            raise RuntimeError("Event subscriber not initialized")
-
-        # Check key configuration
-        if not self.spider.name:
-            raise ValueError("Spider name cannot be empty")
-
-        logger.debug(f"Spider {self.spider.name} state validation passed")
-
-    def _get_total_duration(self) -> float:
-        """Get total runtime"""
-        if self._start_time and self._end_time:
-            return self._end_time - self._start_time
-        return 0.0
-
-    def _log_startup_info(self):
-        """Print startup information, including run mode and key configuration checks"""
-        # Get run mode
-        run_mode = self.settings.get('RUN_MODE', 'standalone')
-
-        # Get version number
-        version = self.settings.get('VERSION', '1.0.0')
-        if not version or version == 'None':
-            version = '1.0.0'
-
-        # Print framework start info
-        logger.info(f"Crawlo Framework Started {version}")
-
-        # Add mode info if available
-        mode_info = self.settings.get('_mode_info')
-        if mode_info:
-            logger.info(mode_info)
-        else:
-            # 如果没有_mode_info，添加默认信息
-            logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
-
-        # Get actual queue type
-        queue_type = self.settings.get('QUEUE_TYPE', 'memory')
-
-        # Display information based on run mode and queue type combination
-        if run_mode == 'distributed':
-            logger.info("Run Mode: distributed")
-            logger.info("Distributed Mode - Multi-node collaboration supported")
-            # Show Redis configuration
-            redis_host = self.settings.get('REDIS_HOST', 'localhost')
-            redis_port = self.settings.get('REDIS_PORT', 6379)
-            logger.info(f"Redis Address: {redis_host}:{redis_port}")
-        elif run_mode == 'standalone':
-            if queue_type == 'redis':
-                logger.info("Run Mode: standalone+redis")
-                # Show Redis configuration
-                redis_host = self.settings.get('REDIS_HOST', 'localhost')
-                redis_port = self.settings.get('REDIS_PORT', 6379)
-                logger.info(f"Redis Address: {redis_host}:{redis_port}")
-            elif queue_type == 'auto':
-                logger.info("Run Mode: standalone+auto")
-            else:  # memory
-                logger.info("Run Mode: standalone")
-        else:
-            logger.info(f"Run Mode: {run_mode}")
-
-    async def _ensure_cleanup(self):
-        """Ensure resource cleanup"""
-        try:
-            if not self._closed:
-                await self.close()
-        except Exception as e:
-            logger.warning(f"Error cleaning up resources: {e}")
-
-    def get_performance_metrics(self) -> Dict[str, Any]:
-        """Get performance metrics"""
-        metrics = self._performance_metrics.copy()
-        metrics['total_duration'] = self._get_total_duration()
-        if self.stats:
-            # Add statistics data
-            stats_data = getattr(self.stats, 'get_stats', lambda: {})()
-            metrics.update(stats_data)
-        return metrics
-
-    @staticmethod
-    def _create_subscriber() -> Subscriber:
-        """Create event subscriber"""
-        return Subscriber()
-
-    def _create_spider(self) -> Spider:
-        """
-        Create and validate spider instance (enhanced version)
-
-        Performs the following validations:
-        - Spider name must exist
-        - start_requests method must be callable
-        - start_urls cannot be a string
-        - parse method is recommended to exist
-        """
-        spider = self.spider_cls.create_instance(self)
-
-        # Required attribute check
-        if not getattr(spider, 'name', None):
-            raise AttributeError(
-                f"Spider class '{self.spider_cls.__name__}' must define 'name' attribute.\n"
-                f"Example: name = 'my_spider'"
-            )
-
-        if not callable(getattr(spider, 'start_requests', None)):
-            raise AttributeError(
-                f"Spider '{spider.name}' must implement a callable 'start_requests' method.\n"
-                f"Example: def start_requests(self): yield Request(url='...')"
-            )
-
-        # start_urls type check
-        start_urls = getattr(spider, 'start_urls', [])
-        if isinstance(start_urls, str):
-            raise TypeError(
-                f"Spider '{spider.name}' 'start_urls' must be a list or tuple, not a string.\n"
-                f"Correct: start_urls = ['http://example.com']\n"
-                f"Incorrect: start_urls = 'http://example.com'"
-            )
-
-        # parse method check (warning instead of error)
-        if not callable(getattr(spider, 'parse', None)):
-            logger.warning(
-                f"Spider '{spider.name}' does not define 'parse' method.\n"
-                f"Ensure all Requests specify a callback function, otherwise responses will be ignored."
-            )
-
-        # Set spider configuration
-        self._set_spider(spider)
-
-        logger.debug(f"Spider '{spider.name}' initialized successfully")
-        return spider
-
-    def _create_engine(self) -> Engine:
-        """Create and initialize engine"""
-        engine = Engine(self)
-        engine.engine_start()
-        logger.debug(f"Engine initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
-        return engine
-
-    def _create_stats(self) -> StatsCollector:
-        """Create stats collector"""
-        stats = StatsCollector(self)
-        logger.debug(
-            f"Stats collector initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
-        return stats
-
-    def _create_extension(self) -> ExtensionManager:
-        """Create extension manager"""
-        # Modify extension manager creation method, delay initialization until needed
-        extension = ExtensionManager.create_instance(self)
-        logger.debug(
-            f"Extension manager initialized successfully, spider: {getattr(self.spider, 'name', 'Unknown')}")
-        return extension
-
-    def _set_spider(self, spider: Spider):
-        """
-        Set spider configuration and event subscription
-        Bind spider lifecycle events with subscriber
-        """
-        # Subscribe to spider lifecycle events
-        self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
-        self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
-
-        # Merge spider custom configuration
-        merge_settings(spider, self.settings)
-
-        logger.debug(f"Spider '{spider.name}' configuration merged successfully")
-
-    async def close(self, reason='finished') -> None:
-        """
-        Close crawler and clean up resources (enhanced version)
-
-        Ensure closing only once and handle all cleanup operations
-        """
-        async with self._close_lock:
-            if self._closed:
-                return
-
-            self._closed = True
-            self._end_time = time.time()
-
-            try:
-                # Notify spider close event
-                if self.subscriber:
-                    await self.subscriber.notify(spider_closed)
-
-                # Statistics data collection
-                if self.stats and self.spider:
-                    self.stats.close_spider(spider=self.spider, reason=reason)
-                    # Record statistics data
-                    try:
-                        from crawlo.commands.stats import record_stats
-                        record_stats(self)
-                    except ImportError:
-                        logger.debug("Statistics recording module does not exist, skipping statistics recording")
-
-                logger.info(
-                    f"Spider '{getattr(self.spider, 'name', 'Unknown')}' closed, "
-                    f"reason: {reason}, took: {self._get_total_duration():.2f} seconds"
-                )
-
-            except Exception as e:
-                logger.error(f"Error closing crawler: {e}", exc_info=True)
-            finally:
-                # Ensure resource cleanup
-                await self._cleanup_resources()
-
-    async def _cleanup_resources(self):
-        """Clean up all resources"""
-        cleanup_tasks = []
-
-        # Engine cleanup
-        if self.engine:
-            try:
-                cleanup_tasks.append(self.engine.close())
-            except AttributeError:
-                pass  # Engine has no close method
-
-        # Extension cleanup
-        if self.extension:
-            try:
-                cleanup_tasks.append(self.extension.close())
-            except AttributeError:
-                pass
-
-        # Stats collector cleanup
-        if self.stats:
-            try:
-                cleanup_tasks.append(self.stats.close())
-            except AttributeError:
-                pass
-
-        # Concurrently execute cleanup tasks
-        if cleanup_tasks:
-            await asyncio.gather(*cleanup_tasks, return_exceptions=True)
-
-        logger.debug("Resource cleanup completed")
-
-
-class CrawlerProcess:
-    """
-    Crawler process manager
-
-    Supported features:
-    - Multi-crawler concurrent scheduling and resource management
-    - Automatic module discovery and spider registration
-    - Intelligent concurrency control and load balancing
-    - Graceful shutdown and signal handling
-    - Real-time status monitoring and statistics
-    - Error recovery and retry mechanism
-    - Large-scale crawler optimization support
-
-    Usage example:
-        # Basic usage
-        process = CrawlerProcess()
-        await process.crawl(MySpider)
-
-        # Multi-crawler concurrency
-        await process.crawl([Spider1, Spider2, 'spider_name'])
-
-        # Custom concurrency
-        process = CrawlerProcess(max_concurrency=8)
-    """
-
-    def __init__(
-            self,
-            settings: Optional[SettingManager] = None,
-            max_concurrency: Optional[int] = None,
-            spider_modules: Optional[List[str]] = None,
-            enable_monitoring: bool = True
-    ):
-        # Basic configuration
-        self.settings: SettingManager = settings or self._get_default_settings()
-        self.crawlers: Set[Crawler] = set()
-        self._active_tasks: Set[asyncio.Task] = set()
-
-        # Context manager
-        self.context = CrawlerContext()
-
-        # Concurrency control configuration
-        self.max_concurrency: int = (
-                max_concurrency
-                or self.settings.get('MAX_RUNNING_SPIDERS')
-                or self.settings.get('CONCURRENCY', 3)
-        )
-        self.semaphore = asyncio.Semaphore(self.max_concurrency)
-
-        # Monitoring configuration
-        self.enable_monitoring = enable_monitoring
-        self._monitoring_task = None
-        self._shutdown_event = asyncio.Event()
-
-        # Automatically discover and import spider modules
-        if spider_modules:
-            self.auto_discover(spider_modules)
-
-        # Use snapshot of global registry (avoid subsequent import impact)
-        self._spider_registry: Dict[str, Type[Spider]] = get_global_spider_registry()
-
-        # Performance monitoring
-        self._performance_stats = {
-            'total_requests': 0,
-            'successful_requests': 0,
-            'failed_requests': 0,
-            'memory_usage_mb': 0,
-            'cpu_usage_percent': 0
-        }
-
-        # Register signal handlers
-        signal.signal(signal.SIGINT, self._shutdown)
-        signal.signal(signal.SIGTERM, self._shutdown)
-
-    def _log_startup_info(self):
-        """Print startup information, including run mode and key configuration checks"""
-        # Get run mode
-        run_mode = self.settings.get('RUN_MODE', 'standalone')
-
-        # Get version number
-        version = self.settings.get('VERSION', '1.0.0')
-        if not version or version == 'None':
-            version = '1.0.0'
-
-        # Print framework start info
-        logger.info(f"Crawlo Framework Started {version}")
-
-        # Add mode info if available
-        mode_info = self.settings.get('_mode_info')
-        if mode_info:
-            logger.info(mode_info)
-        else:
-            # 如果没有_mode_info，添加默认信息
-            logger.info("使用单机模式 - 简单快速，适合开发和中小规模爬取")
-
-        # Get actual queue type
-        queue_type = self.settings.get('QUEUE_TYPE', 'memory')
-
-        # Display information based on run mode and queue type combination
-        if run_mode == 'distributed':
-            logger.info("Run Mode: distributed")
-            logger.info("Distributed Mode - Multi-node collaboration supported")
-            # Show Redis configuration
-            redis_host = self.settings.get('REDIS_HOST', 'localhost')
-            redis_port = self.settings.get('REDIS_PORT', 6379)
-            logger.info(f"Redis Address: {redis_host}:{redis_port}")
-        elif run_mode == 'standalone':
-            if queue_type == 'redis':
-                logger.info("Run Mode: standalone+redis")
-                # Show Redis configuration
-                redis_host = self.settings.get('REDIS_HOST', 'localhost')
-                redis_port = self.settings.get('REDIS_PORT', 6379)
-                logger.info(f"Redis Address: {redis_host}:{redis_port}")
-            elif queue_type == 'auto':
-                logger.info("Run Mode: standalone+auto")
-            else:  # memory
-                logger.info("Run Mode: standalone")
-        else:
-            logger.info(f"Run Mode: {run_mode}")
-
-        logger.debug(
-            f"CrawlerProcess initialized successfully\n"
-            f"  - Max concurrent crawlers: {self.max_concurrency}\n"
-            f"  - Registered crawlers: {len(self._spider_registry)}\n"
-            f"  - Monitoring enabled: {self.enable_monitoring}"
-        )
-
-    async def start_monitoring(self):
-        """Start monitoring task"""
-        if not self.enable_monitoring:
-            return
-
-        self._monitoring_task = asyncio.create_task(self._monitor_loop())
-        logger.debug("Monitoring task started")
-
-    async def stop_monitoring(self):
-        """Stop monitoring task"""
-        if self._monitoring_task and not self._monitoring_task.done():
-            self._monitoring_task.cancel()
-            try:
-                await self._monitoring_task
-            except asyncio.CancelledError:
-                pass
-            logger.debug("Monitoring task stopped")
-
-    async def _monitor_loop(self):
-        """Monitoring loop, periodically collect and report status"""
-        try:
-            while not self._shutdown_event.is_set():
-                await self._collect_performance_stats()
-
-                # Output status every 30 seconds
-                stats = self.context.get_stats()
-                if stats['active_crawlers'] > 0:
-                    logger.debug(
-                        f"Crawler status: Active {stats['active_crawlers']}, "
-                        f"Completed {stats['completed_crawlers']}, "
-                        f"Failed {stats['failed_crawlers']}, "
-                        f"Success rate {stats['success_rate']:.1f}%"
-                    )
-
-                await asyncio.sleep(30)  # 30 second interval
-
-        except asyncio.CancelledError:
-            logger.debug("Monitoring loop cancelled")
-        except Exception as e:
-            logger.error(f"Monitoring loop error: {e}", exc_info=True)
-
-    async def _collect_performance_stats(self):
-        """Collect performance statistics data"""
-        try:
-            import psutil
-            import os
-
-            process = psutil.Process(os.getpid())
-            memory_info = process.memory_info()
-
-            self._performance_stats.update({
-                'memory_usage_mb': round(memory_info.rss / 1024 / 1024, 2),
-                'cpu_usage_percent': round(process.cpu_percent(), 2)
-            })
-
-        except ImportError:
-            # Skip performance monitoring when psutil is not available
-            pass
-        except Exception as e:
-            logger.debug(f"Failed to collect performance statistics: {e}")
-
-    @staticmethod
-    def auto_discover(modules: List[str]):
-        """
-        Automatically import modules, trigger Spider class definition and registration (enhanced version)
-
-        Supports recursive scanning and error recovery
-        """
-        import importlib
-        import pkgutil
-
-        discovered_count = 0
-        error_count = 0
-
-        for module_name in modules:
-            try:
-                module = importlib.import_module(module_name)
-
-                if hasattr(module, '__path__'):
-                    # Package module, recursive scanning
-                    for _, name, _ in pkgutil.walk_packages(module.__path__, module.__name__ + "."):
-                        try:
-                            importlib.import_module(name)
-                            discovered_count += 1
-                        except Exception as sub_e:
-                            error_count += 1
-                            logger.warning(f"Failed to import submodule {name}: {sub_e}")
-                else:
-                    # Single module
-                    importlib.import_module(module_name)
-                    discovered_count += 1
-
-                logger.debug(f"Module scanned: {module_name}")
-
-            except Exception as e:
-                error_count += 1
-                logger.error(f"Failed to scan module {module_name}: {e}", exc_info=True)
-
-        logger.debug(
-            f"Spider registration completed: {discovered_count} succeeded, {error_count} failed"
-        )
-
-    # === Public read-only interface: Avoid direct access to _spider_registry ===
-
-    def get_spider_names(self) -> List[str]:
-        """Get all registered spider names"""
-        return list(self._spider_registry.keys())
-
-    def get_spider_class(self, name: str) -> Optional[Type[Spider]]:
-        """Get spider class by name"""
-        return self._spider_registry.get(name)
-
-    def is_spider_registered(self, name: str) -> bool:
-        """Check if a name is registered"""
-        return name in self._spider_registry
-
-    async def crawl(self, spiders: Union[Type[Spider], str, List[Union[Type[Spider], str]]]):
-        """
-        Start one or more crawlers
-
-        Enhanced features:
-        - Intelligent concurrency control
-        - Real-time monitoring and statistics
-        - Error recovery and retry
-        - Graceful shutdown handling
-        """
-        # Phase 1: Preprocessing and validation
-        spider_classes_to_run = self._resolve_spiders_to_run(spiders)
-        total = len(spider_classes_to_run)
-
-        if total == 0:
-            raise ValueError("At least one spider class or name must be provided")
-
-        # 打印启动信息，确保在日志系统配置之后打印
-        # 在这里调用_log_startup_info，确保框架启动信息能正确输出到日志文件中
-        self._log_startup_info()
-
-        # Phase 2: Initialize context and monitoring
-        for _ in range(total):
-            self.context.increment_total()
-
-        # Start monitoring task
-        await self.start_monitoring()
-
-        try:
-            # Phase 3: Initialize context and monitoring
-            spider_classes_to_run.sort(key=lambda cls: cls.__name__.lower())
-
-            logger.debug(
-                f"Starting {total} crawlers\n"
-                f"  - Max concurrency: {self.max_concurrency}\n"
-                f"  - Spider list: {[cls.__name__ for cls in spider_classes_to_run]}"
-            )
-
-            # Phase 4: Stream start all crawler tasks
-            tasks = [
-                asyncio.create_task(
-                    self._run_spider_with_limit(spider_cls, index + 1, total),
-                    name=f"spider-{spider_cls.__name__}-{index + 1}"
-                )
-                for index, spider_cls in enumerate(spider_classes_to_run)
-            ]
-
-            # Phase 5: Wait for all tasks to complete (failures do not interrupt)
-            results = await asyncio.gather(*tasks, return_exceptions=True)
-
-            # Phase 6: Statistics exceptions and results
-            failed = [i for i, r in enumerate(results) if isinstance(r, Exception)]
-            successful = total - len(failed)
-
-            if failed:
-                failed_spiders = [spider_classes_to_run[i].__name__ for i in failed]
-                logger.error(
-                    f"Crawler execution result: {successful}/{total} succeeded, {len(failed)}/{total} failed\n"
-                    f"  - Failed crawlers: {failed_spiders}"
-                )
-
-                # Record detailed error information
-                for i in failed:
-                    error = results[i]
-                    logger.error(f"Spider {spider_classes_to_run[i].__name__} error details: {error}")
-            else:
-                logger.info(f"All {total} crawlers completed successfully!")
-
-            # Return statistics results
-            return {
-                'total': total,
-                'successful': successful,
-                'failed': len(failed),
-                'success_rate': (successful / total) * 100 if total > 0 else 0,
-                'context_stats': self.context.get_stats()
-            }
-
-        finally:
-            # Phase 7: Cleanup and shutdown
-            await self.stop_monitoring()
-            await self._cleanup_process()
-
-    async def _cleanup_process(self):
-        """Clean up process resources"""
-        try:
-            # Wait for all active crawlers to complete
-            if self.crawlers:
-                close_tasks = [crawler.close() for crawler in self.crawlers]
-                await asyncio.gather(*close_tasks, return_exceptions=True)
-                self.crawlers.clear()
-
-            # Clean up active tasks
-            if self._active_tasks:
-                for task in list(self._active_tasks):
-                    if not task.done():
-                        task.cancel()
-                await asyncio.gather(*self._active_tasks, return_exceptions=True)
-                self._active_tasks.clear()
-
-            logger.debug("Process resources cleanup completed")
-
-        except Exception as e:
-            logger.error(f"Error cleaning up process resources: {e}", exc_info=True)
-
-    def get_process_stats(self) -> Dict[str, Any]:
-        """Get process statistics information"""
-        context_stats = self.context.get_stats()
-
-        return {
-            'context': context_stats,
-            'performance': self._performance_stats.copy(),
-            'crawlers': {
-                'total_registered': len(self._spider_registry),
-                'active_crawlers': len(self.crawlers),
-                'max_concurrency': self.max_concurrency
-            },
-            'registry': {
-                'spider_names': list(self._spider_registry.keys()),
-                'spider_classes': [cls.__name__ for cls in self._spider_registry.values()]
-            }
-        }
-
-    def _resolve_spiders_to_run(
-            self,
-            spiders_input: Union[Type[Spider], str, List[Union[Type[Spider], str]]]
-    ) -> List[Type[Spider]]:
-        """
-        Resolve input to spider class list
-
-        Supports various input formats and validates uniqueness
-        """
-        inputs = self._normalize_inputs(spiders_input)
-        seen_spider_names: Set[str] = set()
-        spider_classes: List[Type[Spider]] = []
-
-        for item in inputs:
-            try:
-                spider_cls = self._resolve_spider_class(item)
-                spider_name = getattr(spider_cls, 'name', None)
-
-                if not spider_name:
-                    raise ValueError(f"Spider class {spider_cls.__name__} missing 'name' attribute")
-
-                if spider_name in seen_spider_names:
-                    raise ValueError(
-                        f"Duplicate spider name '{spider_name}' in this run.\n"
-                        f"Ensure each spider's name attribute is unique in this run."
-                    )
-
-                seen_spider_names.add(spider_name)
-                spider_classes.append(spider_cls)
-
-                logger.debug(
-                    f"Spider resolved successfully: {item} -> {spider_cls.__name__} (name='{spider_name}')")
-
-            except Exception as e:
-                logger.error(f"Failed to resolve spider: {item} - {e}")
-                raise
-
-        return spider_classes
-
-    @staticmethod
-    def _normalize_inputs(spiders_input) -> List[Union[Type[Spider], str]]:
-        """
-        Normalize input to list
-
-        Supports more input types and provides better error information
-        """
-        if isinstance(spiders_input, (type, str)):
-            return [spiders_input]
-        elif isinstance(spiders_input, (list, tuple, set)):
-            spider_list = list(spiders_input)
-            if not spider_list:
-                raise ValueError("Spider list cannot be empty")
-            return spider_list
-        else:
-            raise TypeError(
-                f"Unsupported spiders parameter type: {type(spiders_input)}\n"
-                f"Supported types: Spider class, name string, or their list/tuple/set"
-            )
-
-    def _resolve_spider_class(self, item: Union[Type[Spider], str]) -> Type[Spider]:
-        """
-        Resolve single input item to spider class
-
-        Provides better error prompts and debugging information
-        """
-        if isinstance(item, type) and issubclass(item, Spider):
-            # Direct Spider class
-            return item
-        elif isinstance(item, str):
-            # String name, need to look up registry
-            spider_cls = self._spider_registry.get(item)
-            if not spider_cls:
-                available_spiders = list(self._spider_registry.keys())
-                raise ValueError(
-                    f"Spider named '{item}' not found.\n"
-                    f"Registered spiders: {available_spiders}\n"
-                    f"Please check if the spider name is correct, or ensure the spider has been properly imported and registered."
-                )
-            return spider_cls
-        else:
-            raise TypeError(
-                f"Invalid type {type(item)}: {item}\n"
-                f"Must be Spider class or string name.\n"
-                f"Example: MySpider or 'my_spider'"
-            )
-
-    async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
-        """
-        Spider running function limited by semaphore
-
-        Includes enhanced error handling and monitoring functionality
-        """
-        task = asyncio.current_task()
-        crawler = None
-
-        try:
-            # Register task
-            if task:
-                self._active_tasks.add(task)
-
-            # Acquire concurrency permit
-            await self.semaphore.acquire()
-
-            # start_msg = f"[{seq}/{total}] Initializing spider: {spider_cls.__name__}"
-            # logger.info(start_msg)
-
-            # Create and run crawler
-            crawler = Crawler(spider_cls, self.settings, self.context)
-            self.crawlers.add(crawler)
-
-            # Record start time
-            start_time = time.time()
-
-            # Run crawler
-            await crawler.crawl()
-
-            # Calculate runtime
-            duration = time.time() - start_time
-
-            end_msg = (
-                f"[{seq}/{total}] Crawler completed: {spider_cls.__name__}, "
-                f"took: {duration:.2f} seconds"
-            )
-            logger.info(end_msg)
-
-            # Record success statistics
-            self._performance_stats['successful_requests'] += 1
-
-        except Exception as e:
-            # Record failure statistics
-            self._performance_stats['failed_requests'] += 1
-
-            error_msg = f"Spider {spider_cls.__name__} execution failed: {e}"
-            logger.error(error_msg, exc_info=True)
-
-            # Record error information to context
-            if hasattr(self, 'context'):
-                self.context.increment_failed(error_msg)
-
-            raise
-        finally:
-            # Clean up resources
-            try:
-                if crawler and crawler in self.crawlers:
-                    self.crawlers.remove(crawler)
-
-                if task and task in self._active_tasks:
-                    self._active_tasks.remove(task)
-
-                self.semaphore.release()
-
-            except Exception as cleanup_error:
-                logger.warning(f"Error cleaning up resources: {cleanup_error}")
-
-    def _shutdown(self, _signum, _frame):
-        """
-        Graceful shutdown signal handling
-
-        Provides better shutdown experience and resource cleanup
-        """
-        signal_name = {signal.SIGINT: 'SIGINT', signal.SIGTERM: 'SIGTERM'}.get(_signum, str(_signum))
-        logger.warning(f"Received shutdown signal {signal_name}, stopping all crawlers...")
-
-        # Set shutdown event
-        if hasattr(self, '_shutdown_event'):
-            self._shutdown_event.set()
-
-        # Stop all crawler engines
-        for crawler in list(self.crawlers):
-            if crawler.engine:
-                crawler.engine.running = False
-                crawler.engine.normal = False
-                logger.debug(f"Crawler engine stopped: {getattr(crawler.spider, 'name', 'Unknown')}")
-
-        # Create shutdown task
-        asyncio.create_task(self._wait_for_shutdown())
-
-        logger.info("Shutdown command sent, waiting for crawlers to complete current tasks...")
-
-    async def _wait_for_shutdown(self):
-        """
-        Wait for all active tasks to complete
-
-        Provides better shutdown time control and progress feedback
-        """
-        try:
-            # Stop monitoring task
-            await self.stop_monitoring()
-
-            # Wait for active tasks to complete
-            pending = [t for t in self._active_tasks if not t.done()]
-
-            if pending:
-                logger.info(
-                    f"Waiting for {len(pending)} active tasks to complete..."
-                    f"(Maximum wait time: 30 seconds)"
-                )
-
-                # Set timeout
-                try:
-                    await asyncio.wait_for(
-                        asyncio.gather(*pending, return_exceptions=True),
-                        timeout=30.0
-                    )
-                except asyncio.TimeoutError:
-                    logger.warning("Some tasks timed out, forcing cancellation...")
-
-                    # Force cancel timed out tasks
-                    for task in pending:
-                        if not task.done():
-                            task.cancel()
-
-                    # Wait for cancellation to complete
-                    await asyncio.gather(*pending, return_exceptions=True)
-
-            # Final cleanup
-            await self._cleanup_process()
-
-            # Output final statistics
-            final_stats = self.context.get_stats()
-            logger.info(
-                f"All crawlers gracefully shut down 👋\n"
-                f"  - Total crawlers: {final_stats['total_crawlers']}\n"
-                f"  - Successfully completed: {final_stats['completed_crawlers']}\n"
-                f"  - Failed: {final_stats['failed_crawlers']}\n"
-                f"  - Success rate: {final_stats['success_rate']:.1f}%\n"
-                f"  - Total runtime: {final_stats['duration_seconds']} seconds"
-            )
-
-        except Exception as e:
-            logger.error(f"Error during shutdown process: {e}", exc_info=True)
-
-    @classmethod
-    def _get_default_settings(cls) -> SettingManager:
-        """
-        Load default configuration
-
-        Provides better error handling and fallback strategy
-        """
-        try:
-            settings = get_settings()
-            logger.debug("Default configuration loaded successfully")
-            return settings
-        except Exception as e:
-            logger.warning(f"Unable to load default configuration: {e}, using empty configuration")
-            return SettingManager()
-
-# === Utility functions ===
-
-def create_crawler_with_optimizations(
-        spider_cls: Type[Spider],
-        settings: Optional[SettingManager] = None,
-        **optimization_kwargs
-) -> Crawler:
-    """
-    Create an optimized crawler instance
-
-    :param spider_cls: Spider class
-    :param settings: Settings manager
-    :param optimization_kwargs: Optimization parameters
-    :return: Crawler instance
-    """
-    if settings is None:
-        settings = SettingManager()
-
-    # Apply optimization configuration
-    for key, value in optimization_kwargs.items():
-        settings.set(key, value)
-
-    context = CrawlerContext()
-    return Crawler(spider_cls, settings, context)
-
-
-def create_process_with_large_scale_config(
-        config_type: str = 'balanced',
-        concurrency: int = 16,
-        **kwargs
-) -> CrawlerProcess:
-    """
-    Create a process manager that supports large-scale optimization
-
-    :param config_type: Configuration type ('conservative', 'balanced', 'aggressive', 'memory_optimized')
-    :param concurrency: Concurrency count
-    :param kwargs: Other parameters
-    :return: Process manager
-    """
-    try:
-        from crawlo.utils.large_scale_config import LargeScaleConfig
-
-        # Get optimization configuration
-        config_methods = {
-            'conservative': LargeScaleConfig.conservative_config,
-            'balanced': LargeScaleConfig.balanced_config,
-            'aggressive': LargeScaleConfig.aggressive_config,
-            'memory_optimized': LargeScaleConfig.memory_optimized_config
-        }
-
-        if config_type not in config_methods:
-            logger.warning(f"Unknown configuration type: {config_type}, using default configuration")
-            settings = SettingManager()
-        else:
-            config = config_methods[config_type](concurrency)
-            settings = SettingManager()
-            settings.update(config)
-
-        return CrawlerProcess(
-            settings=settings,
-            max_concurrency=concurrency,
-            **kwargs
-        )
-
-    except ImportError:
-        logger.warning("Large-scale configuration module does not exist, using default configuration")
-        return CrawlerProcess(max_concurrency=concurrency, **kwargs)
-
-
-# === Exported interfaces ===
-
-__all__ = [
-    'Crawler',
-    'CrawlerProcess',
-    'CrawlerContext',
-    'create_crawler_with_optimizations',
-    'create_process_with_large_scale_config'
-]
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+重构后的Crawler系统
+==================
+
+设计原则：
+1. 单一职责 - 每个类只负责一个明确的功能
+2. 依赖注入 - 通过工厂创建组件，便于测试
+3. 状态管理 - 清晰的状态转换和生命周期
+4. 错误处理 - 优雅的错误处理和恢复机制
+"""
+
+import asyncio
+import time
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from enum import Enum
+from typing import Optional, Type, Dict, Any, List
+
+from crawlo.factories import get_component_registry
+from crawlo.initialization import initialize_framework, is_framework_ready
+from crawlo.logging import get_logger
+
+
+class CrawlerState(Enum):
+    """Crawler状态枚举"""
+    CREATED = "created"
+    INITIALIZING = "initializing"
+    READY = "ready"
+    RUNNING = "running"
+    CLOSING = "closing"
+    CLOSED = "closed"
+    ERROR = "error"
+
+
+@dataclass
+class CrawlerMetrics:
+    """Crawler性能指标"""
+    start_time: Optional[float] = None
+    end_time: Optional[float] = None
+    initialization_duration: float = 0.0
+    crawl_duration: float = 0.0
+    request_count: int = 0
+    success_count: int = 0
+    error_count: int = 0
+    
+    def get_total_duration(self) -> float:
+        if self.start_time and self.end_time:
+            return self.end_time - self.start_time
+        return 0.0
+    
+    def get_success_rate(self) -> float:
+        total = self.success_count + self.error_count
+        return (self.success_count / total * 100) if total > 0 else 0.0
+
+
+class ModernCrawler:
+    """
+    现代化的Crawler实现
+    
+    特点：
+    1. 清晰的状态管理
+    2. 依赖注入
+    3. 组件化架构
+    4. 完善的错误处理
+    """
+    
+    def __init__(self, spider_cls: Type, settings=None):
+        self._spider_cls = spider_cls
+        self._settings = settings
+        self._state = CrawlerState.CREATED
+        self._state_lock = asyncio.Lock()
+        
+        # 组件
+        self._spider = None
+        self._engine = None
+        self._stats = None
+        self._subscriber = None
+        self._extension = None
+        
+        # 指标
+        self._metrics = CrawlerMetrics()
+        
+        # 日志
+        self._logger = get_logger(f'crawler.{spider_cls.__name__ if spider_cls else "unknown"}')
+        
+        # 确保框架已初始化
+        self._ensure_framework_ready()
+    
+    def _ensure_framework_ready(self):
+        """确保框架已准备就绪"""
+        if not is_framework_ready():
+            try:
+                self._settings = initialize_framework(self._settings)
+                self._logger.debug("Framework initialized successfully")
+            except Exception as e:
+                self._logger.warning(f"Framework initialization failed: {e}")
+                # 使用降级策略
+                if not self._settings:
+                    from crawlo.settings.setting_manager import SettingManager
+                    self._settings = SettingManager()
+        
+        # 确保是SettingManager实例
+        if isinstance(self._settings, dict):
+            from crawlo.settings.setting_manager import SettingManager
+            settings_manager = SettingManager()
+            settings_manager.update_attributes(self._settings)
+            self._settings = settings_manager
+    
+    @property
+    def state(self) -> CrawlerState:
+        """获取当前状态"""
+        return self._state
+    
+    @property
+    def spider(self):
+        """获取Spider实例"""
+        return self._spider
+    
+    @property
+    def stats(self):
+        """获取Stats实例（向后兼容）"""
+        return self._stats
+    
+    @property 
+    def metrics(self) -> CrawlerMetrics:
+        """获取性能指标"""
+        return self._metrics
+    
+    @property
+    def settings(self):
+        """获取配置"""
+        return self._settings
+    
+    @property
+    def engine(self):
+        """获取Engine实例（向后兼容）"""
+        return self._engine
+    
+    @property
+    def subscriber(self):
+        """获取Subscriber实例（向后兼容）"""
+        return self._subscriber
+    
+    @property
+    def extension(self):
+        """获取Extension实例（向后兼容）"""
+        return self._extension
+    
+    @extension.setter
+    def extension(self, value):
+        """设置Extension实例（向后兼容）"""
+        self._extension = value
+    
+    def _create_extension(self):
+        """创建Extension管理器（向后兼容）"""
+        if self._extension is None:
+            try:
+                registry = get_component_registry()
+                self._extension = registry.create('extension_manager', crawler=self)
+            except Exception as e:
+                self._logger.warning(f"Failed to create extension manager: {e}")
+        return self._extension
+    
+    async def close(self):
+        """关闭爹虫（向后兼容）"""
+        await self._cleanup()
+    
+    async def crawl(self):
+        """执行爬取任务"""
+        async with self._lifecycle_manager():
+            await self._initialize_components()
+            await self._run_crawler()
+    
+    @asynccontextmanager
+    async def _lifecycle_manager(self):
+        """生命周期管理"""
+        self._metrics.start_time = time.time()
+        
+        try:
+            yield
+        except Exception as e:
+            await self._handle_error(e)
+            raise
+        finally:
+            await self._cleanup()
+            self._metrics.end_time = time.time()
+    
+    async def _initialize_components(self):
+        """初始化组件"""
+        async with self._state_lock:
+            if self._state != CrawlerState.CREATED:
+                raise RuntimeError(f"Cannot initialize from state {self._state}")
+            
+            self._state = CrawlerState.INITIALIZING
+        
+        init_start = time.time()
+        
+        try:
+            # 使用组件工厂创建组件
+            registry = get_component_registry()
+            
+            # 创建Subscriber（无依赖）
+            self._subscriber = registry.create('subscriber')
+            
+            # 创建Spider
+            self._spider = self._create_spider()
+            
+            # 创建Engine（需要crawler参数）
+            self._engine = registry.create('engine', crawler=self)
+            
+            # 创建Stats（需要crawler参数）
+            self._stats = registry.create('stats', crawler=self)
+            
+            # 创建Extension Manager (可选，需要crawler参数)
+            try:
+                self._extension = registry.create('extension_manager', crawler=self)
+            except Exception as e:
+                self._logger.warning(f"Failed to create extension manager: {e}")
+            
+            self._metrics.initialization_duration = time.time() - init_start
+            
+            async with self._state_lock:
+                self._state = CrawlerState.READY
+            
+            self._logger.debug(f"Crawler components initialized successfully in {self._metrics.initialization_duration:.2f}s")
+            
+        except Exception as e:
+            async with self._state_lock:
+                self._state = CrawlerState.ERROR
+            raise RuntimeError(f"Component initialization failed: {e}")
+    
+    def _create_spider(self):
+        """创建Spider实例"""
+        if not self._spider_cls:
+            raise ValueError("Spider class not provided")
+        
+        # 检查Spider类的有效性
+        if not hasattr(self._spider_cls, 'name'):
+            raise ValueError("Spider class must have 'name' attribute")
+        
+        # 创建Spider实例
+        spider = self._spider_cls()
+        
+        # 设置crawler引用
+        if hasattr(spider, 'crawler'):
+            spider.crawler = self
+        
+        return spider
+    
+    async def _run_crawler(self):
+        """运行爬虫引擎"""
+        async with self._state_lock:
+            if self._state != CrawlerState.READY:
+                raise RuntimeError(f"Cannot run from state {self._state}")
+            
+            self._state = CrawlerState.RUNNING
+        
+        crawl_start = time.time()
+        
+        try:
+            # 启动引擎
+            if self._engine:
+                await self._engine.start_spider(self._spider)
+            else:
+                raise RuntimeError("Engine not initialized")
+            
+            self._metrics.crawl_duration = time.time() - crawl_start
+            
+            self._logger.info(f"Crawler completed successfully in {self._metrics.crawl_duration:.2f}s")
+            
+        except Exception as e:
+            self._metrics.crawl_duration = time.time() - crawl_start
+            raise RuntimeError(f"Crawler execution failed: {e}")
+    
+    async def _handle_error(self, error: Exception):
+        """处理错误"""
+        async with self._state_lock:
+            self._state = CrawlerState.ERROR
+        
+        self._metrics.error_count += 1
+        self._logger.error(f"Crawler error: {error}", exc_info=True)
+        
+        # 这里可以添加错误恢复逻辑
+    
+    async def _cleanup(self):
+        """清理资源"""
+        async with self._state_lock:
+            if self._state not in [CrawlerState.CLOSING, CrawlerState.CLOSED]:
+                self._state = CrawlerState.CLOSING
+        
+        try:
+            # 关闭各个组件
+            if self._engine and hasattr(self._engine, 'close'):
+                try:
+                    await self._engine.close()
+                except Exception as e:
+                    self._logger.warning(f"Engine cleanup failed: {e}")
+            
+            # 调用Spider的spider_closed方法
+            if self._spider:
+                try:
+                    if asyncio.iscoroutinefunction(self._spider.spider_closed):
+                        await self._spider.spider_closed()
+                    else:
+                        self._spider.spider_closed()
+                except Exception as e:
+                    self._logger.warning(f"Spider cleanup failed: {e}")
+            
+            if self._stats and hasattr(self._stats, 'close'):
+                try:
+                    close_result = self._stats.close()
+                    if asyncio.iscoroutine(close_result):
+                        await close_result
+                except Exception as e:
+                    self._logger.warning(f"Stats cleanup failed: {e}")
+            
+            async with self._state_lock:
+                self._state = CrawlerState.CLOSED
+            
+            self._logger.debug("Crawler cleanup completed")
+            
+        except Exception as e:
+            self._logger.error(f"Cleanup error: {e}")
+
+
+class CrawlerProcess:
+    """
+    Crawler进程管理器 - 管理多个Crawler的执行
+    
+    简化版本，专注于核心功能
+    """
+    
+    def __init__(self, settings=None, max_concurrency: int = 3, spider_modules=None):
+        self._settings = settings or initialize_framework()
+        self._max_concurrency = max_concurrency
+        self._crawlers: List[ModernCrawler] = []
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+        self._logger = get_logger('crawler.process')
+        self._spider_modules = spider_modules  # 保存spider_modules
+        
+        # 如果提供了spider_modules，自动注册这些模块中的爬虫
+        if spider_modules:
+            self._register_spider_modules(spider_modules)
+        
+        # 指标
+        self._start_time: Optional[float] = None
+        self._end_time: Optional[float] = None
+    
+    def _register_spider_modules(self, spider_modules):
+        """注册爬虫模块"""
+        try:
+            from crawlo.spider import get_global_spider_registry
+            registry = get_global_spider_registry()
+            
+            self._logger.debug(f"Registering spider modules: {spider_modules}")
+            
+            initial_spider_count = len(registry)
+            
+            for module_path in spider_modules:
+                try:
+                    # 导入模块
+                    __import__(module_path)
+                    self._logger.debug(f"Successfully imported spider module: {module_path}")
+                except ImportError as e:
+                    self._logger.warning(f"Failed to import spider module {module_path}: {e}")
+                    # 如果导入失败，尝试自动发现
+                    self._auto_discover_spider_modules([module_path])
+            
+            # 检查注册表中的爬虫
+            spider_names = list(registry.keys())
+            self._logger.debug(f"Registered spiders after import: {spider_names}")
+            
+            # 如果导入模块后没有新的爬虫被注册，则尝试自动发现
+            final_spider_count = len(registry)
+            if final_spider_count == initial_spider_count:
+                self._logger.debug("No new spiders registered after importing modules, attempting auto-discovery")
+                self._auto_discover_spider_modules(spider_modules)
+                spider_names = list(registry.keys())
+                self._logger.debug(f"Registered spiders after auto-discovery: {spider_names}")
+        except Exception as e:
+            self._logger.warning(f"Error registering spider modules: {e}")
+    
+    def _auto_discover_spider_modules(self, spider_modules):
+        """
+        自动发现并导入爬虫模块中的所有爬虫
+        这个方法会扫描指定模块目录下的所有Python文件并自动导入
+        """
+        try:
+            from crawlo.spider import get_global_spider_registry
+            import importlib
+            from pathlib import Path
+            import sys
+            
+            registry = get_global_spider_registry()
+            initial_spider_count = len(registry)
+            
+            for module_path in spider_modules:
+                try:
+                    # 将模块路径转换为文件系统路径
+                    # 例如: ofweek_standalone.spiders -> ofweek_standalone/spiders
+                    package_parts = module_path.split('.')
+                    if len(package_parts) < 2:
+                        continue
+                        
+                    # 获取项目根目录
+                    project_root = None
+                    for path in sys.path:
+                        if path and Path(path).exists():
+                            possible_module_path = Path(path) / package_parts[0]
+                            if possible_module_path.exists():
+                                project_root = path
+                                break
+                    
+                    if not project_root:
+                        # 尝试使用当前工作目录
+                        project_root = str(Path.cwd())
+                    
+                    # 构建模块目录路径
+                    module_dir = Path(project_root)
+                    for part in package_parts:
+                        module_dir = module_dir / part
+                    
+                    # 如果目录存在，扫描其中的Python文件
+                    if module_dir.exists() and module_dir.is_dir():
+                        # 导入目录下的所有Python文件（除了__init__.py）
+                        for py_file in module_dir.glob("*.py"):
+                            if py_file.name.startswith('_'):
+                                continue
+                                
+                            # 构造模块名
+                            module_name = py_file.stem  # 文件名（不含扩展名）
+                            full_module_path = f"{module_path}.{module_name}"
+                            
+                            try:
+                                # 导入模块以触发Spider注册
+                                importlib.import_module(full_module_path)
+                            except ImportError as e:
+                                self._logger.warning(f"Failed to auto-import spider module {full_module_path}: {e}")
+                except Exception as e:
+                    self._logger.warning(f"Error during auto-discovery for module {module_path}: {e}")
+            
+            # 检查是否有新的爬虫被注册
+            final_spider_count = len(registry)
+            if final_spider_count > initial_spider_count:
+                new_spiders = list(registry.keys())
+                self._logger.info(f"Auto-discovered {final_spider_count - initial_spider_count} new spiders: {new_spiders}")
+                
+        except Exception as e:
+            self._logger.warning(f"Error during auto-discovery of spider modules: {e}")
+    
+    def is_spider_registered(self, name: str) -> bool:
+        """检查爬虫是否已注册"""
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        return name in registry
+    
+    def get_spider_class(self, name: str):
+        """获取爬虫类"""
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        return registry.get(name)
+    
+    def get_spider_names(self):
+        """获取所有注册的爬虫名称"""
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        return list(registry.keys())
+    
+    async def crawl(self, spider_cls_or_name, settings=None):
+        """运行单个爬虫"""
+        spider_cls = self._resolve_spider_class(spider_cls_or_name)
+        
+        # 记录启动的爬虫名称（符合规范要求）
+        from crawlo.logging import get_logger
+        logger = get_logger('crawlo.framework')
+        logger.info(f"Starting spider: {spider_cls.name}")
+        
+        merged_settings = self._merge_settings(settings)
+        crawler = ModernCrawler(spider_cls, merged_settings)
+        
+        async with self._semaphore:
+            await crawler.crawl()
+        
+        return crawler
+    
+    async def crawl_multiple(self, spider_classes_or_names, settings=None):
+        """运行多个爬虫"""
+        self._start_time = time.time()
+        
+        try:
+            spider_classes = []
+            for cls_or_name in spider_classes_or_names:
+                spider_cls = self._resolve_spider_class(cls_or_name)
+                spider_classes.append(spider_cls)
+            
+            # 记录启动的爬虫名称（符合规范要求）
+            spider_names = [cls.name for cls in spider_classes]
+            from crawlo.logging import get_logger
+            logger = get_logger('crawlo.framework')
+            if len(spider_names) == 1:
+                logger.info(f"Starting spider: {spider_names[0]}")
+            else:
+                logger.info(f"Starting spiders: {', '.join(spider_names)}")
+            
+            tasks = []
+            for spider_cls in spider_classes:
+                merged_settings = self._merge_settings(settings)
+                crawler = ModernCrawler(spider_cls, merged_settings)
+                self._crawlers.append(crawler)
+                
+                task = asyncio.create_task(self._run_with_semaphore(crawler))
+                tasks.append(task)
+            
+            results = await asyncio.gather(*tasks, return_exceptions=True)
+            
+            # 处理结果
+            successful = sum(1 for r in results if not isinstance(r, Exception))
+            failed = len(results) - successful
+            
+            self._logger.info(f"Crawl completed: {successful} successful, {failed} failed")
+            
+            return results
+            
+        finally:
+            self._end_time = time.time()
+            if self._start_time:
+                duration = self._end_time - self._start_time
+                self._logger.info(f"Total execution time: {duration:.2f}s")
+    
+    async def _run_with_semaphore(self, crawler: ModernCrawler):
+        """在信号量控制下运行爬虫"""
+        async with self._semaphore:
+            await crawler.crawl()
+            return crawler
+    
+    def _resolve_spider_class(self, spider_cls_or_name):
+        """解析Spider类"""
+        if isinstance(spider_cls_or_name, str):
+            # 从注册表中查找
+            try:
+                from crawlo.spider import get_global_spider_registry
+                registry = get_global_spider_registry()
+                if spider_cls_or_name in registry:
+                    return registry[spider_cls_or_name]
+                else:
+                    # 如果在注册表中找不到，尝试通过spider_modules导入所有模块来触发注册
+                    # 然后再次检查注册表
+                    if hasattr(self, '_spider_modules') and self._spider_modules:
+                        for module_path in self._spider_modules:
+                            try:
+                                # 导入模块来触发爬虫注册
+                                __import__(module_path)
+                            except ImportError:
+                                pass  # 忽略导入错误
+                        
+                        # 再次检查注册表
+                        if spider_cls_or_name in registry:
+                            return registry[spider_cls_or_name]
+                    
+                    # 如果仍然找不到，尝试自动发现模式
+                    if hasattr(self, '_spider_modules') and self._spider_modules:
+                        self._auto_discover_spider_modules(self._spider_modules)
+                        if spider_cls_or_name in registry:
+                            return registry[spider_cls_or_name]
+                    
+                    # 如果仍然找不到，尝试直接导入模块
+                    try:
+                        # 假设格式为 module.SpiderClass
+                        if '.' in spider_cls_or_name:
+                            module_path, class_name = spider_cls_or_name.rsplit('.', 1)
+                            module = __import__(module_path, fromlist=[class_name])
+                            spider_class = getattr(module, class_name)
+                            # 注册到全局注册表
+                            registry[spider_class.name] = spider_class
+                            return spider_class
+                        else:
+                            # 尝试在spider_modules中查找
+                            if hasattr(self, '_spider_modules') and self._spider_modules:
+                                for module_path in self._spider_modules:
+                                    try:
+                                        # 构造完整的模块路径
+                                        full_module_path = f"{module_path}.{spider_cls_or_name}"
+                                        module = __import__(full_module_path, fromlist=[spider_cls_or_name])
+                                        # 获取模块中的Spider子类
+                                        for attr_name in dir(module):
+                                            attr_value = getattr(module, attr_name)
+                                            if (isinstance(attr_value, type) and
+                                                    issubclass(attr_value, registry.__class__.__bases__[0]) and
+                                                    hasattr(attr_value, 'name') and
+                                                    attr_value.name == spider_cls_or_name):
+                                                # 注册到全局注册表
+                                                registry[spider_cls_or_name] = attr_value
+                                                return attr_value
+                                    except ImportError:
+                                        continue
+                            raise ValueError(f"Spider '{spider_cls_or_name}' not found in registry")
+                    except (ImportError, AttributeError):
+                        raise ValueError(f"Spider '{spider_cls_or_name}' not found in registry")
+            except ImportError:
+                raise ValueError(f"Cannot resolve spider name '{spider_cls_or_name}'")
+        else:
+            return spider_cls_or_name
+    
+    def _merge_settings(self, additional_settings):
+        """合并配置"""
+        if not additional_settings:
+            return self._settings
+        
+        # 这里可以实现更复杂的配置合并逻辑
+        from crawlo.settings.setting_manager import SettingManager
+        merged = SettingManager()
+        
+        # 复制基础配置
+        if self._settings:
+            merged.update_attributes(self._settings.__dict__)
+        
+        # 应用额外配置
+        merged.update_attributes(additional_settings)
+        
+        return merged
+    
+    def get_metrics(self) -> Dict[str, Any]:
+        """获取整体指标"""
+        total_duration = 0.0
+        if self._start_time and self._end_time:
+            total_duration = self._end_time - self._start_time
+        
+        crawler_metrics = [crawler.metrics for crawler in self._crawlers]
+        
+        return {
+            'total_duration': total_duration,
+            'crawler_count': len(self._crawlers),
+            'total_requests': sum(m.request_count for m in crawler_metrics),
+            'total_success': sum(m.success_count for m in crawler_metrics),
+            'total_errors': sum(m.error_count for m in crawler_metrics),
+            'average_success_rate': sum(m.get_success_rate() for m in crawler_metrics) / len(crawler_metrics) if crawler_metrics else 0.0
+        }