crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/__init__.py

@@ -3,6 +3,8 @@
 """
 Crawlo - 一个异步爬虫框架
 """
+from typing import TYPE_CHECKING
+
 from crawlo.spider import Spider
 from crawlo.items import Item, Field
 from crawlo.network.request import Request
@@ -24,9 +26,29 @@ from crawlo.utils import (
 )
 from crawlo import tools
 
+# 框架核心模块 - 使用TYPE_CHECKING避免循环导入
+if TYPE_CHECKING:
+    from crawlo.core.framework_initializer import get_framework_initializer, initialize_framework
+
 # 为了向后兼容，从tools中导入cleaners相关的功能
 import crawlo.tools as cleaners
 
+# 延迟导入的辅助函数
+def get_framework_initializer():
+    """延迟导入get_framework_initializer以避免循环依赖"""
+    from crawlo.core.framework_initializer import get_framework_initializer as _get_framework_initializer
+    return _get_framework_initializer()
+
+def initialize_framework(custom_settings=None):
+    """延迟导入initialize_framework以避免循环依赖"""
+    from crawlo.core.framework_initializer import initialize_framework as _initialize_framework
+    return _initialize_framework(custom_settings)
+
+# 向后兼容的别名
+def get_bootstrap_manager():
+    """向后兼容的别名"""
+    return get_framework_initializer()
+
 # 版本号：优先从元数据读取
 try:
     from importlib.metadata import version
@@ -60,5 +82,7 @@ __all__ = [
     'from_timestamp_with_tz',
     'cleaners',
     'tools',
+    'get_framework_initializer',
+    'get_bootstrap_manager',
     '__version__',
 ]

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = '1.3.2'
+__version__ = '1.3.4'

crawlo/commands/run.py

@@ -21,10 +21,23 @@ from rich.text import Text
 from crawlo.commands.stats import record_stats
 from crawlo.crawler import CrawlerProcess
 from crawlo.project import get_settings, _find_project_root
-# 使用自定义日志系统
+# 使用新的统一初始化系统
+from crawlo.initialization import initialize_framework
+from crawlo.core import get_framework_initializer
 from crawlo.utils.log import get_logger
 
-logger = get_logger(__name__)
+# 延迟获取logger，确保在日志系统配置之后获取
+_logger = None
+
+
+def logger():
+    """延迟获取logger实例，确保在日志系统配置之后获取"""
+    global _logger
+    if _logger is None:
+        # 使用改进后的日志系统，可以安全地在任何时候创建
+        _logger = get_logger(__name__)
+    return _logger
+
 
 console = Console()
 
@@ -35,15 +48,15 @@ def check_redis_connection(settings):
         # 检查是否为分布式模式
         run_mode = settings.get('RUN_MODE', 'standalone')
         queue_type = settings.get('QUEUE_TYPE', 'memory')
-        
+
         if run_mode == 'distributed' or queue_type == 'redis':
             import redis.asyncio as redis
             redis_url = settings.get('REDIS_URL', 'redis://127.0.0.1:6379/0')
             redis_host = settings.get('REDIS_HOST', '127.0.0.1')
             redis_port = settings.get('REDIS_PORT', 6379)
-            
+
             console.print(f"检查 Redis 连接: {redis_host}:{redis_port}")
-            
+
             # 创建Redis连接进行测试
             async def _test_redis():
                 try:
@@ -54,11 +67,11 @@ def check_redis_connection(settings):
                 except Exception as e:
                     console.print(f"Redis 连接失败: {e}")
                     return False
-            
+
             # 运行异步测试
             if not asyncio.run(_test_redis()):
                 raise ConnectionError(f"无法连接到 Redis 服务器 {redis_host}:{redis_port}")
-                
+
             console.print("Redis 连接正常")
             return True
         else:
@@ -78,11 +91,15 @@ def main(args):
     用法:
         crawlo run <spider_name>|all [--json] [--no-stats]
     """
+    # 确保框架已初始化
+    init_manager = get_framework_initializer()
+
     # 添加调试信息
-    logger.debug("DEBUG: 进入main函数")
-    
+    logger().debug("DEBUG: 进入main函数")
+
     if len(args) < 1:
-        console.print("[bold red]用法:[/bold red] [blue]crawlo run[/blue] <爬虫名称>|all [bold yellow][--json] [--no-stats][/bold yellow]")
+        console.print(
+            "[bold red]用法:[/bold red] [blue]crawlo run[/blue] <爬虫名称>|all [bold yellow][--json] [--no-stats][/bold yellow]")
         console.print("示例:")
         console.print("   [blue]crawlo run baidu[/blue]")
         console.print("   [blue]crawlo run all[/blue]")
@@ -153,9 +170,9 @@ def main(args):
                 console.print(Panel(msg, title="导入错误", border_style="red"))
                 return 1
 
-        # 4. 加载 settings 和爬虫模块
-        settings = get_settings()
-        
+        # 4. 启动框架并加载 settings
+        settings = initialize_framework()
+
         # 检查Redis连接（如果是分布式模式）
         if not check_redis_connection(settings):
             if show_json:
@@ -163,9 +180,22 @@ def main(args):
                 return 1
             else:
                 return 1
-        
-        spider_modules = [f"{project_package}.spiders"]
+
+        # 从配置中获取SPIDER_MODULES
+        spider_modules = settings.get('SPIDER_MODULES', [f"{project_package}.spiders"])
+        logger().debug(f"SPIDER_MODULES from settings: {spider_modules}")
         process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
+        
+        # 不再需要手动导入爬虫模块，框架内部会自动处理
+        # 检查注册表中的爬虫
+        from crawlo.spider import get_global_spider_registry
+        registry = get_global_spider_registry()
+        spider_names = list(registry.keys())
+        logger().debug(f"Registered spiders after import: {spider_names}")
+        
+        # 调试信息
+        logger().debug(f"SPIDER_MODULES: {spider_modules}")
+        logger().debug(f"Available spiders: {process.get_spider_names()}")
 
         # === 情况1：运行所有爬虫 ===
         if spider_arg.lower() == "all":
@@ -193,19 +223,14 @@ def main(args):
             # 显示即将运行的爬虫列表
             # 根据用户要求，不再显示详细的爬虫列表信息
 
-            # 注册 stats 记录（除非 --no-stats）
-            if not no_stats:
-                for crawler in process.crawlers:
-                    crawler.signals.connect(record_stats, signal="spider_closed")
-
             # 并行运行所有爬虫
             with Progress(
-                SpinnerColumn(),
-                TextColumn("[progress.description]{task.description}"),
-                transient=True,
+                    SpinnerColumn(),
+                    TextColumn("[progress.description]{task.description}"),
+                    transient=True,
             ) as progress:
                 task = progress.add_task("正在运行所有爬虫...", total=None)
-                asyncio.run(process.crawl(spider_names))
+                asyncio.run(process.crawl_multiple(spider_names))
 
             if show_json:
                 console.print_json(data={"success": True, "spiders": spider_names})
@@ -267,15 +292,16 @@ def main(args):
         #     console.print()
 
         # 注册 stats 记录
-        if not no_stats:
-            for crawler in process.crawlers:
-                crawler.signals.connect(record_stats, signal="spider_closed")
+        # 注意：CrawlerProcess没有crawlers属性，我们需要在运行时注册
+        # if not no_stats:
+        #     for crawler in process.crawlers:
+        #         crawler.signals.connect(record_stats, signal="spider_closed")
 
         # 运行爬虫
         with Progress(
-            SpinnerColumn(),
-            TextColumn("[progress.description]{task.description}"),
-            transient=True,
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                transient=True,
         ) as progress:
             task = progress.add_task(f"正在运行 {spider_name}...", total=None)
             asyncio.run(process.crawl(spider_name))
@@ -298,7 +324,7 @@ def main(args):
             console.print(f"[bold yellow]{msg}[/bold yellow]")
         return 1
     except Exception as e:
-        logger.exception("Exception during 'crawlo run'")
+        logger().exception("Exception during 'crawlo run'")
         msg = f"意外错误: {e}"
         if show_json:
             console.print_json(data={"success": False, "error": msg})
@@ -312,4 +338,4 @@ if __name__ == "__main__":
     支持直接运行：
         python -m crawlo.commands.run spider_name
     """
-    sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

crawlo/core/__init__.py

@@ -1,2 +1,46 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
+
+# Crawlo核心模块
+# 提供框架的核心组件和初始化功能
+
+# 使用新的初始化系统
+from ..initialization import (
+    initialize_framework,
+    is_framework_ready
+)
+
+# 向后兼容的别名
+def async_initialize_framework(*args, **kwargs):
+    """Async wrapper for framework initialization"""
+    return initialize_framework(*args, **kwargs)
+
+def get_framework_initializer():
+    """Get framework initializer - compatibility function"""
+    from ..initialization.core import CoreInitializer
+    return CoreInitializer()
+
+def get_framework_logger(name='crawlo.core'):
+    """Get framework logger - compatibility function"""
+    from ..logging import get_logger
+    return get_logger(name)
+
+# 向后兼容
+def bootstrap_framework(*args, **kwargs):
+    """Bootstrap framework - compatibility function"""
+    return initialize_framework(*args, **kwargs)
+
+def get_bootstrap_manager():
+    """Get bootstrap manager - compatibility function"""
+    return get_framework_initializer()
+
+__all__ = [
+    'initialize_framework',
+    'async_initialize_framework', 
+    'get_framework_initializer',
+    'is_framework_ready',
+    'get_framework_logger',
+    # 向后兼容
+    'bootstrap_framework',
+    'get_bootstrap_manager'
+]

crawlo/core/engine.py

@@ -6,16 +6,16 @@ from inspect import iscoroutine
 from typing import Optional, Generator, Callable
 
 from crawlo import Request, Item
-from crawlo.spider import Spider
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import OutputError
-from crawlo.core.scheduler import Scheduler
 from crawlo.core.processor import Processor
-from crawlo.task_manager import TaskManager
-from crawlo.project import load_class
+from crawlo.core.scheduler import Scheduler
 from crawlo.downloader import DownloaderBase
-from crawlo.utils.func_tools import transform
 from crawlo.event import spider_opened, spider_error, request_scheduled
+from crawlo.exceptions import OutputError
+from crawlo.utils.class_loader import load_class
+from crawlo.spider import Spider
+from crawlo.task_manager import TaskManager
+from crawlo.utils.func_tools import transform
+from crawlo.utils.log import get_logger
 
 
 class Engine(object):
@@ -35,8 +35,8 @@ class Engine(object):
         # Enhanced control parameters
         self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
         self.generation_batch_size = self.settings.get_int('REQUEST_GENERATION_BATCH_SIZE', 10)
-        self.generation_interval = self.settings.get_float('REQUEST_GENERATION_INTERVAL', 0.05)
-        self.backpressure_ratio = self.settings.get_float('BACKPRESSURE_RATIO', 0.8)  # Start backpressure when queue reaches 80%
+        self.generation_interval = self.settings.get_float('REQUEST_GENERATION_INTERVAL', 0.01)  # 优化默认值
+        self.backpressure_ratio = self.settings.get_float('BACKPRESSURE_RATIO', 0.9)  # 优化默认值
         
         # State tracking
         self._generation_paused = False
@@ -74,9 +74,7 @@ class Engine(object):
         if not version or version == 'None':
             version = '1.0.0'
         # Change INFO level log to DEBUG level to avoid duplication with CrawlerProcess startup log
-        self.logger.debug(
-            f"Crawlo Started version {version}"
-        )
+        self.logger.debug(f"Crawlo Framework Started {version}")
 
     async def start_spider(self, spider):
         self.spider = spider
@@ -110,7 +108,20 @@ class Engine(object):
         if not hasattr(self.crawler, 'extension') or not self.crawler.extension:
             self.crawler.extension = self.crawler._create_extension()
 
-        self.start_requests = iter(spider.start_requests())
+        # 启动引擎
+        self.engine_start()
+        
+        self.logger.debug("开始创建start_requests迭代器")
+        try:
+            # 先收集所有请求到列表中，避免在检查时消耗迭代器
+            requests_list = list(spider.start_requests())
+            self.logger.debug(f"收集到 {len(requests_list)} 个请求")
+            self.start_requests = iter(requests_list)
+            self.logger.debug("start_requests迭代器创建成功")
+        except Exception as e:
+            self.logger.error(f"创建start_requests迭代器失败: {e}")
+            import traceback
+            self.logger.error(traceback.format_exc())
         await self._open_spider()
 
     async def crawl(self):
@@ -124,32 +135,46 @@ class Engine(object):
             # 启动请求生成任务（如果启用了受控生成）
             if (self.start_requests and 
                 self.settings.get_bool('ENABLE_CONTROLLED_REQUEST_GENERATION', False)):
+                self.logger.debug("创建受控请求生成任务")
                 generation_task = asyncio.create_task(
                     self._controlled_request_generation()
                 )
             else:
                 # 传统方式处理启动请求
+                self.logger.debug("创建传统请求生成任务")
                 generation_task = asyncio.create_task(
                     self._traditional_request_generation()
                 )
             
+            self.logger.debug("请求生成任务创建完成")
+            
             # 主爬取循环
+            loop_count = 0
+            last_exit_check = 0  # 记录上次检查退出条件的时间
+            exit_check_interval = 1  # 每1次循环检查一次退出条件，进一步提高检查频率
+            
             while self.running:
+                loop_count += 1
                 # 获取并处理请求
                 if request := await self._get_next_request():
                     await self._crawl(request)
                 
-                # 检查退出条件
-                if await self._should_exit():
-                    break
+                # 优化退出条件检查频率
+                if loop_count - last_exit_check >= exit_check_interval:
+                    should_exit = await self._should_exit()
+                    if should_exit:
+                        self.logger.debug("满足退出条件，准备退出循环")
+                        break
+                    last_exit_check = loop_count
                 
-                # 短暂休息避免忙等
-                await asyncio.sleep(0.001)
+                # 短暂休息避免忙等，但减少休息时间以提高效率
+                await asyncio.sleep(0.000001)  # 从0.00001减少到0.000001
+            
+            self.logger.debug(f"主爬取循环结束，总共执行了 {loop_count} 次")
         
         finally:
-            # 清理生成任务
+            # 确保请求生成任务完成
             if generation_task and not generation_task.done():
-                generation_task.cancel()
                 try:
                     await generation_task
                 except asyncio.CancelledError:
@@ -159,15 +184,24 @@ class Engine(object):
 
     async def _traditional_request_generation(self):
         """Traditional request generation method (compatible with older versions)"""
+        self.logger.debug("开始处理传统请求生成")
+        processed_count = 0
         while self.running:
             try:
                 start_request = next(self.start_requests)
+                self.logger.debug(f"获取到请求: {getattr(start_request, 'url', 'Unknown URL')}")
                 # 请求入队
                 await self.enqueue_request(start_request)
+                processed_count += 1
+                self.logger.debug(f"已处理请求数量: {processed_count}")
             except StopIteration:
+                self.logger.debug("所有起始请求处理完成")
                 self.start_requests = None
                 break
             except Exception as exp:
+                self.logger.error(f"处理请求时发生异常: {exp}")
+                import traceback
+                self.logger.error(traceback.format_exc())
                 # 1. All requests have been processed
                 # 2. Is scheduler idle
                 # 3. Is downloader idle
@@ -176,11 +210,13 @@ class Engine(object):
                 self.running = False
                 if self.start_requests is not None:
                     self.logger.error(f"Error occurred while starting request: {str(exp)}")
-            await asyncio.sleep(0.001)
+            # 减少等待时间以提高效率
+            await asyncio.sleep(0.00001)  # 从0.0001减少到0.00001
+        self.logger.debug(f"传统请求生成完成，总共处理了 {processed_count} 个请求")
 
     async def _controlled_request_generation(self):
         """Controlled request generation (enhanced features)"""
-        self.logger.info("Starting controlled request generation")
+        self.logger.debug("Starting controlled request generation")
         
         batch = []
         total_generated = 0
@@ -209,7 +245,7 @@ class Engine(object):
         
         finally:
             self.start_requests = None
-            self.logger.info(f"Request generation completed, total: {total_generated}")
+            self.logger.debug(f"Request generation completed, total: {total_generated}")
 
     async def _process_generation_batch(self, batch) -> int:
         """Process a batch of requests"""
@@ -221,14 +257,14 @@ class Engine(object):
             
             # 等待队列有空间
             while await self._is_queue_full() and self.running:
-                await asyncio.sleep(0.1)
+                await asyncio.sleep(0.01)  # 减少等待时间
             
             if self.running:
                 await self.enqueue_request(request)
                 generated += 1
                 self._generation_stats['total_generated'] += 1
             
-            # 控制生成速度
+            # 控制生成速度，但使用更小的间隔
             if self.generation_interval > 0:
                 await asyncio.sleep(self.generation_interval)
         
@@ -263,8 +299,8 @@ class Engine(object):
         self._generation_stats['backpressure_events'] += 1
         self.logger.debug("Backpressure triggered, pausing request generation")
         
-        wait_time = 0.1
-        max_wait = 2.0
+        wait_time = 0.01  # 减少初始等待时间
+        max_wait = 1.0  # 减少最大等待时间
         
         while await self._should_pause_generation() and self.running:
             await asyncio.sleep(wait_time)
@@ -272,16 +308,38 @@ class Engine(object):
 
     async def _open_spider(self):
         asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
-        crawling = asyncio.create_task(self.crawl())
-        await crawling
+        # 直接调用crawl方法而不是创建任务，确保等待完成
+        await self.crawl()
 
     async def _crawl(self, request):
-        # TODO 实现并发
         async def crawl_task():
-            outputs = await self._fetch(request)
-            # TODO 处理output
-            if outputs:
-                await self._handle_spider_output(outputs)
+            start_time = time.time()
+            try:
+                outputs = await self._fetch(request)
+                # 记录响应时间
+                response_time = time.time() - start_time
+                if self.task_manager:
+                    self.task_manager.record_response_time(response_time)
+                
+                # TODO 处理output
+                if outputs:
+                    await self._handle_spider_output(outputs)
+            except Exception as e:
+                # 记录详细的异常信息
+                self.logger.error(
+                    f"处理请求失败: {getattr(request, 'url', 'Unknown URL')} - {type(e).__name__}: {e}"
+                )
+                self.logger.debug(f"详细异常信息", exc_info=True)
+                
+                # 发送统计事件
+                if hasattr(self.crawler, 'stats'):
+                    self.crawler.stats.inc_value('downloader/exception_count')
+                    self.crawler.stats.inc_value(f'downloader/exception_type_count/{type(e).__name__}')
+                    if hasattr(request, 'url'):
+                        self.crawler.stats.inc_value(f'downloader/failed_urls_count')
+                
+                # 不再重新抛出异常，避免未处理的Task异常
+                return None
 
         # 使用异步任务创建，遵守并发限制
         await self.task_manager.create_task(crawl_task())
@@ -331,31 +389,47 @@ class Engine(object):
 
     async def _should_exit(self) -> bool:
         """检查是否应该退出"""
+        self.logger.debug(f"检查退出条件: start_requests={self.start_requests is not None}")
         # 没有启动请求，且所有队列都空闲
         if self.start_requests is None:
+            self.logger.debug("start_requests 为 None，检查其他组件状态")
             # 使用异步的idle检查方法以获得更精确的结果
             scheduler_idle = await self.scheduler.async_idle() if hasattr(self.scheduler, 'async_idle') else self.scheduler.idle()
+            downloader_idle = self.downloader.idle()
+            task_manager_done = self.task_manager.all_done()
+            processor_idle = self.processor.idle()
+            
+            self.logger.debug(f"组件状态 - Scheduler: {scheduler_idle}, Downloader: {downloader_idle}, TaskManager: {task_manager_done}, Processor: {processor_idle}")
             
             if (scheduler_idle and 
-                self.downloader.idle() and 
-                self.task_manager.all_done() and 
-                self.processor.idle()):
-                # 增加额外检查确保所有任务都完成
-                await asyncio.sleep(0.1)  # 短暂等待确保没有新的任务加入
-                if (await self.scheduler.async_idle() and 
-                    self.downloader.idle() and 
-                    self.task_manager.all_done() and 
-                    self.processor.idle()):
+                downloader_idle and 
+                task_manager_done and 
+                processor_idle):
+                # 立即进行二次检查，不等待
+                scheduler_idle = await self.scheduler.async_idle() if hasattr(self.scheduler, 'async_idle') else self.scheduler.idle()
+                downloader_idle = self.downloader.idle()
+                task_manager_done = self.task_manager.all_done()
+                processor_idle = self.processor.idle()
+                
+                self.logger.debug(f"二次检查组件状态 - Scheduler: {scheduler_idle}, Downloader: {downloader_idle}, TaskManager: {task_manager_done}, Processor: {processor_idle}")
+                
+                if (scheduler_idle and 
+                    downloader_idle and 
+                    task_manager_done and 
+                    processor_idle):
+                    self.logger.info("所有组件都空闲，准备退出")
                     return True
+        else:
+            self.logger.debug("start_requests 不为 None，不退出")
         
         return False
 
     async def close_spider(self):
+        # 不再调用crawler.close()，避免重复清理
+        # 清理工作应该由crawler的_lifecycle_manager上下文管理器来处理
         await asyncio.gather(*self.task_manager.current_task)
         await self.scheduler.close()
         await self.downloader.close()
-        if self.normal:
-            await self.crawler.close()
     
     def get_generation_stats(self) -> dict:
         """获取生成统计"""

crawlo/core/scheduler.py

@@ -1,14 +1,15 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
-from typing import Optional, Callable
 import traceback
+from typing import Optional, Callable
 
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import set_request
-from crawlo.utils.request_serializer import RequestSerializer
 from crawlo.utils.error_handler import ErrorHandler
+from crawlo.utils.class_loader import load_class
+from crawlo.project import common_call
+from crawlo.utils.request_serializer import RequestSerializer
 from crawlo.queue.queue_manager import QueueManager, QueueConfig, QueueType
-from crawlo.project import load_class, common_call
 
 
 class Scheduler: