crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/commands/utils.py

@@ -0,0 +1,187 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+命令行工具公共模块
+提供命令行工具的公共函数和工具
+"""
+import sys
+import configparser
+from pathlib import Path
+from importlib import import_module
+from typing import Optional, Tuple
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+console = Console()
+
+
+def get_project_root() -> Optional[Path]:
+    """
+    自动检测项目根目录：从当前目录向上查找 crawlo.cfg
+    
+    Returns:
+        Path: 项目根目录路径，如果未找到返回 None
+    """
+    current = Path.cwd()
+    for _ in range(10):  # 最多向上查找10层
+        cfg_file = current / "crawlo.cfg"
+        if cfg_file.exists():
+            return current
+        if current == current.parent:
+            break
+        current = current.parent
+    return None
+
+
+def validate_project_environment() -> Tuple[bool, Optional[str], Optional[str]]:
+    """
+    验证项目环境，确保在正确的 Crawlo 项目中
+    
+    Returns:
+        Tuple[bool, Optional[str], Optional[str]]: 
+        (是否有效, 项目包名, 错误信息)
+    """
+    # 1. 查找项目根目录
+    project_root = get_project_root()
+    if not project_root:
+        return False, None, "Cannot find 'crawlo.cfg'. Run this command inside your project directory."
+    
+    # 2. 将项目根加入 Python 路径
+    project_root_str = str(project_root)
+    if project_root_str not in sys.path:
+        sys.path.insert(0, project_root_str)
+    
+    # 3. 读取配置文件
+    cfg_file = project_root / "crawlo.cfg"
+    config = configparser.ConfigParser()
+    
+    try:
+        config.read(cfg_file, encoding="utf-8")
+    except Exception as e:
+        return False, None, f"Failed to read crawlo.cfg: {e}"
+    
+    if not config.has_section("settings") or not config.has_option("settings", "default"):
+        return False, None, "Invalid crawlo.cfg: missing [settings] section or 'default' option"
+    
+    # 4. 获取项目包名
+    settings_module = config.get("settings", "default")
+    project_package = settings_module.split(".")[0]
+    
+    # 5. 验证项目包是否可导入
+    try:
+        import_module(project_package)
+    except ImportError as e:
+        return False, None, f"Failed to import project package '{project_package}': {e}"
+    
+    return True, project_package, None
+
+
+def show_error_panel(title: str, message: str, show_json: bool = False) -> None:
+    """
+    显示错误面板或JSON格式错误
+    
+    Args:
+        title: 错误标题
+        message: 错误消息
+        show_json: 是否以JSON格式输出
+    """
+    if show_json:
+        console.print_json(data={"success": False, "error": message})
+    else:
+        console.print(Panel(
+            Text.from_markup(f":cross_mark: [bold red]{message}[/bold red]"),
+            title=f"❌ {title}",
+            border_style="red",
+            padding=(1, 2)
+        ))
+
+
+def show_success_panel(title: str, message: str, show_json: bool = False, data: dict = None) -> None:
+    """
+    显示成功面板或JSON格式结果
+    
+    Args:
+        title: 成功标题
+        message: 成功消息
+        show_json: 是否以JSON格式输出
+        data: JSON数据（当show_json=True时）
+    """
+    if show_json:
+        result = {"success": True, "message": message}
+        if data:
+            result.update(data)
+        console.print_json(data=result)
+    else:
+        console.print(Panel(
+            Text.from_markup(f":white_check_mark: [bold green]{message}[/bold green]"),
+            title=f"✅ {title}",
+            border_style="green",
+            padding=(1, 2)
+        ))
+
+
+def validate_spider_name(spider_name: str) -> bool:
+    """
+    验证爬虫名称是否符合规范
+    
+    Args:
+        spider_name: 爬虫名称
+        
+    Returns:
+        bool: 是否有效
+    """
+    import re
+    # 爬虫名称应该是有效的Python标识符
+    return spider_name.isidentifier() and re.match(r'^[a-z][a-z0-9_]*$', spider_name)
+
+
+def format_file_size(size_bytes: int) -> str:
+    """
+    格式化文件大小
+    
+    Args:
+        size_bytes: 字节数
+        
+    Returns:
+        str: 格式化后的大小字符串
+    """
+    for unit in ['B', 'KB', 'MB', 'GB']:
+        if size_bytes < 1024.0:
+            return f"{size_bytes:.1f} {unit}"
+        size_bytes /= 1024.0
+    return f"{size_bytes:.1f} TB"
+
+
+def truncate_text(text: str, max_length: int = 80) -> str:
+    """
+    截断过长的文本
+    
+    Args:
+        text: 原始文本
+        max_length: 最大长度
+        
+    Returns:
+        str: 截断后的文本
+    """
+    if len(text) <= max_length:
+        return text
+    return text[:max_length-3] + "..."
+
+
+def is_valid_domain(domain: str) -> bool:
+    """
+    验证域名格式是否正确
+    
+    Args:
+        domain: 域名
+        
+    Returns:
+        bool: 是否有效
+    """
+    import re
+    pattern = re.compile(
+        r'^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)*[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$'
+    )
+    return bool(pattern.match(domain))

crawlo/config.py

@@ -0,0 +1,280 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo 配置工厂
+===============
+提供优雅的配置方式，让用户能够轻松选择运行模式。
+
+使用示例：
+    # 单机模式（默认）
+    config = CrawloConfig.standalone()
+    
+    # 分布式模式
+    config = CrawloConfig.distributed(redis_host='192.168.1.100')
+    
+    # 自动检测模式
+    config = CrawloConfig.auto()
+    
+    # 从环境变量
+    config = CrawloConfig.from_env()
+"""
+
+from typing import Dict, Any, Optional, Union
+import os
+from crawlo.mode_manager import ModeManager, standalone_mode, distributed_mode, auto_mode, from_env
+from crawlo.utils.log import get_logger
+
+
+class CrawloConfig:
+    """Crawlo 配置工厂类"""
+    
+    def __init__(self, settings: Dict[str, Any]):
+        self.settings = settings
+        self.logger = get_logger(self.__class__.__name__)
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        """获取配置项"""
+        return self.settings.get(key, default)
+    
+    def set(self, key: str, value: Any) -> 'CrawloConfig':
+        """设置配置项（链式调用）"""
+        self.settings[key] = value
+        return self
+    
+    def update(self, settings: Dict[str, Any]) -> 'CrawloConfig':
+        """更新配置（链式调用）"""
+        self.settings.update(settings)
+        return self
+    
+    def set_concurrency(self, concurrency: int) -> 'CrawloConfig':
+        """设置并发数"""
+        return self.set('CONCURRENCY', concurrency)
+    
+    def set_delay(self, delay: float) -> 'CrawloConfig':
+        """设置请求延迟"""
+        return self.set('DOWNLOAD_DELAY', delay)
+    
+    def enable_debug(self) -> 'CrawloConfig':
+        """启用调试模式"""
+        return self.set('LOG_LEVEL', 'DEBUG')
+    
+    def enable_mysql(self) -> 'CrawloConfig':
+        """启用 MySQL 存储"""
+        pipelines = self.get('PIPELINES', [])
+        if 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline' not in pipelines:
+            pipelines.append('crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline')
+        return self.set('PIPELINES', pipelines)
+    
+    def set_redis_host(self, host: str) -> 'CrawloConfig':
+        """设置 Redis 主机"""
+        return self.set('REDIS_HOST', host)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return self.settings.copy()
+    
+    def print_summary(self) -> 'CrawloConfig':
+        """打印配置摘要"""
+        mode_info = {
+            'memory': '🏠 单机模式',
+            'redis': '🌐 分布式模式', 
+            'auto': '🤖 自动检测模式'
+        }
+        
+        queue_type = self.settings.get('QUEUE_TYPE', 'memory')
+        filter_class = self.settings.get('FILTER_CLASS', '').split('.')[-1]
+        concurrency = self.settings.get('CONCURRENCY', 8)
+        
+        print("=" * 50)
+        print(f"📋 Crawlo 配置摘要")
+        print("=" * 50)
+        print(f"运行模式: {mode_info.get(queue_type, queue_type)}")
+        print(f"队列类型: {queue_type}")
+        print(f"去重方式: {filter_class}")
+        print(f"并发数量: {concurrency}")
+        
+        if queue_type == 'redis':
+            redis_host = self.settings.get('REDIS_HOST', 'localhost')
+            print(f"Redis 服务器: {redis_host}")
+        
+        print("=" * 50)
+        return self
+    
+    # ==================== 静态工厂方法 ====================
+    
+    @staticmethod
+    def standalone(
+        concurrency: int = 8,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建单机模式配置
+        
+        Args:
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = standalone_mode(
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def distributed(
+        redis_host: str = '127.0.0.1',
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        project_name: str = 'crawlo',
+        concurrency: int = 16,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建分布式模式配置
+        
+        Args:
+            redis_host: Redis 服务器地址
+            redis_port: Redis 端口
+            redis_password: Redis 密码
+            project_name: 项目名称（用于命名空间）
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = distributed_mode(
+            redis_host=redis_host,
+            redis_port=redis_port,
+            redis_password=redis_password,
+            project_name=project_name,
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def auto(
+        concurrency: int = 12,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建自动检测模式配置
+        
+        Args:
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = auto_mode(
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def from_env(default_mode: str = 'standalone') -> 'CrawloConfig':
+        """
+        从环境变量创建配置
+        
+        支持的环境变量：
+        - CRAWLO_MODE: 运行模式 (standalone/distributed/auto)
+        - REDIS_HOST: Redis 主机
+        - REDIS_PORT: Redis 端口
+        - REDIS_PASSWORD: Redis 密码
+        - CONCURRENCY: 并发数
+        - PROJECT_NAME: 项目名称
+        """
+        settings = from_env(default_mode)
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def custom(settings: Dict[str, Any]) -> 'CrawloConfig':
+        """
+        创建自定义配置
+        
+        Args:
+            settings: 自定义配置字典
+        """
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def presets() -> 'Presets':
+        """获取预设配置对象"""
+        return Presets()
+
+
+# ==================== 便利函数 ====================
+
+def create_config(
+    mode: str = 'standalone',
+    **kwargs
+) -> CrawloConfig:
+    """
+    便利函数：创建配置
+    
+    Args:
+        mode: 运行模式 ('standalone', 'distributed', 'auto')
+        **kwargs: 配置参数
+    """
+    if mode.lower() == 'standalone':
+        return CrawloConfig.standalone(**kwargs)
+    elif mode.lower() == 'distributed':
+        return CrawloConfig.distributed(**kwargs)
+    elif mode.lower() == 'auto':
+        return CrawloConfig.auto(**kwargs)
+    else:
+        raise ValueError(f"不支持的运行模式: {mode}")
+
+
+# ==================== 预设配置 ====================
+
+class Presets:
+    """预设配置类"""
+    
+    @staticmethod
+    def development() -> CrawloConfig:
+        """开发环境配置"""
+        return CrawloConfig.standalone(
+            concurrency=4,
+            download_delay=2.0,
+            LOG_LEVEL='DEBUG',
+            STATS_DUMP=True
+        )
+    
+    @staticmethod
+    def production() -> CrawloConfig:
+        """生产环境配置"""
+        return CrawloConfig.auto(
+            concurrency=16,
+            download_delay=1.0,
+            LOG_LEVEL='INFO',
+            RETRY_TIMES=5
+        )
+    
+    @staticmethod
+    def large_scale(redis_host: str, project_name: str) -> CrawloConfig:
+        """大规模分布式配置"""
+        return CrawloConfig.distributed(
+            redis_host=redis_host,
+            project_name=project_name,
+            concurrency=32,
+            download_delay=0.5,
+            SCHEDULER_MAX_QUEUE_SIZE=10000,
+            LARGE_SCALE_BATCH_SIZE=2000
+        )
+    
+    @staticmethod
+    def gentle() -> CrawloConfig:
+        """温和模式配置（避免被封）"""
+        return CrawloConfig.standalone(
+            concurrency=2,
+            download_delay=3.0,
+            RANDOMNESS=True,
+            RANDOM_RANGE=(2.0, 5.0)
+        )

crawlo/core/engine.py

@@ -34,6 +34,19 @@ class Engine(object):
         self.logger = get_logger(name=self.__class__.__name__)
 
     def _get_downloader_cls(self):
+        """获取下载器类，支持多种配置方式"""
+        # 方式1: 使用 DOWNLOADER_TYPE 简化名称（推荐）
+        downloader_type = self.settings.get('DOWNLOADER_TYPE')
+        if downloader_type:
+            try:
+                from crawlo.downloader import get_downloader_class
+                downloader_cls = get_downloader_class(downloader_type)
+                self.logger.debug(f"使用下载器类型: {downloader_type} -> {downloader_cls.__name__}")
+                return downloader_cls
+            except (ImportError, ValueError) as e:
+                self.logger.warning(f"无法使用下载器类型 '{downloader_type}': {e}，回退到默认配置")
+        
+        # 方式2: 使用 DOWNLOADER 完整类路径（兼容旧版本）
         downloader_cls = load_class(self.settings.get('DOWNLOADER'))
         if not issubclass(downloader_cls, DownloaderBase):
             raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
@@ -51,7 +64,7 @@ class Engine(object):
 
         self.scheduler = Scheduler.create_instance(self.crawler)
         if hasattr(self.scheduler, 'open'):
-            self.scheduler.open()
+            await self.scheduler.open()
 
         downloader_cls = self._get_downloader_cls()
         self.downloader = downloader_cls(self.crawler)
@@ -105,8 +118,8 @@ class Engine(object):
             if outputs:
                 await self._handle_spider_output(outputs)
 
-        # asyncio.create_task(crawl_task())
-        self.task_manager.create_task(crawl_task())
+        # 使用异步任务创建，遵守并发限制
+        await self.task_manager.create_task(crawl_task())
 
     async def _fetch(self, request):
         async def _successful(_response):

crawlo/core/enhanced_engine.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+增强的引擎实现
+解决大规模请求生成时的并发控制和背压问题
+"""
+import asyncio
+
+from crawlo.core.engine import Engine as BaseEngine
+from crawlo.utils.log import get_logger
+
+
+class EnhancedEngine(BaseEngine):
+    """
+    增强的引擎实现
+    
+    主要改进：
+    1. 智能的请求生成控制
+    2. 背压感知的调度
+    3. 动态并发调整
+    """
+    
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        
+        # 增强控制参数
+        self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
+        self.generation_batch_size = 10
+        self.generation_interval = 0.05
+        self.backpressure_ratio = 0.8  # 队列达到80%时启动背压
+        
+        # 状态跟踪
+        self._generation_paused = False
+        self._last_generation_time = 0
+        self._generation_stats = {
+            'total_generated': 0,
+            'backpressure_events': 0
+        }
+        
+        self.logger = get_logger(self.__class__.__name__)
+    
+    async def crawl(self):
+        """
+        增强的爬取循环
+        支持智能请求生成和背压控制
+        """
+        generation_task = None
+        
+        try:
+            # 启动请求生成任务
+            if self.start_requests:
+                generation_task = asyncio.create_task(
+                    self._controlled_request_generation()
+                )
+            
+            # 主爬取循环
+            while self.running:
+                # 获取并处理请求
+                if request := await self._get_next_request():
+                    await self._crawl(request)
+                
+                # 检查退出条件
+                if await self._should_exit():
+                    break
+                
+                # 短暂休息避免忙等
+                await asyncio.sleep(0.001)
+        
+        finally:
+            # 清理生成任务
+            if generation_task and not generation_task.done():
+                generation_task.cancel()
+                try:
+                    await generation_task
+                except asyncio.CancelledError:
+                    pass
+            
+            await self.close_spider()
+    
+    async def _controlled_request_generation(self):
+        """受控的请求生成"""
+        self.logger.info("🎛️ 启动受控请求生成")
+        
+        batch = []
+        total_generated = 0
+        
+        try:
+            for request in self.start_requests:
+                batch.append(request)
+                
+                # 批量处理
+                if len(batch) >= self.generation_batch_size:
+                    generated = await self._process_generation_batch(batch)
+                    total_generated += generated
+                    batch = []
+                
+                # 背压检查
+                if await self._should_pause_generation():
+                    await self._wait_for_capacity()
+            
+            # 处理剩余请求
+            if batch:
+                generated = await self._process_generation_batch(batch)
+                total_generated += generated
+        
+        except Exception as e:
+            self.logger.error(f"❌ 请求生成失败: {e}")
+        
+        finally:
+            self.start_requests = None
+            self.logger.info(f"🎉 请求生成完成，总计: {total_generated}")
+    
+    async def _process_generation_batch(self, batch) -> int:
+        """处理一批请求"""
+        generated = 0
+        
+        for request in batch:
+            if not self.running:
+                break
+            
+            # 等待队列有空间
+            while await self._is_queue_full() and self.running:
+                await asyncio.sleep(0.1)
+            
+            if self.running:
+                await self.enqueue_request(request)
+                generated += 1
+                self._generation_stats['total_generated'] += 1
+            
+            # 控制生成速度
+            if self.generation_interval > 0:
+                await asyncio.sleep(self.generation_interval)
+        
+        return generated
+    
+    async def _should_pause_generation(self) -> bool:
+        """判断是否应该暂停生成"""
+        # 检查队列大小
+        if await self._is_queue_full():
+            return True
+        
+        # 检查任务管理器负载
+        if self.task_manager:
+            current_tasks = len(self.task_manager.current_task)
+            if hasattr(self.task_manager, 'semaphore'):
+                max_concurrency = getattr(self.task_manager.semaphore, '_initial_value', 8)
+                if current_tasks >= max_concurrency * self.backpressure_ratio:
+                    return True
+        
+        return False
+    
+    async def _is_queue_full(self) -> bool:
+        """检查队列是否已满"""
+        if not self.scheduler:
+            return False
+        
+        queue_size = len(self.scheduler)
+        return queue_size >= self.max_queue_size * self.backpressure_ratio
+    
+    async def _wait_for_capacity(self):
+        """等待系统有足够容量"""
+        self._generation_stats['backpressure_events'] += 1
+        self.logger.debug("⏸️ 触发背压，暂停请求生成")
+        
+        wait_time = 0.1
+        max_wait = 2.0
+        
+        while await self._should_pause_generation() and self.running:
+            await asyncio.sleep(wait_time)
+            wait_time = min(wait_time * 1.1, max_wait)
+    
+    async def _should_exit(self) -> bool:
+        """检查是否应该退出"""
+        # 没有启动请求，且所有队列都空闲
+        if (self.start_requests is None and 
+            self.scheduler.idle() and 
+            self.downloader.idle() and 
+            self.task_manager.all_done() and 
+            self.processor.idle()):
+            return True
+        
+        return False
+    
+    def get_generation_stats(self) -> dict:
+        """获取生成统计"""
+        return {
+            **self._generation_stats,
+            'queue_size': len(self.scheduler) if self.scheduler else 0,
+            'active_tasks': len(self.task_manager.current_task) if self.task_manager else 0
+        }