crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/commands/startproject.py

@@ -7,18 +7,73 @@
 """
 import shutil
 import re
+import sys
+import os
 from pathlib import Path
-from rich.console import Console
-from rich.panel import Panel
-from rich.text import Text
+from typing import Optional
 
-from .utils import show_error_panel, show_success_panel
+# 添加项目根目录到路径，以便能够导入utils模块
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
 
-# 初始化 rich 控制台
-console = Console()
+try:
+    from rich.console import Console
+    from rich.panel import Panel
+    from rich.text import Text
+    from rich.table import Table
+    RICH_AVAILABLE = True
+except ImportError:
+    RICH_AVAILABLE = False
+
+try:
+    from .utils import show_error_panel, show_success_panel
+    UTILS_AVAILABLE = True
+except ImportError:
+    # 如果相对导入失败，尝试绝对导入
+    try:
+        from crawlo.commands.utils import show_error_panel, show_success_panel
+        UTILS_AVAILABLE = True
+    except ImportError:
+        UTILS_AVAILABLE = False
+
+# 初始化 rich 控制台（如果可用）
+if RICH_AVAILABLE:
+    console = Console()
+else:
+    # 简单的控制台输出替代
+    class Console:
+        def print(self, text):
+            print(text)
+    console = Console()
 
 TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
 
+# 可用的模板类型
+TEMPLATE_TYPES = {
+    'default': '默认模板 - 通用配置，适合大多数项目',
+    'simple': '简化模板 - 最小配置，适合快速开始',
+    'distributed': '分布式模板 - 针对分布式爬取优化',
+    'high-performance': '高性能模板 - 针对大规模高并发优化',
+    'gentle': '温和模板 - 低负载配置，对目标网站友好'
+}
+
+
+def show_error_panel(title, content):
+    """显示错误面板的简单实现"""
+    if RICH_AVAILABLE:
+        from rich.panel import Panel
+        console.print(Panel(content, title=title, border_style="red"))
+    else:
+        print(f"❌ {title}")
+        print(content)
+
+def show_success_panel(title, content):
+    """显示成功面板的简单实现"""
+    if RICH_AVAILABLE:
+        from rich.panel import Panel
+        console.print(Panel(content, title=title, border_style="green"))
+    else:
+        print(f"✅ {title}")
+        print(content)
 
 def _render_template(tmpl_path, context):
     """读取模板文件，替换 {{key}} 为 context 中的值"""
@@ -29,7 +84,7 @@ def _render_template(tmpl_path, context):
     return content
 
 
-def _copytree_with_templates(src, dst, context):
+def _copytree_with_templates(src, dst, context, template_type='default'):
     """
     递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
     """
@@ -45,7 +100,19 @@ def _copytree_with_templates(src, dst, context):
             dst_item.mkdir(parents=True, exist_ok=True)
         else:
             if item.suffix == '.tmpl':
-                rendered_content = _render_template(item, context)
+                # 处理特定模板类型的设置文件
+                if item.name == 'settings.py.tmpl' and template_type != 'default':
+                    # 使用特定模板类型的设置文件
+                    template_file_name = f'settings_{template_type}.py.tmpl'
+                    template_file_path = src_path / template_file_name
+                    if template_file_path.exists():
+                        rendered_content = _render_template(template_file_path, context)
+                    else:
+                        # 如果特定模板不存在，使用默认模板
+                        rendered_content = _render_template(item, context)
+                else:
+                    rendered_content = _render_template(item, context)
+                
                 final_dst = dst_item.with_suffix('')
                 final_dst.parent.mkdir(parents=True, exist_ok=True)
                 with open(final_dst, 'w', encoding='utf-8') as f:
@@ -100,16 +167,44 @@ def validate_project_name(project_name: str) -> tuple[bool, str]:
     return True, ""
 
 
+def show_template_options():
+    """显示可用的模板选项"""
+    if RICH_AVAILABLE:
+        table = Table(title="可用模板类型", show_header=True, header_style="bold magenta")
+        table.add_column("模板类型", style="cyan", no_wrap=True)
+        table.add_column("描述", style="green")
+        
+        for template_type, description in TEMPLATE_TYPES.items():
+            table.add_row(template_type, description)
+        
+        console.print(table)
+    else:
+        print("可用模板类型:")
+        for template_type, description in TEMPLATE_TYPES.items():
+            print(f"  {template_type}: {description}")
+
+
 def main(args):
-    if len(args) != 1:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name>")
+    if len(args) < 1 or len(args) > 2:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type]")
         console.print("💡 Examples:")
         console.print("   [blue]crawlo startproject[/blue] my_spider_project")
-        console.print("   [blue]crawlo startproject[/blue] news_crawler")
-        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider")
+        console.print("   [blue]crawlo startproject[/blue] news_crawler simple")
+        console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed")
+        show_template_options()
         return 1
 
     project_name = args[0]
+    template_type = args[1] if len(args) > 1 else 'default'
+    
+    # 验证模板类型
+    if template_type not in TEMPLATE_TYPES:
+        show_error_panel(
+            "Invalid Template Type",
+            f"Template type '[cyan]{template_type}[/cyan]' is not supported.\n"
+        )
+        show_template_options()
+        return 1
     
     # 验证项目名称
     is_valid, error_msg = validate_project_name(project_name)
@@ -154,7 +249,7 @@ def main(args):
 
         # 3. 复制并渲染项目包内容
         package_dir = project_dir / project_name
-        _copytree_with_templates(template_dir, package_dir, context)
+        _copytree_with_templates(template_dir, package_dir, context, template_type)
         console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
 
         # 4. 创建 logs 目录
@@ -168,6 +263,10 @@ def main(args):
         # 成功面板
         success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
         console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
+        
+        # 显示使用的模板类型
+        if template_type != 'default':
+            console.print(f":information: 使用模板类型: [bold blue]{template_type}[/bold blue] - {TEMPLATE_TYPES[template_type]}")
 
         # 下一步操作提示（对齐美观 + 语法高亮）
         next_steps = f"""
@@ -194,3 +293,8 @@ def main(args):
             shutil.rmtree(project_dir, ignore_errors=True)
             console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
         return 1
+
+if __name__ == "__main__":
+    import sys
+    exit_code = main(sys.argv[1:])
+    sys.exit(exit_code)

crawlo/config.py

@@ -23,6 +23,7 @@ from typing import Dict, Any, Optional, Union
 import os
 from crawlo.mode_manager import ModeManager, standalone_mode, distributed_mode, auto_mode, from_env
 from crawlo.utils.log import get_logger
+from crawlo.config_validator import validate_config
 
 
 class CrawloConfig:
@@ -31,6 +32,19 @@ class CrawloConfig:
     def __init__(self, settings: Dict[str, Any]):
         self.settings = settings
         self.logger = get_logger(self.__class__.__name__)
+        # 验证配置
+        self._validate_settings()
+    
+    def _validate_settings(self):
+        """验证配置"""
+        is_valid, errors, warnings = validate_config(self.settings)
+        if not is_valid:
+            error_msg = "配置验证失败:\n" + "\n".join([f"  - {error}" for error in errors])
+            raise ValueError(error_msg)
+        
+        if warnings:
+            warning_msg = "配置警告:\n" + "\n".join([f"  - {warning}" for warning in warnings])
+            self.logger.warning(warning_msg)
     
     def get(self, key: str, default: Any = None) -> Any:
         """获取配置项"""
@@ -100,6 +114,22 @@ class CrawloConfig:
         print("=" * 50)
         return self
     
+    def validate(self) -> bool:
+        """验证当前配置"""
+        is_valid, errors, warnings = validate_config(self.settings)
+        if not is_valid:
+            print("配置验证失败:")
+            for error in errors:
+                print(f"  - {error}")
+            return False
+        
+        if warnings:
+            print("配置警告:")
+            for warning in warnings:
+                print(f"  - {warning}")
+        
+        return True
+    
     # ==================== 静态工厂方法 ====================
     
     @staticmethod

crawlo/config_validator.py

@@ -0,0 +1,253 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+配置验证器
+==========
+提供配置项的验证和默认值设置功能，确保配置的合理性和一致性。
+"""
+
+from typing import Dict, Any, List, Tuple, Optional
+import re
+from crawlo.utils.log import get_logger
+
+
+class ConfigValidator:
+    """配置验证器"""
+    
+    def __init__(self):
+        self.logger = get_logger(self.__class__.__name__)
+        self.errors = []
+        self.warnings = []
+    
+    def validate(self, config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
+        """
+        验证配置
+        
+        Args:
+            config: 配置字典
+            
+        Returns:
+            Tuple[bool, List[str], List[str]]: (是否有效, 错误列表, 警告列表)
+        """
+        self.errors = []
+        self.warnings = []
+        
+        # 验证各个配置项
+        self._validate_basic_settings(config)
+        self._validate_network_settings(config)
+        self._validate_concurrency_settings(config)
+        self._validate_queue_settings(config)
+        self._validate_storage_settings(config)
+        self._validate_redis_settings(config)
+        self._validate_middleware_settings(config)
+        self._validate_pipeline_settings(config)
+        self._validate_logging_settings(config)
+        
+        is_valid = len(self.errors) == 0
+        return is_valid, self.errors, self.warnings
+    
+    def _validate_basic_settings(self, config: Dict[str, Any]):
+        """验证基本设置"""
+        project_name = config.get('PROJECT_NAME', 'crawlo')
+        if not isinstance(project_name, str) or not project_name.strip():
+            self.errors.append("PROJECT_NAME 必须是非空字符串")
+        
+        version = config.get('VERSION', '1.0')
+        if not isinstance(version, str):
+            self.errors.append("VERSION 必须是字符串")
+    
+    def _validate_network_settings(self, config: Dict[str, Any]):
+        """验证网络设置"""
+        # 下载器验证
+        downloader = config.get('DOWNLOADER', 'crawlo.downloader.aiohttp_downloader.AioHttpDownloader')
+        if not isinstance(downloader, str):
+            self.errors.append("DOWNLOADER 必须是字符串")
+        
+        # 超时设置验证
+        timeout = config.get('DOWNLOAD_TIMEOUT', 30)
+        if not isinstance(timeout, (int, float)) or timeout <= 0:
+            self.errors.append("DOWNLOAD_TIMEOUT 必须是正数")
+        
+        # 延迟设置验证
+        delay = config.get('DOWNLOAD_DELAY', 1.0)
+        if not isinstance(delay, (int, float)) or delay < 0:
+            self.errors.append("DOWNLOAD_DELAY 必须是非负数")
+        
+        # 重试次数验证
+        max_retries = config.get('MAX_RETRY_TIMES', 3)
+        if not isinstance(max_retries, int) or max_retries < 0:
+            self.errors.append("MAX_RETRY_TIMES 必须是非负整数")
+        
+        # 连接池限制验证
+        pool_limit = config.get('CONNECTION_POOL_LIMIT', 50)
+        if not isinstance(pool_limit, int) or pool_limit <= 0:
+            self.errors.append("CONNECTION_POOL_LIMIT 必须是正整数")
+    
+    def _validate_concurrency_settings(self, config: Dict[str, Any]):
+        """验证并发设置"""
+        concurrency = config.get('CONCURRENCY', 8)
+        if not isinstance(concurrency, int) or concurrency <= 0:
+            self.errors.append("CONCURRENCY 必须是正整数")
+        
+        max_running_spiders = config.get('MAX_RUNNING_SPIDERS', 3)
+        if not isinstance(max_running_spiders, int) or max_running_spiders <= 0:
+            self.errors.append("MAX_RUNNING_SPIDERS 必须是正整数")
+    
+    def _validate_queue_settings(self, config: Dict[str, Any]):
+        """验证队列设置"""
+        queue_type = config.get('QUEUE_TYPE', 'memory')
+        valid_queue_types = ['memory', 'redis', 'auto']
+        if queue_type not in valid_queue_types:
+            self.errors.append(f"QUEUE_TYPE 必须是以下值之一: {valid_queue_types}")
+        
+        # 队列大小验证
+        max_queue_size = config.get('SCHEDULER_MAX_QUEUE_SIZE', 2000)
+        if not isinstance(max_queue_size, int) or max_queue_size <= 0:
+            self.errors.append("SCHEDULER_MAX_QUEUE_SIZE 必须是正整数")
+        
+        # 队列名称验证（如果是Redis队列）
+        if queue_type == 'redis':
+            queue_name = config.get('SCHEDULER_QUEUE_NAME', '')
+            if not queue_name:
+                self.errors.append("使用Redis队列时，SCHEDULER_QUEUE_NAME 不能为空")
+            elif not self._is_valid_redis_key(queue_name):
+                self.warnings.append(f"Redis队列名称 '{queue_name}' 不符合命名规范，建议使用 'crawlo:{config.get('PROJECT_NAME', 'project')}:queue:requests' 格式")
+    
+    def _validate_storage_settings(self, config: Dict[str, Any]):
+        """验证存储设置"""
+        # MySQL设置验证
+        mysql_host = config.get('MYSQL_HOST')
+        if mysql_host is not None and not isinstance(mysql_host, str):
+            self.errors.append("MYSQL_HOST 必须是字符串")
+        
+        mysql_port = config.get('MYSQL_PORT')
+        if mysql_port is not None and (not isinstance(mysql_port, int) or mysql_port <= 0 or mysql_port > 65535):
+            self.errors.append("MYSQL_PORT 必须是1-65535之间的整数")
+        
+        # MongoDB设置验证
+        mongo_uri = config.get('MONGO_URI')
+        if mongo_uri is not None and not isinstance(mongo_uri, str):
+            self.errors.append("MONGO_URI 必须是字符串")
+    
+    def _validate_redis_settings(self, config: Dict[str, Any]):
+        """验证Redis设置"""
+        queue_type = config.get('QUEUE_TYPE', 'memory')
+        if queue_type == 'redis':
+            # Redis主机验证
+            redis_host = config.get('REDIS_HOST', '127.0.0.1')
+            if not isinstance(redis_host, str) or not redis_host.strip():
+                self.errors.append("REDIS_HOST 必须是非空字符串")
+            
+            # Redis端口验证
+            redis_port = config.get('REDIS_PORT', 6379)
+            if not isinstance(redis_port, int) or redis_port <= 0 or redis_port > 65535:
+                self.errors.append("REDIS_PORT 必须是1-65535之间的整数")
+            
+            # Redis URL验证
+            redis_url = config.get('REDIS_URL')
+            if redis_url is not None and not isinstance(redis_url, str):
+                self.errors.append("REDIS_URL 必须是字符串")
+    
+    def _validate_middleware_settings(self, config: Dict[str, Any]):
+        """验证中间件设置"""
+        middlewares = config.get('MIDDLEWARES', [])
+        if not isinstance(middlewares, list):
+            self.errors.append("MIDDLEWARES 必须是列表")
+        else:
+            for i, middleware in enumerate(middlewares):
+                if not isinstance(middleware, str):
+                    self.errors.append(f"MIDDLEWARES[{i}] 必须是字符串")
+    
+    def _validate_pipeline_settings(self, config: Dict[str, Any]):
+        """验证管道设置"""
+        pipelines = config.get('PIPELINES', [])
+        if not isinstance(pipelines, list):
+            self.errors.append("PIPELINES 必须是列表")
+        else:
+            for i, pipeline in enumerate(pipelines):
+                if not isinstance(pipeline, str):
+                    self.errors.append(f"PIPELINES[{i}] 必须是字符串")
+    
+    def _validate_logging_settings(self, config: Dict[str, Any]):
+        """验证日志设置"""
+        log_level = config.get('LOG_LEVEL', 'INFO')
+        valid_log_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        if log_level not in valid_log_levels:
+            self.errors.append(f"LOG_LEVEL 必须是以下值之一: {valid_log_levels}")
+        
+        log_file = config.get('LOG_FILE')
+        if log_file is not None and not isinstance(log_file, str):
+            self.errors.append("LOG_FILE 必须是字符串")
+    
+    def _is_valid_redis_key(self, key: str) -> bool:
+        """检查Redis key是否符合命名规范"""
+        # 检查是否以 crawlo: 开头
+        if not key.startswith('crawlo:'):
+            return False
+        
+        # 检查是否包含必要的部分
+        parts = key.split(':')
+        if len(parts) < 3:
+            return False
+        
+        # 检查是否包含 queue 部分
+        return 'queue' in parts
+    
+    def get_validation_report(self, config: Dict[str, Any]) -> str:
+        """
+        获取验证报告
+        
+        Args:
+            config: 配置字典
+            
+        Returns:
+            str: 验证报告
+        """
+        is_valid, errors, warnings = self.validate(config)
+        
+        report = []
+        report.append("=" * 50)
+        report.append("配置验证报告")
+        report.append("=" * 50)
+        
+        if is_valid:
+            report.append("✅ 配置验证通过")
+        else:
+            report.append("❌ 配置验证失败")
+            report.append("错误:")
+            for error in errors:
+                report.append(f"  - {error}")
+        
+        if warnings:
+            report.append("警告:")
+            for warning in warnings:
+                report.append(f"  - {warning}")
+        
+        report.append("=" * 50)
+        return "\n".join(report)
+
+
+# 便利函数
+def validate_config(config: Dict[str, Any]) -> Tuple[bool, List[str], List[str]]:
+    """
+    验证配置
+    
+    Args:
+        config: 配置字典
+        
+    Returns:
+        Tuple[bool, List[str], List[str]]: (是否有效, 错误列表, 警告列表)
+    """
+    validator = ConfigValidator()
+    return validator.validate(config)
+
+
+def print_validation_report(config: Dict[str, Any]):
+    """
+    打印验证报告
+    
+    Args:
+        config: 配置字典
+    """
+    validator = ConfigValidator()
+    print(validator.get_validation_report(config))