crawlo 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

crawlo/commands/genspider.py

@@ -1,126 +1,152 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-08-31 22:36
-# @Author  : crawl-coder
-# @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
-"""
-import sys
-from pathlib import Path
-import configparser
-import importlib
-from rich.console import Console
-
-# 初始化 rich 控制台
-console = Console()
-
-TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
-
-
-def _render_template(tmpl_path, context):
-    """读取模板文件，替换 {{key}} 为 context 中的值"""
-    with open(tmpl_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-    for key, value in context.items():
-        content = content.replace(f'{{{{{key}}}}}', str(value))
-    return content
-
-
-def main(args):
-    if len(args) < 2:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo genspider[/blue] <spider_name> <domain>")
-        return 1
-
-    spider_name = args[0]
-    domain = args[1]
-
-    # 查找项目根目录
-    project_root = None
-    current = Path.cwd()
-    while True:
-        cfg_file = current / 'crawlo.cfg'
-        if cfg_file.exists():
-            project_root = current
-            break
-        parent = current.parent
-        if parent == current:
-            break
-        current = parent
-
-    if not project_root:
-        console.print("[bold red]:cross_mark: Error:[/bold red] Not a crawlo project. [cyan]crawlo.cfg[/cyan] not found.")
-        return 1
-
-    # 将项目根目录加入 sys.path
-    if str(project_root) not in sys.path:
-        sys.path.insert(0, str(project_root))
-
-    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
-    config = configparser.ConfigParser()
-    try:
-        config.read(cfg_file, encoding='utf-8')
-        settings_module = config.get('settings', 'default')
-        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
-    except Exception as e:
-        console.print(f"[bold red]:cross_mark: Error reading crawlo.cfg:[/bold red] {e}")
-        return 1
-
-    # 确定 items 模块的路径
-    items_module_path = f"{project_package}.items"
-
-    # 尝试导入 items 模块
-    default_item_class = "ExampleItem"  # 默认回退
-    try:
-        items_module = importlib.import_module(items_module_path)
-        # 获取模块中所有大写开头的类
-        item_classes = [
-            cls for cls in items_module.__dict__.values()
-            if isinstance(cls, type) and cls.__name__[0].isupper()  # 首字母大写
-        ]
-
-        if item_classes:
-            default_item_class = item_classes[0].__name__
-        else:
-            console.print("[yellow]:warning: Warning:[/yellow] No item class found in [cyan]items.py[/cyan], using [green]ExampleItem[/green].")
-
-    except ImportError as e:
-        console.print(f"[yellow]:warning: Warning:[/yellow] Failed to import [cyan]{items_module_path}[/cyan]: {e}")
-        # 仍使用默认 ExampleItem，不中断流程
-
-    # 创建爬虫文件
-    spiders_dir = project_root / project_package / 'spiders'
-    spiders_dir.mkdir(parents=True, exist_ok=True)
-
-    spider_file = spiders_dir / f'{spider_name}.py'
-    if spider_file.exists():
-        console.print(f"[bold red]:cross_mark: Error:[/bold red] Spider '[cyan]{spider_name}[/cyan]' already exists at [green]{spider_file}[/green]")
-        return 1
-
-    # 模板路径
-    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
-    if not tmpl_path.exists():
-        console.print(f"[bold red]:cross_mark: Error:[/bold red] Template file not found at [cyan]{tmpl_path}[/cyan]")
-        return 1
-
-    # 生成类名
-    class_name = f"{spider_name.capitalize()}Spider"
-
-    context = {
-        'spider_name': spider_name,
-        'domain': domain,
-        'project_name': project_package,
-        'item_class': default_item_class,
-        'class_name': class_name
-    }
-
-    content = _render_template(tmpl_path, context)
-
-    with open(spider_file, 'w', encoding='utf-8') as f:
-        f.write(content)
-
-    console.print(f":white_check_mark: [green]Spider '[bold]{spider_name}[/bold]' created successfully![/green]")
-    console.print(f"  → Location: [cyan]{spider_file}[/cyan]")
-    console.print("\n[bold]Next step:[/bold]")
-    console.print(f"  [blue]crawlo run[/blue] {spider_name}")
-
-    return 0
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
+"""
+import sys
+from pathlib import Path
+import configparser
+import importlib
+from rich.console import Console
+
+from .utils import (
+    get_project_root, 
+    validate_project_environment, 
+    show_error_panel, 
+    show_success_panel,
+    validate_spider_name,
+    is_valid_domain
+)
+
+# 初始化 rich 控制台
+console = Console()
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo genspider[/blue] <spider_name> <domain>")
+        console.print("💡 Examples:")
+        console.print("   [blue]crawlo genspider[/blue] news_spider news.example.com")
+        console.print("   [blue]crawlo genspider[/blue] product_spider shop.example.com")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+    
+    # 验证爬虫名称
+    if not validate_spider_name(spider_name):
+        show_error_panel(
+            "Invalid Spider Name", 
+            f"Spider name '[cyan]{spider_name}[/cyan]' is invalid.\n"
+            "💡 Spider name should:\n"
+            "  • Start with lowercase letter\n"
+            "  • Contain only lowercase letters, numbers, and underscores\n"
+            "  • Be a valid Python identifier"
+        )
+        return 1
+    
+    # 验证域名格式
+    if not is_valid_domain(domain):
+        show_error_panel(
+            "Invalid Domain", 
+            f"Domain '[cyan]{domain}[/cyan]' format is invalid.\n"
+            "💡 Please provide a valid domain name like 'example.com'"
+        )
+        return 1
+
+    # 验证项目环境
+    is_valid, project_package, error_msg = validate_project_environment()
+    if not is_valid:
+        show_error_panel("Not a Crawlo Project", error_msg)
+        return 1
+    
+    project_root = get_project_root()
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    default_item_class = "ExampleItem"  # 默认回退
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [
+            cls for cls in items_module.__dict__.values()
+            if isinstance(cls, type) and cls.__name__[0].isupper()  # 首字母大写
+        ]
+
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            console.print("[yellow]:warning: Warning:[/yellow] No item class found in [cyan]items.py[/cyan], using [green]ExampleItem[/green].")
+
+    except ImportError as e:
+        console.print(f"[yellow]:warning: Warning:[/yellow] Failed to import [cyan]{items_module_path}[/cyan]: {e}")
+        # 仍使用默认 ExampleItem，不中断流程
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    spiders_dir.mkdir(parents=True, exist_ok=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        show_error_panel(
+            "Spider Already Exists", 
+            f"Spider '[cyan]{spider_name}[/cyan]' already exists at\n[green]{spider_file}[/green]"
+        )
+        return 1
+
+    # 模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+    if not tmpl_path.exists():
+        show_error_panel(
+            "Template Not Found", 
+            f"Template file not found at [cyan]{tmpl_path}[/cyan]"
+        )
+        return 1
+
+    # 生成类名
+    class_name = f"{spider_name.replace('_', '').capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name
+    }
+
+    try:
+        content = _render_template(tmpl_path, context)
+        with open(spider_file, 'w', encoding='utf-8') as f:
+            f.write(content)
+        
+        console.print(f":white_check_mark: [green]Spider '[bold]{spider_name}[/bold]' created successfully![/green]")
+        console.print(f"  → Location: [cyan]{spider_file}[/cyan]")
+        console.print(f"  → Class: [yellow]{class_name}[/yellow]")
+        console.print(f"  → Domain: [blue]{domain}[/blue]")
+        console.print("\n[bold]Next steps:[/bold]")
+        console.print(f"  [blue]crawlo run[/blue] {spider_name}")
+        console.print(f"  [blue]crawlo check[/blue] {spider_name}")
+        
+        return 0
+        
+    except Exception as e:
+        show_error_panel(
+            "Creation Failed", 
+            f"Failed to create spider: {e}"
+        )
+        return 1

crawlo/commands/list.py

@@ -1,147 +1,156 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-08-31 22:33
-# @Author  : crawl-coder
-# @Desc    : 命令行入口：crawlo list，用于列出所有已注册的爬虫
-"""
-import sys
-import configparser
-from pathlib import Path
-from importlib import import_module
-
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.text import Text
-from rich import box
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.utils.log import get_logger
-
-logger = get_logger(__name__)
-console = Console()
-
-
-def get_project_root():
-    """
-    自动检测项目根目录：从当前目录向上查找 crawlo.cfg
-    找到后返回该目录路径（字符串），最多向上查找10层。
-    """
-    current = Path.cwd()
-    for _ in range(10):
-        cfg = current / "crawlo.cfg"
-        if cfg.exists():
-            return str(current)
-        if current == current.parent:
-            break
-        current = current.parent
-    return None  # 未找到
-
-
-def main(args):
-    """
-    主函数：列出所有可用爬虫
-    用法: crawlo list
-    """
-    if args:
-        console.print("[bold red]❌ Error:[/bold red] Usage: [blue]crawlo list[/blue]")
-        return 1
-
-    try:
-        # 1. 查找项目根目录
-        project_root = get_project_root()
-        if not project_root:
-            console.print(Panel(
-                Text.from_markup(
-                    ":cross_mark: [bold red]Cannot find 'crawlo.cfg'[/bold red]\n"
-                    "💡 Run this command inside your project directory.\n"
-                    "🚀 Or create a new project with:\n"
-                    "   [blue]crawlo startproject myproject[/blue]"
-                ),
-                title="❌ Not in a Crawlo Project",
-                border_style="red",
-                padding=(1, 2)
-            ))
-            return 1
-
-        project_root_path = Path(project_root)
-        project_root_str = str(project_root_path)
-
-        # 2. 将项目根加入 Python 路径
-        if project_root_str not in sys.path:
-            sys.path.insert(0, project_root_str)
-
-        # 3. 读取 crawlo.cfg 获取 settings 模块
-        cfg_file = project_root_path / "crawlo.cfg"
-        config = configparser.ConfigParser()
-        config.read(cfg_file, encoding="utf-8")
-
-        if not config.has_section("settings") or not config.has_option("settings", "default"):
-            console.print(Panel(
-                ":cross_mark: [bold red]Invalid crawlo.cfg[/bold red]\n"
-                "Missing [settings] section or 'default' option.",
-                title="❌ Config Error",
-                border_style="red"
-            ))
-            return 1
-
-        settings_module = config.get("settings", "default")
-        project_package = settings_module.split(".")[0]
-
-        # 4. 确保项目包可导入
-        try:
-            import_module(project_package)
-        except ImportError as e:
-            console.print(Panel(
-                f":cross_mark: Failed to import project package '[cyan]{project_package}[/cyan]':\n{e}",
-                title="❌ Import Error",
-                border_style="red"
-            ))
-            return 1
-
-        # 5. 初始化 CrawlerProcess 并加载爬虫模块
-        spider_modules = [f"{project_package}.spiders"]
-        process = CrawlerProcess(spider_modules=spider_modules)
-
-        # 6. 获取所有爬虫名称
-        spider_names = process.get_spider_names()
-        if not spider_names:
-            console.print(Panel(
-                Text.from_markup(
-                    ":envelope_with_arrow: [bold]No spiders found[/bold] in '[cyan]spiders/[/cyan]' directory.\n\n"
-                    "[bold]💡 Make sure:[/bold]\n"
-                    "  • Spider classes inherit from [blue]`crawlo.spider.Spider`[/blue]\n"
-                    "  • Each spider has a [green]`name`[/green] attribute\n"
-                    "  • Spiders are imported in [cyan]`spiders/__init__.py`[/cyan] (if using package)"
-                ),
-                title="📭 No Spiders Found",
-                border_style="yellow",
-                padding=(1, 2)
-            ))
-            return 1
-
-        # 7. 输出爬虫列表 —— 使用表格
-        table = Table(
-            title=f"📋 Found {len(spider_names)} spider(s)",
-            box=box.ROUNDED,
-            show_header=True,
-            header_style="bold magenta",
-            title_style="bold green"
-        )
-        table.add_column("Name", style="cyan", no_wrap=True)
-        table.add_column("Class", style="green")
-        table.add_column("Module", style="dim")
-
-        for name in sorted(spider_names):
-            spider_cls = process.get_spider_class(name)
-            module_name = spider_cls.__module__.replace(f"{project_package}.", "")
-            table.add_row(name, spider_cls.__name__, module_name)
-
-        console.print(table)
-        return 0
-
-    except Exception as e:
-        console.print(f"[bold red]❌ Unexpected error:[/bold red] {e}")
-        logger.exception("Exception during 'crawlo list'")
-        return 1
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:33
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo list，用于列出所有已注册的爬虫
+"""
+import sys
+from pathlib import Path
+from importlib import import_module
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.log import get_logger
+from .utils import validate_project_environment, show_error_panel
+
+logger = get_logger(__name__)
+console = Console()
+
+
+def main(args):
+    """
+    主函数：列出所有可用爬虫
+    用法: crawlo list [--json]
+    """
+    show_json = "--json" in args
+    
+    # 过滤掉参数后检查是否有额外参数
+    filtered_args = [arg for arg in args if not arg.startswith('--')]
+    if filtered_args:
+        if show_json:
+            console.print_json(data={"success": False, "error": "Usage: crawlo list [--json]"})
+        else:
+            console.print("[bold red]❌ Error:[/bold red] Usage: [blue]crawlo list[/blue] [--json]")
+        return 1
+
+    try:
+        # 验证项目环境
+        is_valid, project_package, error_msg = validate_project_environment()
+        if not is_valid:
+            if show_json:
+                console.print_json(data={"success": False, "error": error_msg})
+            else:
+                show_error_panel("Not a Crawlo Project", error_msg)
+            return 1
+
+        # 初始化 CrawlerProcess 并加载爬虫模块
+        spider_modules = [f"{project_package}.spiders"]
+        process = CrawlerProcess(spider_modules=spider_modules)
+
+        # 获取所有爬虫名称
+        spider_names = process.get_spider_names()
+        if not spider_names:
+            if show_json:
+                console.print_json(data={
+                    "success": True, 
+                    "spiders": [],
+                    "message": "No spiders found in project"
+                })
+            else:
+                console.print(Panel(
+                    Text.from_markup(
+                        ":envelope_with_arrow: [bold]No spiders found[/bold] in '[cyan]spiders/[/cyan]' directory.\n\n"
+                        "[bold]💡 Make sure:[/bold]\n"
+                        "  • Spider classes inherit from [blue]`crawlo.spider.Spider`[/blue]\n"
+                        "  • Each spider has a [green]`name`[/green] attribute\n"
+                        "  • Spiders are imported in [cyan]`spiders/__init__.py`[/cyan] (if using package)"
+                    ),
+                    title="📭 No Spiders Found",
+                    border_style="yellow",
+                    padding=(1, 2)
+                ))
+            return 0
+
+        # 准备爬虫信息
+        spider_info = []
+        for name in sorted(spider_names):
+            spider_cls = process.get_spider_class(name)
+            module_name = spider_cls.__module__.replace(f"{project_package}.", "")
+            
+            # 获取额外信息
+            start_urls_count = len(getattr(spider_cls, 'start_urls', []))
+            allowed_domains = getattr(spider_cls, 'allowed_domains', [])
+            custom_settings = getattr(spider_cls, 'custom_settings', {})
+            
+            spider_info.append({
+                "name": name,
+                "class": spider_cls.__name__,
+                "module": module_name,
+                "start_urls_count": start_urls_count,
+                "allowed_domains": allowed_domains,
+                "has_custom_settings": bool(custom_settings)
+            })
+
+        # JSON 输出
+        if show_json:
+            console.print_json(data={
+                "success": True,
+                "count": len(spider_info),
+                "spiders": spider_info
+            })
+            return 0
+
+        # 表格输出
+        table = Table(
+            title=f"📋 Found {len(spider_names)} spider(s)",
+            box=box.ROUNDED,
+            show_header=True,
+            header_style="bold magenta",
+            title_style="bold green"
+        )
+        table.add_column("Name", style="cyan", no_wrap=True)
+        table.add_column("Class", style="green")
+        table.add_column("Module", style="dim")
+        table.add_column("URLs", style="blue", justify="center")
+        table.add_column("Domains", style="yellow")
+        table.add_column("Custom Settings", style="magenta", justify="center")
+
+        for info in spider_info:
+            domains_display = ", ".join(info["allowed_domains"][:2])  # 显示前2个域名
+            if len(info["allowed_domains"]) > 2:
+                domains_display += f" (+{len(info['allowed_domains'])-2})"
+            elif not domains_display:
+                domains_display = "-"
+                
+            table.add_row(
+                info["name"],
+                info["class"],
+                info["module"],
+                str(info["start_urls_count"]),
+                domains_display,
+                "✓" if info["has_custom_settings"] else "-"
+            )
+
+        console.print(table)
+        
+        # 显示使用提示
+        console.print("\n[bold]🚀 Next steps:[/bold]")
+        console.print("  [blue]crawlo run[/blue] <spider_name>    # Run a specific spider")
+        console.print("  [blue]crawlo run[/blue] all             # Run all spiders")
+        console.print("  [blue]crawlo check[/blue] <spider_name>  # Check spider validity")
+        
+        return 0
+
+    except Exception as e:
+        if show_json:
+            console.print_json(data={"success": False, "error": str(e)})
+        else:
+            console.print(f"[bold red]❌ Unexpected error:[/bold red] {e}")
+        logger.exception("Exception during 'crawlo list'")
+        return 1