crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/commands/run.py

@@ -1,181 +1,286 @@
-"""
-命令行入口：crawlo run <spider_name>
-用于运行指定名称的爬虫。
-"""
-
-import asyncio
-from pathlib import Path
-import configparser
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.utils.project import get_settings
-from crawlo.utils.log import get_logger
-
-logger = get_logger(__name__)
-
-
-def main(args):
-    """
-    运行指定爬虫的主函数
-    用法:
-        crawlo run <spider_name>
-        crawlo run all
-    """
-    if len(args) < 1:
-        print("Usage: crawlo run <spider_name>|all")
-        print("Examples:")
-        print("  crawlo run baidu")
-        print("  crawlo run all")
-        return 1
-
-    spider_arg = args[0]
-
-    try:
-        # 1. 获取项目根目录
-        project_root = get_settings().get('PROJECT_ROOT')
-        if not project_root:
-            print("❌ Error: Cannot determine project root.")
-            return 1
-
-        if str(project_root) not in sys.path:
-            sys.path.insert(0, str(project_root))
-
-        # 2. 读取 crawlo.cfg 获取项目包名
-        cfg_file = project_root / 'crawlo.cfg'
-        if not cfg_file.exists():
-            print(f"❌ Error: crawlo.cfg not found in {project_root}")
-            return 1
-
-        config = configparser.ConfigParser()
-        config.read(cfg_file, encoding='utf-8')
-
-        if not config.has_section('settings') or not config.has_option('settings', 'default'):
-            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
-            return 1
-
-        settings_module = config.get('settings', 'default')
-        project_package = settings_module.split('.')[0]
-
-        # 3. 创建 CrawlerProcess 并自动发现爬虫模块
-        spider_modules = [f"{project_package}.spiders"]
-        settings = get_settings()
-        process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
-
-        # === 新增：支持 'all' ===
-        if spider_arg.lower() == "all":
-            spider_names = process.get_spider_names()
-            if not spider_names:
-                print("❌ No spiders found. Make sure spiders are defined and imported.")
-                return 1
-
-            print(f"🚀 Starting ALL {len(spider_names)} spiders:")
-            for name in sorted(spider_names):
-                cls = process.get_spider_class(name)
-                print(f"   🕷️  {name} ({cls.__name__})")
-            print("-" * 50)
-
-            # 启动所有爬虫
-            asyncio.run(process.crawl(spider_names))
-            return 0
-
-        # === 原有：启动单个爬虫 ===
-        spider_name = spider_arg
-        if not process.is_spider_registered(spider_name):
-            print(f"❌ Error: Spider with name '{spider_name}' not found.")
-            available_names = process.get_spider_names()
-            if available_names:
-                print("💡 Available spiders:")
-                for name in sorted(available_names):
-                    cls = process.get_spider_class(name)
-                    print(f"   - {name} (class: {cls.__name__})")
-            else:
-                print("💡 No spiders found. Make sure your spider classes are defined and imported.")
-            return 1
-
-        spider_class = process.get_spider_class(spider_name)
-
-        # 打印启动信息
-        print(f"🚀 Starting spider: {spider_name}")
-        print(f"📁 Project: {project_package}")
-        print(f"🕷️  Class: {spider_class.__name__}")
-        print("-" * 50)
-
-        # 启动爬虫
-        asyncio.run(process.crawl(spider_name))
-
-        print("-" * 50)
-        print("✅ Spider completed successfully!")
-        return 0
-
-    except KeyboardInterrupt:
-        print("\n⚠️  Spider interrupted by user.")
-        return 1
-    except Exception as e:
-        print(f"❌ Error running spider: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-
-
-def list_available_spiders(project_package: str):
-    """
-    列出指定项目包中所有可用的爬虫（用于调试或命令行扩展）
-    """
-    try:
-        # 临时创建一个 CrawlerProcess 来发现爬虫
-        process = CrawlerProcess(spider_modules=[f"{project_package}.spiders"])
-        available_names = process.get_spider_names()
-
-        if not available_names:
-            print("   No spiders found. Make sure:")
-            print("   - spiders/ 目录存在")
-            print("   - 爬虫类继承 Spider 且定义了 name")
-            print("   - 模块被导入（可通过 __init__.py 触发）")
-            return
-
-        print(f"Found {len(available_names)} spider(s):")
-        for name in sorted(available_names):
-            cls = process.get_spider_class(name)
-            module = cls.__module__.replace(project_package + ".", "")
-            print(f"   - {name} ({cls.__name__} @ {module})")
-    except Exception as e:
-        print(f"❌ Failed to list spiders: {e}")
-        import traceback
-        traceback.print_exc()
-
-
-def run_spider_by_name(spider_name: str, project_package: str = None):
-    """
-    在代码中直接运行某个爬虫（需提供 project_package）
-    """
-    if project_package is None:
-        # 尝试从配置读取
-        cfg_file = Path('crawlo.cfg')
-        if cfg_file.exists():
-            config = configparser.ConfigParser()
-            config.read(cfg_file, encoding='utf-8')
-            if config.has_option('settings', 'default'):
-                project_package = config.get('settings', 'default').split('.')[0]
-
-    if not project_package:
-        print("❌ Error: project_package is required.")
-        return 1
-
-    # 添加项目路径
-    project_root = get_settings().get('PROJECT_ROOT')
-    if project_root and str(project_root) not in sys.path:
-        sys.path.insert(0, str(project_root))
-
-    # 复用 main 函数逻辑
-    args = [spider_name]
-    return main(args)
-
-
-if __name__ == '__main__':
-    """
-    允许直接运行：
-        python -m crawlo.commands.run <spider_name>
-    """
-    import sys
-
-    sys.exit(main(sys.argv[1:]))
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo run <spider_name>|all，用于运行指定爬虫。
+"""
+import sys
+import asyncio
+import configparser
+from pathlib import Path
+from importlib import import_module
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+from rich import box
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.log import get_logger
+from crawlo.project import get_settings
+from crawlo.commands.stats import record_stats
+
+logger = get_logger(__name__)
+console = Console()
+
+
+def get_project_root():
+    """
+    向上查找 crawlo.cfg 来确定项目根目录
+    """
+    current = Path.cwd()
+    for _ in range(10):
+        cfg = current / "crawlo.cfg"
+        if cfg.exists():
+            return current
+        if current == current.parent:
+            break
+        current = current.parent
+    return None
+
+
+def main(args):
+    """
+    主函数：运行指定爬虫
+    用法:
+        crawlo run <spider_name>|all [--json] [--no-stats]
+    """
+    if len(args) < 1:
+        console.print("[bold red]❌ Usage:[/bold red] [blue]crawlo run[/blue] <spider_name>|all [bold yellow][--json] [--no-stats][/bold yellow]")
+        console.print("💡 Examples:")
+        console.print("   [blue]crawlo run baidu[/blue]")
+        console.print("   [blue]crawlo run all[/blue]")
+        console.print("   [blue]crawlo run all --json --no-stats[/blue]")
+        return 1
+
+    # 解析参数
+    spider_arg = args[0]
+    show_json = "--json" in args
+    no_stats = "--no-stats" in args
+
+    try:
+        # 1. 查找项目根目录
+        project_root = get_project_root()
+        if not project_root:
+            msg = ":cross_mark: [bold red]Cannot find 'crawlo.cfg'[/bold red]\n💡 Run this command inside your project directory."
+            if show_json:
+                console.print_json(data={"success": False, "error": "Project root not found"})
+                return 1
+            else:
+                console.print(Panel(
+                    Text.from_markup(msg),
+                    title="❌ Not in a Crawlo Project",
+                    border_style="red",
+                    padding=(1, 2)
+                ))
+                return 1
+
+        project_root_str = str(project_root)
+        if project_root_str not in sys.path:
+            sys.path.insert(0, project_root_str)
+
+        # 2. 读取 crawlo.cfg 获取 settings 模块
+        cfg_file = project_root / "crawlo.cfg"
+        if not cfg_file.exists():
+            msg = f"crawlo.cfg not found in {project_root}"
+            if show_json:
+                console.print_json(data={"success": False, "error": msg})
+                return 1
+            else:
+                console.print(Panel(msg, title="❌ Missing Config", border_style="red"))
+                return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding="utf-8")
+
+        if not config.has_section("settings") or not config.has_option("settings", "default"):
+            msg = "Missing [settings] section or 'default' option in crawlo.cfg"
+            if show_json:
+                console.print_json(data={"success": False, "error": msg})
+                return 1
+            else:
+                console.print(Panel(msg, title="❌ Invalid Config", border_style="red"))
+                return 1
+
+        settings_module = config.get("settings", "default")
+        project_package = settings_module.split(".")[0]
+
+        # 3. 确保项目包可导入
+        try:
+            import_module(project_package)
+        except ImportError as e:
+            msg = f"Failed to import project package '{project_package}': {e}"
+            if show_json:
+                console.print_json(data={"success": False, "error": msg})
+                return 1
+            else:
+                console.print(Panel(msg, title="❌ Import Error", border_style="red"))
+                return 1
+
+        # 4. 加载 settings 和爬虫模块
+        settings = get_settings()
+        spider_modules = [f"{project_package}.spiders"]
+        process = CrawlerProcess(settings=settings, spider_modules=spider_modules)
+
+        # === 情况1：运行所有爬虫 ===
+        if spider_arg.lower() == "all":
+            spider_names = process.get_spider_names()
+            if not spider_names:
+                msg = "No spiders found."
+                if show_json:
+                    console.print_json(data={"success": False, "error": msg})
+                    return 1
+                else:
+                    console.print(Panel(
+                        Text.from_markup(
+                            ":cross_mark: [bold red]No spiders found.[/bold red]\n\n"
+                            "[bold]💡 Make sure:[/bold]\n"
+                            "  • Spiders are defined in '[cyan]spiders/[/cyan]'\n"
+                            "  • They have a [green]`name`[/green] attribute\n"
+                            "  • Modules are imported (e.g. via [cyan]__init__.py[/cyan])"
+                        ),
+                        title="❌ No Spiders",
+                        border_style="red",
+                        padding=(1, 2)
+                    ))
+                    return 1
+
+            # 显示即将运行的爬虫列表
+            table = Table(
+                title=f"🚀 Starting ALL {len(spider_names)} spider(s)",
+                box=box.ROUNDED,
+                show_header=True,
+                header_style="bold magenta"
+            )
+            table.add_column("Name", style="cyan")
+            table.add_column("Class", style="green")
+
+            for name in sorted(spider_names):
+                cls = process.get_spider_class(name)
+                table.add_row(name, cls.__name__)
+
+            console.print(table)
+            console.print()
+
+            # 注册 stats 记录（除非 --no-stats）
+            if not no_stats:
+                for crawler in process.crawlers:
+                    crawler.signals.connect(record_stats, signal="spider_closed")
+
+            # 并行运行所有爬虫
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                transient=True,
+            ) as progress:
+                task = progress.add_task("Running all spiders...", total=None)
+                asyncio.run(process.crawl(spider_names))
+
+            if show_json:
+                console.print_json(data={"success": True, "spiders": spider_names})
+            else:
+                console.print(Panel(
+                    ":tada: [bold green]All spiders completed successfully![/bold green]",
+                    title="✅ All Done",
+                    border_style="green"
+                ))
+            return 0
+
+        # === 情况2：运行单个爬虫 ===
+        spider_name = spider_arg
+        if not process.is_spider_registered(spider_name):
+            available = process.get_spider_names()
+            msg = f"Spider '[cyan]{spider_name}[/cyan]' not found."
+            if show_json:
+                console.print_json(data={
+                    "success": False,
+                    "error": msg,
+                    "available": available
+                })
+                return 1
+            else:
+                panel_content = Text.from_markup(msg + "\n")
+                if available:
+                    panel_content.append("\n💡 Available spiders:\n")
+                    for name in sorted(available):
+                        cls = process.get_spider_class(name)
+                        panel_content.append(f"  • [cyan]{name}[/cyan] ([green]{cls.__name__}[/green])\n")
+                else:
+                    panel_content.append("\n💡 No spiders found. Check your spiders module.")
+
+                console.print(Panel(
+                    panel_content,
+                    title="❌ Spider Not Found",
+                    border_style="red",
+                    padding=(1, 2)
+                ))
+                return 1
+
+        spider_class = process.get_spider_class(spider_name)
+
+        # 显示启动信息
+        if not show_json:
+            info_table = Table(
+                title=f"🚀 Starting Spider: [bold cyan]{spider_name}[/bold cyan]",
+                box=box.SIMPLE,
+                show_header=False,
+                title_style="bold green"
+            )
+            info_table.add_column("Key", style="yellow")
+            info_table.add_column("Value", style="cyan")
+            info_table.add_row("Project", project_package)
+            info_table.add_row("Class", spider_class.__name__)
+            info_table.add_row("Module", spider_class.__module__)
+            console.print(info_table)
+            console.print()
+
+        # 注册 stats 记录
+        if not no_stats:
+            for crawler in process.crawlers:
+                crawler.signals.connect(record_stats, signal="spider_closed")
+
+        # 运行爬虫
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[progress.description]{task.description}"),
+            transient=True,
+        ) as progress:
+            task = progress.add_task(f"Running {spider_name}...", total=None)
+            asyncio.run(process.crawl(spider_name))
+
+        if show_json:
+            console.print_json(data={"success": True, "spider": spider_name})
+        else:
+            console.print(Panel(
+                f":tada: [bold green]Spider '[cyan]{spider_name}[/cyan]' completed successfully![/bold green]",
+                title="✅ Done",
+                border_style="green"
+            ))
+        return 0
+
+    except KeyboardInterrupt:
+        msg = "⚠️  Spider interrupted by user."
+        if show_json:
+            console.print_json(data={"success": False, "error": msg})
+        else:
+            console.print(f"[bold yellow]{msg}[/bold yellow]")
+        return 1
+    except Exception as e:
+        logger.exception("Exception during 'crawlo run'")
+        msg = f"Unexpected error: {e}"
+        if show_json:
+            console.print_json(data={"success": False, "error": msg})
+        else:
+            console.print(f"[bold red]❌ {msg}[/bold red]")
+        return 1
+
+
+if __name__ == "__main__":
+    """
+    支持直接运行：
+        python -m crawlo.commands.run spider_name
+    """
+    sys.exit(main(sys.argv[1:]))