crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/commands/genspider.py

@@ -1,111 +1,126 @@
-import os
-import sys
-from pathlib import Path
-import configparser
-import importlib
-
-TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
-
-
-def _render_template(tmpl_path, context):
-    """读取模板文件，替换 {{key}} 为 context 中的值"""
-    with open(tmpl_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-    for key, value in context.items():
-        content = content.replace(f'{{{{{key}}}}}', str(value))
-    return content
-
-
-def main(args):
-    if len(args) < 2:
-        print("Usage: crawlo genspider <spider_name> <domain>")
-        return 1
-
-    spider_name = args[0]
-    domain = args[1]
-
-    # 查找项目根目录
-    project_root = None
-    current = Path.cwd()
-    while True:
-        cfg_file = current / 'crawlo.cfg'
-        if cfg_file.exists():
-            project_root = current
-            break
-        parent = current.parent
-        if parent == current:
-            break
-        current = parent
-
-    if not project_root:
-        print("Error: Not a crawlo project. crawlo.cfg not found.")
-        return 1
-
-    # 将项目根目录加入 sys.path
-    if str(project_root) not in sys.path:
-        sys.path.insert(0, str(project_root))
-
-    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
-    config = configparser.ConfigParser()
-    try:
-        config.read(cfg_file, encoding='utf-8')
-        settings_module = config.get('settings', 'default')
-        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
-    except Exception as e:
-        print(f"Error reading crawlo.cfg: {e}")
-        return 1
-
-    # 确定 items 模块的路径
-    items_module_path = f"{project_package}.items"
-
-    # 尝试导入 items 模块
-    try:
-        items_module = importlib.import_module(items_module_path)
-        # 获取模块中所有大写开头的类
-        item_classes = [cls for cls in items_module.__dict__.values()
-                        if isinstance(cls, type) and cls.__name__.isupper()]
-
-        # 如果找到了类，使用第一个作为默认
-        if item_classes:
-            default_item_class = item_classes[0].__name__
-        else:
-            default_item_class = "ExampleItem"  # 回退到示例
-    except ImportError as e:
-        print(f"Error importing items module '{items_module_path}': {e}")
-        default_item_class = "ExampleItem"
-
-    # 创建爬虫文件
-    spiders_dir = project_root / project_package / 'spiders'
-    if not spiders_dir.exists():
-        spiders_dir.mkdir(parents=True)
-
-    spider_file = spiders_dir / f'{spider_name}.py'
-    if spider_file.exists():
-        print(f"Error: Spider '{spider_name}' already exists.")
-        return 1
-
-    # ✅ 修正模板路径
-    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
-
-    if not tmpl_path.exists():
-        print(f"Error: Template file not found at {tmpl_path}")
-        return 1
-
-    # ✅ 生成正确的类名
-    class_name = f"{spider_name.capitalize()}Spider"
-
-    context = {
-        'spider_name': spider_name,
-        'domain': domain,
-        'project_name': project_package,
-        'item_class': default_item_class,
-        'class_name': class_name  # ✅ 添加处理好的类名
-    }
-
-    content = _render_template(tmpl_path, context)
-
-    with open(spider_file, 'w', encoding='utf-8') as f:
-        f.write(content)
-
-    print(f"Spider '{spider_name}' created in {spider_file}")
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
+"""
+import sys
+from pathlib import Path
+import configparser
+import importlib
+from rich.console import Console
+
+# 初始化 rich 控制台
+console = Console()
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo genspider[/blue] <spider_name> <domain>")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 查找项目根目录
+    project_root = None
+    current = Path.cwd()
+    while True:
+        cfg_file = current / 'crawlo.cfg'
+        if cfg_file.exists():
+            project_root = current
+            break
+        parent = current.parent
+        if parent == current:
+            break
+        current = parent
+
+    if not project_root:
+        console.print("[bold red]:cross_mark: Error:[/bold red] Not a crawlo project. [cyan]crawlo.cfg[/cyan] not found.")
+        return 1
+
+    # 将项目根目录加入 sys.path
+    if str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
+
+    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_file, encoding='utf-8')
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
+    except Exception as e:
+        console.print(f"[bold red]:cross_mark: Error reading crawlo.cfg:[/bold red] {e}")
+        return 1
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    default_item_class = "ExampleItem"  # 默认回退
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [
+            cls for cls in items_module.__dict__.values()
+            if isinstance(cls, type) and cls.__name__[0].isupper()  # 首字母大写
+        ]
+
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            console.print("[yellow]:warning: Warning:[/yellow] No item class found in [cyan]items.py[/cyan], using [green]ExampleItem[/green].")
+
+    except ImportError as e:
+        console.print(f"[yellow]:warning: Warning:[/yellow] Failed to import [cyan]{items_module_path}[/cyan]: {e}")
+        # 仍使用默认 ExampleItem，不中断流程
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    spiders_dir.mkdir(parents=True, exist_ok=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        console.print(f"[bold red]:cross_mark: Error:[/bold red] Spider '[cyan]{spider_name}[/cyan]' already exists at [green]{spider_file}[/green]")
+        return 1
+
+    # 模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+    if not tmpl_path.exists():
+        console.print(f"[bold red]:cross_mark: Error:[/bold red] Template file not found at [cyan]{tmpl_path}[/cyan]")
+        return 1
+
+    # 生成类名
+    class_name = f"{spider_name.capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name
+    }
+
+    content = _render_template(tmpl_path, context)
+
+    with open(spider_file, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    console.print(f":white_check_mark: [green]Spider '[bold]{spider_name}[/bold]' created successfully![/green]")
+    console.print(f"  → Location: [cyan]{spider_file}[/cyan]")
+    console.print("\n[bold]Next step:[/bold]")
+    console.print(f"  [blue]crawlo run[/blue] {spider_name}")
+
     return 0

crawlo/commands/list.py

@@ -1,92 +1,147 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-08-31 22:33
-# @Author  :   crawl-coder
-# @Desc    :   命令行入口：crawlo list，用于列出所有已注册的爬虫
-"""
-import sys
-import configparser
-
-from crawlo.crawler import CrawlerProcess
-from crawlo.utils.project import get_settings
-from crawlo.utils.log import get_logger
-
-
-logger = get_logger(__name__)
-
-
-def main(args):
-    """
-    列出所有可用爬虫
-    用法: crawlo list
-    """
-    if args:
-        print("Usage: crawlo list")
-        return 1
-
-    try:
-        # 1. 获取项目根目录
-        project_root = get_settings().get('PROJECT_ROOT')
-        if not project_root:
-            print("❌ Error: Cannot determine project root.")
-            return 1
-
-        # 将项目根目录加入 sys.path
-        project_root_str = str(project_root)
-        if project_root_str not in sys.path:
-            sys.path.insert(0, project_root_str)
-
-        # 2. 读取 crawlo.cfg 获取项目包名
-        cfg_file = project_root / 'crawlo.cfg'
-        if not cfg_file.exists():
-            print(f"❌ Error: crawlo.cfg not found in {project_root}")
-            return 1
-
-        config = configparser.ConfigParser()
-        config.read(cfg_file, encoding='utf-8')
-
-        if not config.has_section('settings') or not config.has_option('settings', 'default'):
-            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
-            return 1
-
-        settings_module = config.get('settings', 'default')
-        project_package = settings_module.split('.')[0]
-
-        # 3. 创建 CrawlerProcess 并自动发现爬虫
-        spider_modules = [f"{project_package}.spiders"]
-        process = CrawlerProcess(spider_modules=spider_modules)
-
-        # 4. 获取所有爬虫信息
-        spider_names = process.get_spider_names()
-        if not spider_names:
-            print("📭 No spiders found.")
-            print("💡 Make sure:")
-            print("   - Your spider classes inherit from `Spider`")
-            print("   - They define a `name` attribute")
-            print("   - The modules are imported (e.g. via __init__.py)")
-            return 1
-
-        # 5. 输出爬虫列表
-        print(f"📋 Found {len(spider_names)} spider(s):")
-        print("-" * 50)
-        for name in sorted(spider_names):
-            cls = process.get_spider_class(name)
-            module = cls.__module__.replace(project_package + ".", "")  # 简化模块名
-            print(f"🕷️  {name:<20} {cls.__name__:<25} ({module})")
-        print("-" * 50)
-        return 0
-
-    except Exception as e:
-        print(f"❌ Error listing spiders: {e}")
-        import traceback
-        traceback.print_exc()
-        return 1
-
-
-if __name__ == '__main__':
-    """
-    允许直接运行：
-        python -m crawlo.commands.list
-    """
-    sys.exit(main(sys.argv[1:]))
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:33
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo list，用于列出所有已注册的爬虫
+"""
+import sys
+import configparser
+from pathlib import Path
+from importlib import import_module
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.log import get_logger
+
+logger = get_logger(__name__)
+console = Console()
+
+
+def get_project_root():
+    """
+    自动检测项目根目录：从当前目录向上查找 crawlo.cfg
+    找到后返回该目录路径（字符串），最多向上查找10层。
+    """
+    current = Path.cwd()
+    for _ in range(10):
+        cfg = current / "crawlo.cfg"
+        if cfg.exists():
+            return str(current)
+        if current == current.parent:
+            break
+        current = current.parent
+    return None  # 未找到
+
+
+def main(args):
+    """
+    主函数：列出所有可用爬虫
+    用法: crawlo list
+    """
+    if args:
+        console.print("[bold red]❌ Error:[/bold red] Usage: [blue]crawlo list[/blue]")
+        return 1
+
+    try:
+        # 1. 查找项目根目录
+        project_root = get_project_root()
+        if not project_root:
+            console.print(Panel(
+                Text.from_markup(
+                    ":cross_mark: [bold red]Cannot find 'crawlo.cfg'[/bold red]\n"
+                    "💡 Run this command inside your project directory.\n"
+                    "🚀 Or create a new project with:\n"
+                    "   [blue]crawlo startproject myproject[/blue]"
+                ),
+                title="❌ Not in a Crawlo Project",
+                border_style="red",
+                padding=(1, 2)
+            ))
+            return 1
+
+        project_root_path = Path(project_root)
+        project_root_str = str(project_root_path)
+
+        # 2. 将项目根加入 Python 路径
+        if project_root_str not in sys.path:
+            sys.path.insert(0, project_root_str)
+
+        # 3. 读取 crawlo.cfg 获取 settings 模块
+        cfg_file = project_root_path / "crawlo.cfg"
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding="utf-8")
+
+        if not config.has_section("settings") or not config.has_option("settings", "default"):
+            console.print(Panel(
+                ":cross_mark: [bold red]Invalid crawlo.cfg[/bold red]\n"
+                "Missing [settings] section or 'default' option.",
+                title="❌ Config Error",
+                border_style="red"
+            ))
+            return 1
+
+        settings_module = config.get("settings", "default")
+        project_package = settings_module.split(".")[0]
+
+        # 4. 确保项目包可导入
+        try:
+            import_module(project_package)
+        except ImportError as e:
+            console.print(Panel(
+                f":cross_mark: Failed to import project package '[cyan]{project_package}[/cyan]':\n{e}",
+                title="❌ Import Error",
+                border_style="red"
+            ))
+            return 1
+
+        # 5. 初始化 CrawlerProcess 并加载爬虫模块
+        spider_modules = [f"{project_package}.spiders"]
+        process = CrawlerProcess(spider_modules=spider_modules)
+
+        # 6. 获取所有爬虫名称
+        spider_names = process.get_spider_names()
+        if not spider_names:
+            console.print(Panel(
+                Text.from_markup(
+                    ":envelope_with_arrow: [bold]No spiders found[/bold] in '[cyan]spiders/[/cyan]' directory.\n\n"
+                    "[bold]💡 Make sure:[/bold]\n"
+                    "  • Spider classes inherit from [blue]`crawlo.spider.Spider`[/blue]\n"
+                    "  • Each spider has a [green]`name`[/green] attribute\n"
+                    "  • Spiders are imported in [cyan]`spiders/__init__.py`[/cyan] (if using package)"
+                ),
+                title="📭 No Spiders Found",
+                border_style="yellow",
+                padding=(1, 2)
+            ))
+            return 1
+
+        # 7. 输出爬虫列表 —— 使用表格
+        table = Table(
+            title=f"📋 Found {len(spider_names)} spider(s)",
+            box=box.ROUNDED,
+            show_header=True,
+            header_style="bold magenta",
+            title_style="bold green"
+        )
+        table.add_column("Name", style="cyan", no_wrap=True)
+        table.add_column("Class", style="green")
+        table.add_column("Module", style="dim")
+
+        for name in sorted(spider_names):
+            spider_cls = process.get_spider_class(name)
+            module_name = spider_cls.__module__.replace(f"{project_package}.", "")
+            table.add_row(name, spider_cls.__name__, module_name)
+
+        console.print(table)
+        return 0
+
+    except Exception as e:
+        console.print(f"[bold red]❌ Unexpected error:[/bold red] {e}")
+        logger.exception("Exception during 'crawlo list'")
+        return 1