crawlo 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/commands/startproject.py

@@ -1,101 +1,111 @@
-# crawlo/commands/startproject.py
-import os
-import shutil
-from pathlib import Path
-
-TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
-
-
-def _render_template(tmpl_path, context):
-    """读取模板文件，替换 {{key}} 为 context 中的值"""
-    with open(tmpl_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-    for key, value in context.items():
-        content = content.replace(f'{{{{{key}}}}}', str(value))
-    return content
-
-
-def _copytree_with_templates(src, dst, context):
-    """
-    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
-    """
-    src_path = Path(src)
-    dst_path = Path(dst)
-    dst_path.mkdir(parents=True, exist_ok=True)
-
-    for item in src_path.rglob('*'):
-        rel_path = item.relative_to(src_path)
-        dst_item = dst_path / rel_path
-
-        if item.is_dir():
-            # 创建目标目录
-            dst_item.mkdir(parents=True, exist_ok=True)
-        else:
-            if item.suffix == '.tmpl':
-                # 渲染模板文件，并去掉 .tmpl 后缀
-                rendered_content = _render_template(item, context)
-                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
-                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
-                with open(final_dst, 'w', encoding='utf-8') as f:
-                    f.write(rendered_content)
-            else:
-                # 普通文件，直接复制
-                shutil.copy2(item, dst_item)
-
-
-def main(args):
-    if len(args) != 1:
-        print("Usage: crawlo startproject <project_name>")
-        return 1
-
-    project_name = args[0]
-    project_dir = Path(project_name)
-
-    if project_dir.exists():
-        print(f"Error: Directory '{project_dir}' already exists.")
-        return 1
-
-    context = {'project_name': project_name}
-    template_dir = TEMPLATES_DIR / 'project'
-
-    try:
-        # 1. 创建项目根目录
-        project_dir.mkdir()
-
-        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
-        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
-        if cfg_template.exists():
-            cfg_content = _render_template(cfg_template, context)
-            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
-        else:
-            print("Warning: crawlo.cfg.tmpl not found in templates.")
-
-        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
-        package_dir = project_dir / project_name
-        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
-        # 并将它们渲染为 .py 文件
-        _copytree_with_templates(template_dir, package_dir, context)
-
-        # 4. 创建 logs 目录
-        (project_dir / 'logs').mkdir(exist_ok=True)
-
-        print(f"""
-        ✔ 项目 '{project_name}' 创建成功！
-
-        进入项目目录:
-            cd {project_name}
-
-        创建一个爬虫:
-            crawlo genspider example example.com
-
-        运行爬虫:
-            crawlo run example
-        """)
-        return 0
-
-    except Exception as e:
-        print(f"Error creating project: {e}")
-        # 如果出错，尝试清理已创建的目录
-        if project_dir.exists():
-            shutil.rmtree(project_dir, ignore_errors=True)
-        return 1
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo startproject baidu，创建项目。
+"""
+import shutil
+from pathlib import Path
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+# 初始化 rich 控制台
+console = Console()
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('')
+                final_dst.parent.mkdir(parents=True, exist_ok=True)
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        console.print("[bold red]Error:[/bold red] Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        console.print(f"[bold red]Error:[/bold red] Directory '[cyan]{project_dir}[/cyan]' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 渲染 crawlo.cfg.tmpl
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+            console.print(f":white_check_mark: Created [green]{project_dir / 'crawlo.cfg'}[/green]")
+        else:
+            console.print("[yellow]⚠ Warning:[/yellow] Template 'crawlo.cfg.tmpl' not found.")
+
+        # 3. 复制并渲染项目包内容
+        package_dir = project_dir / project_name
+        _copytree_with_templates(template_dir, package_dir, context)
+        console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+        console.print(":white_check_mark: Created logs directory")
+
+        # 成功面板
+        success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
+        console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
+
+        # 下一步操作提示（对齐美观 + 语法高亮）
+        next_steps = f"""
+        [bold]Next steps:[/bold]
+        [blue]cd[/blue] {project_name}
+        [blue]crawlo genspider[/blue] example example.com
+        [blue]crawlo run[/blue] example
+        """.strip()
+        console.print(next_steps)
+
+        return 0
+
+    except Exception as e:
+        console.print(f"[bold red]Error creating project:[/bold red] {e}")
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+            console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
+        return 1

crawlo/commands/stats.py

@@ -1,167 +1,188 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-08-31 22:36
-# @Author  : crawl-coder
-# @Desc    : 命令行入口：crawlo stats，查看最近运行的爬虫统计信息。
-"""
-
-import sys
-import json
-from pathlib import Path
-from datetime import datetime
-from typing import Dict, Any
-
-from crawlo.utils.log import get_logger
-
-
-logger = get_logger(__name__)
-
-# 默认存储目录（相对于项目根目录）
-STATS_DIR = "logs/stats"
-
-
-def get_stats_dir() -> Path:
-    """
-    获取统计文件存储目录，优先使用项目根下的 logs/stats/
-    如果不在项目中，回退到当前目录
-    """
-    # 尝试查找项目根目录（通过 crawlo.cfg）
-    current = Path.cwd()
-    for _ in range(10):
-        if (current / "crawlo.cfg").exists():
-            return current / STATS_DIR
-        if current == current.parent:
-            break
-        current = current.parent
-
-    # 回退：使用当前目录下的 logs/stats
-    return Path.cwd() / STATS_DIR
-
-
-def record_stats(crawler):
-    """
-    【供爬虫运行时调用】记录爬虫结束后的统计信息到 JSON 文件
-    需在 Crawler 的 closed 回调中调用
-    """
-    spider_name = getattr(crawler.spider, "name", "unknown")
-    stats = crawler.stats.get_stats() if crawler.stats else {}
-
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-    stats_dir = Path(get_stats_dir())
-    stats_dir.mkdir(parents=True, exist_ok=True)
-
-    filename = stats_dir / f"{spider_name}_{timestamp}.json"
-    try:
-        with open(filename, "w", encoding="utf-8") as f:
-            json.dump({
-                "spider": spider_name,
-                "timestamp": datetime.now().isoformat(),
-                "stats": stats
-            }, f, ensure_ascii=False, indent=2, default=str)
-        logger.info(f"📊 Stats saved for spider '{spider_name}' → {filename}")
-    except Exception as e:
-        logger.error(f"Failed to save stats for '{spider_name}': {e}")
-
-
-def load_all_stats() -> Dict[str, list]:
-    """
-    加载所有已保存的统计文件，按 spider name 分组
-    返回: {spider_name: [stats_record, ...]}
-    """
-    stats_dir = get_stats_dir()
-    if not stats_dir.exists():
-        return {}
-
-    result = {}
-    json_files = sorted(stats_dir.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True)
-
-    for file in json_files:
-        try:
-            with open(file, "r", encoding="utf-8") as f:
-                data = json.load(f)
-            spider_name = data.get("spider", "unknown")
-            result.setdefault(spider_name, []).append(data)
-        except Exception as e:
-            logger.warning(f"Failed to load stats file {file}: {e}")
-    return result
-
-
-def format_value(v: Any) -> str:
-    """格式化值，防止太长或不可打印"""
-    if isinstance(v, float):
-        return f"{v:.4f}"
-    return str(v)
-
-
-def main(args):
-    """
-    主函数：查看统计信息
-    用法:
-        crawlo stats                 → 显示所有爬虫最近一次运行
-        crawlo stats myspider        → 显示指定爬虫所有历史记录
-        crawlo stats myspider --all  → 显示所有历史（同上）
-    """
-    if len(args) > 2:
-        print("Usage: crawlo stats [spider_name] [--all]")
-        return 1
-
-    spider_name = None
-    show_all = False
-
-    if args:
-        spider_name = args[0]
-        show_all = "--all" in args or "-a" in args
-
-    # 加载所有 stats
-    all_stats = load_all_stats()
-    if not all_stats:
-        print("📊 No stats found. Run a spider first.")
-        print(f"💡 Stats are saved in: {get_stats_dir()}")
-        return 0
-
-    if not spider_name:
-        # 显示每个爬虫最近一次运行
-        print("📊 Recent Spider Statistics (last run):")
-        print("-" * 60)
-        for name, runs in all_stats.items():
-            latest = runs[0]
-            print(f"🕷️  {name} ({latest['timestamp'][:19]})")
-            stats = latest["stats"]
-            for k in sorted(stats.keys()):
-                print(f"    {k:<30} {format_value(stats[k])}")
-            print()
-        return 0
-
-    else:
-        # 查看指定爬虫
-        if spider_name not in all_stats:
-            print(f"📊 No stats found for spider '{spider_name}'")
-            available = ', '.join(all_stats.keys())
-            if available:
-                print(f"💡 Available spiders: {available}")
-            return 1
-
-        runs = all_stats[spider_name]
-        if show_all:
-            print(f"📊 All runs for '{spider_name}' ({len(runs)} runs):")
-        else:
-            runs = runs[:1]
-            print(f"📊 Last run for '{spider_name}':")
-
-        print("-" * 60)
-        for run in runs:
-            print(f"⏱️  Timestamp: {run['timestamp']}")
-            stats = run["stats"]
-            for k in sorted(stats.keys()):
-                print(f"    {k:<30} {format_value(stats[k])}")
-            print("─" * 60)
-        return 0
-
-
-if __name__ == "__main__":
-    """
-    支持直接运行：
-        python -m crawlo.commands.stats
-    """
-    sys.exit(main(sys.argv[1:]))
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo stats，查看最近运行的爬虫统计信息。
+"""
+import json
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, Any
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+console = Console()
+
+# 默认存储目录（相对于项目根目录）
+STATS_DIR = "logs/stats"
+
+
+def get_stats_dir() -> Path:
+    """
+    获取统计文件存储目录，优先使用项目根下的 logs/stats/
+    如果不在项目中，回退到当前目录
+    """
+    current = Path.cwd()
+    for _ in range(10):
+        if (current / "crawlo.cfg").exists():
+            return current / STATS_DIR
+        if current == current.parent:
+            break
+        current = current.parent
+    return Path.cwd() / STATS_DIR
+
+
+def record_stats(crawler):
+    """
+    【供爬虫运行时调用】记录爬虫结束后的统计信息到 JSON 文件
+    需在 Crawler 的 closed 回调中调用
+    """
+    spider_name = getattr(crawler.spider, "name", "unknown")
+    stats = crawler.stats.get_stats() if crawler.stats else {}
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    stats_dir = Path(get_stats_dir())
+    stats_dir.mkdir(parents=True, exist_ok=True)
+
+    filename = stats_dir / f"{spider_name}_{timestamp}.json"
+    try:
+        with open(filename, "w", encoding="utf-8") as f:
+            json.dump({
+                "spider": spider_name,
+                "timestamp": datetime.now().isoformat(),
+                "stats": stats
+            }, f, ensure_ascii=False, indent=2, default=str)
+        logger.info(f"📊 Stats saved for spider '{spider_name}' → {filename}")
+    except Exception as e:
+        logger.error(f"Failed to save stats for '{spider_name}': {e}")
+
+
+def load_all_stats() -> Dict[str, list]:
+    """
+    加载所有已保存的统计文件，按 spider name 分组
+    返回: {spider_name: [stats_record, ...]}
+    """
+    stats_dir = get_stats_dir()
+    if not stats_dir.exists():
+        return {}
+
+    result = {}
+    json_files = sorted(stats_dir.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True)
+
+    for file in json_files:
+        try:
+            with open(file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            spider_name = data.get("spider", "unknown")
+            result.setdefault(spider_name, []).append(data)
+        except Exception as e:
+            logger.warning(f"Failed to load stats file {file}: {e}")
+    return result
+
+
+def format_value(v: Any) -> str:
+    """格式化值，防止太长或不可打印"""
+    if isinstance(v, float):
+        return f"{v:.4f}"
+    s = str(v)
+    if len(s) > 80:
+        return s[:77] + "..."
+    return s
+
+
+def display_stats_table(stats_data: dict, title: str = "Statistics"):
+    """通用函数：用 rich.table 展示统计数据"""
+    table = Table(title=title, box=box.ROUNDED, show_header=True, header_style="bold magenta")
+    table.add_column("Key", style="cyan", no_wrap=True)
+    table.add_column("Value", style="green")
+
+    for k in sorted(stats_data.keys()):
+        table.add_row(k, format_value(stats_data[k]))
+
+    console.print(table)
+
+
+def main(args):
+    """
+    主函数：查看统计信息
+    用法:
+        crawlo stats                 → 显示所有爬虫最近一次运行
+        crawlo stats myspider        → 显示指定爬虫所有历史记录
+        crawlo stats myspider --all  → 显示所有历史（同上）
+    """
+    if len(args) > 2:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo stats[/blue] [spider_name] [--all]")
+        return 1
+
+    spider_name = None
+    show_all = False
+
+    if args:
+        spider_name = args[0]
+        show_all = "--all" in args or "-a" in args
+
+    all_stats = load_all_stats()
+
+    if not all_stats:
+        console.print(Panel(
+            Text.from_markup(
+                ":chart_with_upwards_trend: [bold]No stats found.[/bold]\n"
+                "💡 Run a spider first to generate statistics.\n"
+                f"📁 Stats directory: [cyan]{get_stats_dir()}[/cyan]"
+            ),
+            title="📊 Statistics",
+            border_style="yellow",
+            padding=(1, 2)
+        ))
+        return 0
+
+    # 显示所有爬虫最近一次运行
+    if not spider_name:
+        console.print(Panel(
+            "[bold]Recent Spider Statistics (last run)[/bold]",
+            title="📊 Spider Stats Overview",
+            border_style="green",
+            padding=(0, 1)
+        ))
+
+        for name, runs in all_stats.items():
+            latest = runs[0]
+            ts = latest['timestamp'][:19]
+            console.print(f"🕷️  [bold cyan]{name}[/bold cyan] ([green]{ts}[/green])")
+            display_stats_table(latest["stats"], title=f"Stats for {name}")
+            console.print()  # 空行分隔
+
+        return 0
+
+    # 显示指定爬虫的历史
+    if spider_name not in all_stats:
+        console.print(f"[bold red]:cross_mark: No stats found for spider '[cyan]{spider_name}[/cyan]'[/bold red]")
+        available = ', '.join(all_stats.keys())
+        if available:
+            console.print(f":bulb: Available spiders: [green]{available}[/green]")
+        return 1
+
+    runs = all_stats[spider_name]
+    if show_all:
+        console.print(f":bar_chart: [bold]All runs for '[cyan]{spider_name}[/cyan]' ({len(runs)} runs):[/bold]")
+    else:
+        runs = runs[:1]
+        console.print(f":bar_chart: [bold]Last run for '[cyan]{spider_name}[/cyan]':[/bold]")
+
+    for i, run in enumerate(runs, 1):
+        ts = run['timestamp']
+        subtitle = f"Run #{i} · {ts}" if show_all else f"Last Run · {ts}"
+        display_stats_table(run["stats"], title=f"Stats for {spider_name} — {subtitle}")
+        if i < len(runs):
+            console.print("─" * 60)
+
+    return 0

crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-