crawlo 1.0.9__py3-none-any.whl → 1.1.1__py3-none-any.whl

crawlo/commands/startproject.py

@@ -1,101 +1,111 @@
-# crawlo/commands/startproject.py
-import os
-import shutil
-from pathlib import Path
-
-TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
-
-
-def _render_template(tmpl_path, context):
-    """读取模板文件，替换 {{key}} 为 context 中的值"""
-    with open(tmpl_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-    for key, value in context.items():
-        content = content.replace(f'{{{{{key}}}}}', str(value))
-    return content
-
-
-def _copytree_with_templates(src, dst, context):
-    """
-    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
-    """
-    src_path = Path(src)
-    dst_path = Path(dst)
-    dst_path.mkdir(parents=True, exist_ok=True)
-
-    for item in src_path.rglob('*'):
-        rel_path = item.relative_to(src_path)
-        dst_item = dst_path / rel_path
-
-        if item.is_dir():
-            # 创建目标目录
-            dst_item.mkdir(parents=True, exist_ok=True)
-        else:
-            if item.suffix == '.tmpl':
-                # 渲染模板文件，并去掉 .tmpl 后缀
-                rendered_content = _render_template(item, context)
-                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
-                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
-                with open(final_dst, 'w', encoding='utf-8') as f:
-                    f.write(rendered_content)
-            else:
-                # 普通文件，直接复制
-                shutil.copy2(item, dst_item)
-
-
-def main(args):
-    if len(args) != 1:
-        print("Usage: crawlo startproject <project_name>")
-        return 1
-
-    project_name = args[0]
-    project_dir = Path(project_name)
-
-    if project_dir.exists():
-        print(f"Error: Directory '{project_dir}' already exists.")
-        return 1
-
-    context = {'project_name': project_name}
-    template_dir = TEMPLATES_DIR / 'project'
-
-    try:
-        # 1. 创建项目根目录
-        project_dir.mkdir()
-
-        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
-        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
-        if cfg_template.exists():
-            cfg_content = _render_template(cfg_template, context)
-            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
-        else:
-            print("Warning: crawlo.cfg.tmpl not found in templates.")
-
-        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
-        package_dir = project_dir / project_name
-        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
-        # 并将它们渲染为 .py 文件
-        _copytree_with_templates(template_dir, package_dir, context)
-
-        # 4. 创建 logs 目录
-        (project_dir / 'logs').mkdir(exist_ok=True)
-
-        print(f"""
-        ✔ 项目 '{project_name}' 创建成功！
-
-        进入项目目录:
-            cd {project_name}
-
-        创建一个爬虫:
-            crawlo genspider example example.com
-
-        运行爬虫:
-            crawlo run example
-        """)
-        return 0
-
-    except Exception as e:
-        print(f"Error creating project: {e}")
-        # 如果出错，尝试清理已创建的目录
-        if project_dir.exists():
-            shutil.rmtree(project_dir, ignore_errors=True)
-        return 1
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo startproject baidu，创建项目。
+"""
+import shutil
+from pathlib import Path
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
+# 初始化 rich 控制台
+console = Console()
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('')
+                final_dst.parent.mkdir(parents=True, exist_ok=True)
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        console.print("[bold red]Error:[/bold red] Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        console.print(f"[bold red]Error:[/bold red] Directory '[cyan]{project_dir}[/cyan]' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 渲染 crawlo.cfg.tmpl
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+            console.print(f":white_check_mark: Created [green]{project_dir / 'crawlo.cfg'}[/green]")
+        else:
+            console.print("[yellow]⚠ Warning:[/yellow] Template 'crawlo.cfg.tmpl' not found.")
+
+        # 3. 复制并渲染项目包内容
+        package_dir = project_dir / project_name
+        _copytree_with_templates(template_dir, package_dir, context)
+        console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+        console.print(":white_check_mark: Created logs directory")
+
+        # 成功面板
+        success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
+        console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
+
+        # 下一步操作提示（对齐美观 + 语法高亮）
+        next_steps = f"""
+        [bold]Next steps:[/bold]
+        [blue]cd[/blue] {project_name}
+        [blue]crawlo genspider[/blue] example example.com
+        [blue]crawlo run[/blue] example
+        """.strip()
+        console.print(next_steps)
+
+        return 0
+
+    except Exception as e:
+        console.print(f"[bold red]Error creating project:[/bold red] {e}")
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+            console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
+        return 1

crawlo/commands/stats.py

@@ -1,59 +1,188 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-"""
-# @Time    :    2025-08-31 22:36
-# @Author  :   crawl-coder
-# @Desc    :   命令行入口：crawlo stats，查看最近运行的爬虫统计信息。
-"""
-import sys
-from crawlo.utils.log import get_logger
-
-
-logger = get_logger(__name__)
-
-# 保存最近运行的爬虫的统计（示例）
-_LAST_RUN_STATS = {}
-
-
-def record_stats(crawler):
-    """在爬虫关闭后记录统计（需在 close 中调用）"""
-    if crawler.stats and crawler.spider:
-        _LAST_RUN_STATS[crawler.spider.name] = crawler.stats.get_stats()
-
-
-def main(args):
-    if len(args) == 0:
-        # 显示所有历史统计
-        if not _LAST_RUN_STATS:
-            print("📊 No stats available. Run a spider first.")
-            return 0
-
-        print("📊 Recent Spider Statistics:")
-        print("-" * 60)
-        for spider_name, stats in _LAST_RUN_STATS.items():
-            print(f"🕷️  {spider_name}")
-            for k, v in stats.items():
-                print(f"    {k:<30} {v}")
-            print()
-        return 0
-
-    elif len(args) == 1:
-        spider_name = args[0]
-        if spider_name not in _LAST_RUN_STATS:
-            print(f"📊 No stats found for spider '{spider_name}'")
-            return 1
-
-        stats = _LAST_RUN_STATS[spider_name]
-        print(f"📊 Stats for '{spider_name}':")
-        print("-" * 60)
-        for k, v in stats.items():
-            print(f"    {k:<30} {v}")
-        return 0
-
-    else:
-        print("Usage: crawlo stats [spider_name]")
-        return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:]))
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo stats，查看最近运行的爬虫统计信息。
+"""
+import json
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, Any
+
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+
+from crawlo.utils.log import get_logger
+
+
+logger = get_logger(__name__)
+console = Console()
+
+# 默认存储目录（相对于项目根目录）
+STATS_DIR = "logs/stats"
+
+
+def get_stats_dir() -> Path:
+    """
+    获取统计文件存储目录，优先使用项目根下的 logs/stats/
+    如果不在项目中，回退到当前目录
+    """
+    current = Path.cwd()
+    for _ in range(10):
+        if (current / "crawlo.cfg").exists():
+            return current / STATS_DIR
+        if current == current.parent:
+            break
+        current = current.parent
+    return Path.cwd() / STATS_DIR
+
+
+def record_stats(crawler):
+    """
+    【供爬虫运行时调用】记录爬虫结束后的统计信息到 JSON 文件
+    需在 Crawler 的 closed 回调中调用
+    """
+    spider_name = getattr(crawler.spider, "name", "unknown")
+    stats = crawler.stats.get_stats() if crawler.stats else {}
+
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    stats_dir = Path(get_stats_dir())
+    stats_dir.mkdir(parents=True, exist_ok=True)
+
+    filename = stats_dir / f"{spider_name}_{timestamp}.json"
+    try:
+        with open(filename, "w", encoding="utf-8") as f:
+            json.dump({
+                "spider": spider_name,
+                "timestamp": datetime.now().isoformat(),
+                "stats": stats
+            }, f, ensure_ascii=False, indent=2, default=str)
+        logger.info(f"📊 Stats saved for spider '{spider_name}' → {filename}")
+    except Exception as e:
+        logger.error(f"Failed to save stats for '{spider_name}': {e}")
+
+
+def load_all_stats() -> Dict[str, list]:
+    """
+    加载所有已保存的统计文件，按 spider name 分组
+    返回: {spider_name: [stats_record, ...]}
+    """
+    stats_dir = get_stats_dir()
+    if not stats_dir.exists():
+        return {}
+
+    result = {}
+    json_files = sorted(stats_dir.glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True)
+
+    for file in json_files:
+        try:
+            with open(file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+            spider_name = data.get("spider", "unknown")
+            result.setdefault(spider_name, []).append(data)
+        except Exception as e:
+            logger.warning(f"Failed to load stats file {file}: {e}")
+    return result
+
+
+def format_value(v: Any) -> str:
+    """格式化值，防止太长或不可打印"""
+    if isinstance(v, float):
+        return f"{v:.4f}"
+    s = str(v)
+    if len(s) > 80:
+        return s[:77] + "..."
+    return s
+
+
+def display_stats_table(stats_data: dict, title: str = "Statistics"):
+    """通用函数：用 rich.table 展示统计数据"""
+    table = Table(title=title, box=box.ROUNDED, show_header=True, header_style="bold magenta")
+    table.add_column("Key", style="cyan", no_wrap=True)
+    table.add_column("Value", style="green")
+
+    for k in sorted(stats_data.keys()):
+        table.add_row(k, format_value(stats_data[k]))
+
+    console.print(table)
+
+
+def main(args):
+    """
+    主函数：查看统计信息
+    用法:
+        crawlo stats                 → 显示所有爬虫最近一次运行
+        crawlo stats myspider        → 显示指定爬虫所有历史记录
+        crawlo stats myspider --all  → 显示所有历史（同上）
+    """
+    if len(args) > 2:
+        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo stats[/blue] [spider_name] [--all]")
+        return 1
+
+    spider_name = None
+    show_all = False
+
+    if args:
+        spider_name = args[0]
+        show_all = "--all" in args or "-a" in args
+
+    all_stats = load_all_stats()
+
+    if not all_stats:
+        console.print(Panel(
+            Text.from_markup(
+                ":chart_with_upwards_trend: [bold]No stats found.[/bold]\n"
+                "💡 Run a spider first to generate statistics.\n"
+                f"📁 Stats directory: [cyan]{get_stats_dir()}[/cyan]"
+            ),
+            title="📊 Statistics",
+            border_style="yellow",
+            padding=(1, 2)
+        ))
+        return 0
+
+    # 显示所有爬虫最近一次运行
+    if not spider_name:
+        console.print(Panel(
+            "[bold]Recent Spider Statistics (last run)[/bold]",
+            title="📊 Spider Stats Overview",
+            border_style="green",
+            padding=(0, 1)
+        ))
+
+        for name, runs in all_stats.items():
+            latest = runs[0]
+            ts = latest['timestamp'][:19]
+            console.print(f"🕷️  [bold cyan]{name}[/bold cyan] ([green]{ts}[/green])")
+            display_stats_table(latest["stats"], title=f"Stats for {name}")
+            console.print()  # 空行分隔
+
+        return 0
+
+    # 显示指定爬虫的历史
+    if spider_name not in all_stats:
+        console.print(f"[bold red]:cross_mark: No stats found for spider '[cyan]{spider_name}[/cyan]'[/bold red]")
+        available = ', '.join(all_stats.keys())
+        if available:
+            console.print(f":bulb: Available spiders: [green]{available}[/green]")
+        return 1
+
+    runs = all_stats[spider_name]
+    if show_all:
+        console.print(f":bar_chart: [bold]All runs for '[cyan]{spider_name}[/cyan]' ({len(runs)} runs):[/bold]")
+    else:
+        runs = runs[:1]
+        console.print(f":bar_chart: [bold]Last run for '[cyan]{spider_name}[/cyan]':[/bold]")
+
+    for i, run in enumerate(runs, 1):
+        ts = run['timestamp']
+        subtitle = f"Run #{i} · {ts}" if show_all else f"Last Run · {ts}"
+        display_stats_table(run["stats"], title=f"Stats for {spider_name} — {subtitle}")
+        if i < len(runs):
+            console.print("─" * 60)
+
+    return 0

crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-