crawlo 1.2.2__py3-none-any.whl → 1.2.3__py3-none-any.whl

crawlo/commands/run.py

@@ -56,8 +56,8 @@ def main(args):
         crawlo run <spider_name>|all [--json] [--no-stats]
     """
     if len(args) < 1:
-        console.print("[bold red]❌ Usage:[/bold red] [blue]crawlo run[/blue] <spider_name>|all [bold yellow][--json] [--no-stats][/bold yellow]")
-        console.print("💡 Examples:")
+        console.print("[bold red]❌ 用法:[/bold red] [blue]crawlo run[/blue] <爬虫名称>|all [bold yellow][--json] [--no-stats][/bold yellow]")
+        console.print("💡 示例:")
         console.print("   [blue]crawlo run baidu[/blue]")
         console.print("   [blue]crawlo run all[/blue]")
         console.print("   [blue]crawlo run all --json --no-stats[/blue]")
@@ -72,14 +72,14 @@ def main(args):
         # 1. 查找项目根目录
         project_root = get_project_root()
         if not project_root:
-            msg = ":cross_mark: [bold red]Cannot find 'crawlo.cfg'[/bold red]\n💡 Run this command inside your project directory."
+            msg = ":cross_mark: [bold red]找不到 'crawlo.cfg'[/bold red]\n💡 请在项目目录中运行此命令。"
             if show_json:
-                console.print_json(data={"success": False, "error": "Project root not found"})
+                console.print_json(data={"success": False, "error": "未找到项目根目录"})
                 return 1
             else:
                 console.print(Panel(
                     Text.from_markup(msg),
-                    title="❌ Not in a Crawlo Project",
+                    title="❌ 非Crawlo项目",
                     border_style="red",
                     padding=(1, 2)
                 ))
@@ -92,24 +92,24 @@ def main(args):
         # 2. 读取 crawlo.cfg 获取 settings 模块
         cfg_file = project_root / "crawlo.cfg"
         if not cfg_file.exists():
-            msg = f"crawlo.cfg not found in {project_root}"
+            msg = f"在 {project_root} 中未找到 crawlo.cfg"
             if show_json:
                 console.print_json(data={"success": False, "error": msg})
                 return 1
             else:
-                console.print(Panel(msg, title="❌ Missing Config", border_style="red"))
+                console.print(Panel(msg, title="❌ 缺少配置文件", border_style="red"))
                 return 1
 
         config = configparser.ConfigParser()
         config.read(cfg_file, encoding="utf-8")
 
         if not config.has_section("settings") or not config.has_option("settings", "default"):
-            msg = "Missing [settings] section or 'default' option in crawlo.cfg"
+            msg = "crawlo.cfg 中缺少 [settings] 部分或 'default' 选项"
             if show_json:
                 console.print_json(data={"success": False, "error": msg})
                 return 1
             else:
-                console.print(Panel(msg, title="❌ Invalid Config", border_style="red"))
+                console.print(Panel(msg, title="❌ 无效配置", border_style="red"))
                 return 1
 
         settings_module = config.get("settings", "default")
@@ -119,12 +119,12 @@ def main(args):
         try:
             import_module(project_package)
         except ImportError as e:
-            msg = f"Failed to import project package '{project_package}': {e}"
+            msg = f"导入项目包 '{project_package}' 失败: {e}"
             if show_json:
                 console.print_json(data={"success": False, "error": msg})
                 return 1
             else:
-                console.print(Panel(msg, title="❌ Import Error", border_style="red"))
+                console.print(Panel(msg, title="❌ 导入错误", border_style="red"))
                 return 1
 
         # 4. 加载 settings 和爬虫模块
@@ -136,20 +136,20 @@ def main(args):
         if spider_arg.lower() == "all":
             spider_names = process.get_spider_names()
             if not spider_names:
-                msg = "No spiders found."
+                msg = "未找到爬虫。"
                 if show_json:
                     console.print_json(data={"success": False, "error": msg})
                     return 1
                 else:
                     console.print(Panel(
                         Text.from_markup(
-                            ":cross_mark: [bold red]No spiders found.[/bold red]\n\n"
-                            "[bold]💡 Make sure:[/bold]\n"
-                            "  • Spiders are defined in '[cyan]spiders/[/cyan]'\n"
-                            "  • They have a [green]`name`[/green] attribute\n"
-                            "  • Modules are imported (e.g. via [cyan]__init__.py[/cyan])"
+                            ":cross_mark: [bold red]未找到爬虫。[/bold red]\n\n"
+                            "[bold]💡 确保:[/bold]\n"
+                            "  • 爬虫定义于 '[cyan]spiders/[/cyan]' 目录\n"
+                            "  • 具有 [green]`name`[/green] 属性\n"
+                            "  • 模块已导入 (例如通过 [cyan]__init__.py[/cyan])"
                         ),
-                        title="❌ No Spiders",
+                        title="❌ 未找到爬虫",
                         border_style="red",
                         padding=(1, 2)
                     ))
@@ -157,13 +157,13 @@ def main(args):
 
             # 显示即将运行的爬虫列表
             table = Table(
-                title=f"🚀 Starting ALL {len(spider_names)} spider(s)",
+                title=f"🚀 启动全部 {len(spider_names)} 个爬虫",
                 box=box.ROUNDED,
                 show_header=True,
                 header_style="bold magenta"
             )
-            table.add_column("Name", style="cyan")
-            table.add_column("Class", style="green")
+            table.add_column("名称", style="cyan")
+            table.add_column("类名", style="green")
 
             for name in sorted(spider_names):
                 cls = process.get_spider_class(name)
@@ -183,15 +183,15 @@ def main(args):
                 TextColumn("[progress.description]{task.description}"),
                 transient=True,
             ) as progress:
-                task = progress.add_task("Running all spiders...", total=None)
+                task = progress.add_task("正在运行所有爬虫...", total=None)
                 asyncio.run(process.crawl(spider_names))
 
             if show_json:
                 console.print_json(data={"success": True, "spiders": spider_names})
             else:
                 console.print(Panel(
-                    ":tada: [bold green]All spiders completed successfully![/bold green]",
-                    title="✅ All Done",
+                    ":tada: [bold green]所有爬虫运行完成！[/bold green]",
+                    title="✅ 全部完成",
                     border_style="green"
                 ))
             return 0
@@ -200,7 +200,7 @@ def main(args):
         spider_name = spider_arg
         if not process.is_spider_registered(spider_name):
             available = process.get_spider_names()
-            msg = f"Spider '[cyan]{spider_name}[/cyan]' not found."
+            msg = f"爬虫 '[cyan]{spider_name}[/cyan]' 未找到。"
             if show_json:
                 console.print_json(data={
                     "success": False,
@@ -211,16 +211,16 @@ def main(args):
             else:
                 panel_content = Text.from_markup(msg + "\n")
                 if available:
-                    panel_content.append("\n💡 Available spiders:\n")
+                    panel_content.append("\n💡 可用爬虫:\n")
                     for name in sorted(available):
                         cls = process.get_spider_class(name)
                         panel_content.append(f"  • [cyan]{name}[/cyan] ([green]{cls.__name__}[/green])\n")
                 else:
-                    panel_content.append("\n💡 No spiders found. Check your spiders module.")
+                    panel_content.append("\n💡 未找到爬虫。请检查爬虫模块。")
 
                 console.print(Panel(
                     panel_content,
-                    title="❌ Spider Not Found",
+                    title="❌ 爬虫未找到",
                     border_style="red",
                     padding=(1, 2)
                 ))
@@ -231,7 +231,7 @@ def main(args):
         # 显示启动信息
         if not show_json:
             info_table = Table(
-                title=f"🚀 Starting Spider: [bold cyan]{spider_name}[/bold cyan]",
+                title=f"🚀 启动爬虫: [bold cyan]{spider_name}[/bold cyan]",
                 box=box.SIMPLE,
                 show_header=False,
                 title_style="bold green"
@@ -255,21 +255,21 @@ def main(args):
             TextColumn("[progress.description]{task.description}"),
             transient=True,
         ) as progress:
-            task = progress.add_task(f"Running {spider_name}...", total=None)
+            task = progress.add_task(f"正在运行 {spider_name}...", total=None)
             asyncio.run(process.crawl(spider_name))
 
         if show_json:
             console.print_json(data={"success": True, "spider": spider_name})
         else:
             console.print(Panel(
-                f":tada: [bold green]Spider '[cyan]{spider_name}[/cyan]' completed successfully![/bold green]",
-                title="✅ Done",
+                f":tada: [bold green]爬虫 '[cyan]{spider_name}[/cyan]' 运行完成！[/bold green]",
+                title="✅ 完成",
                 border_style="green"
             ))
         return 0
 
     except KeyboardInterrupt:
-        msg = "⚠️  Spider interrupted by user."
+        msg = "⚠️  爬虫被用户中断。"
         if show_json:
             console.print_json(data={"success": False, "error": msg})
         else:
@@ -277,7 +277,7 @@ def main(args):
         return 1
     except Exception as e:
         logger.exception("Exception during 'crawlo run'")
-        msg = f"Unexpected error: {e}"
+        msg = f"意外错误: {e}"
         if show_json:
             console.print_json(data={"success": False, "error": msg})
         else:

crawlo/commands/startproject.py

@@ -211,13 +211,13 @@ def validate_project_name(project_name: str) -> tuple[bool, str]:
     """
     # 检查是否为空
     if not project_name or not project_name.strip():
-        return False, "Project name cannot be empty"
+        return False, "项目名称不能为空"
     
     project_name = project_name.strip()
     
     # 检查长度
     if len(project_name) > 50:
-        return False, "Project name too long (max 50 characters)"
+        return False, "项目名称太长（最多50个字符）"
     
     # 检查是否为Python关键字
     python_keywords = {
@@ -228,22 +228,21 @@ def validate_project_name(project_name: str) -> tuple[bool, str]:
         'while', 'with', 'yield'
     }
     if project_name in python_keywords:
-        return False, f"'{project_name}' is a Python keyword and cannot be used as project name"
+        return False, f"'{project_name}' 是Python关键字，不能用作项目名称"
     
     # 检查是否为有效的Python标识符
     if not project_name.isidentifier():
-        return False, "Project name must be a valid Python identifier"
+        return False, "项目名称必须是有效的Python标识符"
     
     # 检查格式（建议使用snake_case）
     if not re.match(r'^[a-z][a-z0-9_]*$', project_name):
         return False, (
-            "Project name should start with lowercase letter and "
-            "contain only lowercase letters, numbers, and underscores"
+            "项目名称应以小写字母开头，只能包含小写字母、数字和下划线"
         )
     
     # 检查是否以数字结尾（不推荐）
     if project_name[-1].isdigit():
-        return False, "Project name should not end with a number"
+        return False, "项目名称不应以数字结尾"
     
     return True, ""
 
@@ -284,8 +283,8 @@ def show_module_options():
 
 def main(args):
     if len(args) < 1:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo startproject[/blue] <project_name> [template_type] [--modules module1,module2]")
-        console.print("💡 Examples:")
+        console.print("[bold red]错误:[/bold red] 用法: [blue]crawlo startproject[/blue] <项目名称> [模板类型] [--modules 模块1,模块2]")
+        console.print("💡 示例:")
         console.print("   [blue]crawlo startproject[/blue] my_spider_project")
         console.print("   [blue]crawlo startproject[/blue] news_crawler simple")
         console.print("   [blue]crawlo startproject[/blue] ecommerce_spider distributed --modules mysql,proxy")
@@ -315,8 +314,8 @@ def main(args):
     # 验证模板类型
     if template_type not in TEMPLATE_TYPES:
         show_error_panel(
-            "Invalid Template Type",
-            f"Template type '[cyan]{template_type}[/cyan]' is not supported.\n"
+            "无效的模板类型",
+            f"不支持模板类型 '[cyan]{template_type}[/cyan]'。\n"
         )
         show_template_options()
         return 1
@@ -325,14 +324,14 @@ def main(args):
     is_valid, error_msg = validate_project_name(project_name)
     if not is_valid:
         show_error_panel(
-            "Invalid Project Name", 
-            f"[cyan]{project_name}[/cyan] is not a valid project name.\n"
+            "无效的项目名称", 
+            f"[cyan]{project_name}[/cyan] 不是有效的项目名称。\n"
             f"❌ {error_msg}\n\n"
-            "💡 Project name should:\n"
-            "  • Start with lowercase letter\n"
-            "  • Contain only lowercase letters, numbers, and underscores\n"
-            "  • Be a valid Python identifier\n"
-            "  • Not be a Python keyword"
+            "💡 项目名称应:\n"
+            "  • 以小写字母开头\n"
+            "  • 只能包含小写字母、数字和下划线\n"
+            "  • 是有效的Python标识符\n"
+            "  • 不能是Python关键字"
         )
         return 1
     
@@ -340,9 +339,9 @@ def main(args):
 
     if project_dir.exists():
         show_error_panel(
-            "Directory Exists",
-            f"Directory '[cyan]{project_dir}[/cyan]' already exists.\n"
-            "💡 Choose a different project name or remove the existing directory."
+            "目录已存在",
+            f"目录 '[cyan]{project_dir}[/cyan]' 已存在。\n"
+            "💡 请选择不同的项目名称或删除现有目录。"
         )
         return 1
 
@@ -358,26 +357,26 @@ def main(args):
         if cfg_template.exists():
             cfg_content = _render_template(cfg_template, context)
             (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
-            console.print(f":white_check_mark: Created [green]{project_dir / 'crawlo.cfg'}[/green]")
+            console.print(f":white_check_mark: 已创建 [green]{project_dir / 'crawlo.cfg'}[/green]")
         else:
-            console.print("[yellow]⚠ Warning:[/yellow] Template 'crawlo.cfg.tmpl' not found.")
+            console.print("[yellow]⚠ 警告:[/yellow] 找不到模板 'crawlo.cfg.tmpl'。")
 
         # 3. 复制并渲染项目包内容
         package_dir = project_dir / project_name
         _copytree_with_templates(template_dir, package_dir, context, template_type, modules)
-        console.print(f":white_check_mark: Created project package: [green]{package_dir}[/green]")
+        console.print(f":white_check_mark: 已创建项目包: [green]{package_dir}[/green]")
 
         # 4. 创建 logs 目录
         (project_dir / 'logs').mkdir(exist_ok=True)
-        console.print(":white_check_mark: Created logs directory")
+        console.print(":white_check_mark: 已创建 logs 目录")
         
         # 5. 创建 output 目录（用于数据输出）
         (project_dir / 'output').mkdir(exist_ok=True)
-        console.print(":white_check_mark: Created output directory")
+        console.print(":white_check_mark: 已创建 output 目录")
 
         # 成功面板
-        success_text = Text.from_markup(f"Project '[bold cyan]{project_name}[/bold cyan]' created successfully!")
-        console.print(Panel(success_text, title=":rocket: Success", border_style="green", padding=(1, 2)))
+        success_text = Text.from_markup(f"项目 '[bold cyan]{project_name}[/bold cyan]' 创建成功！")
+        console.print(Panel(success_text, title=":rocket: 成功", border_style="green", padding=(1, 2)))
         
         # 显示使用的模板类型
         if template_type != 'default':
@@ -389,15 +388,15 @@ def main(args):
 
         # 下一步操作提示（对齐美观 + 语法高亮）
         next_steps = f"""
-        [bold]🚀 Next steps:[/bold]
+        [bold]🚀 下一步操作:[/bold]
         [blue]cd[/blue] {project_name}
         [blue]crawlo genspider[/blue] example example.com
         [blue]crawlo run[/blue] example
         
-        [bold]📚 Learn more:[/bold]
-        [blue]crawlo list[/blue]                    # List all spiders
-        [blue]crawlo check[/blue] example          # Check spider validity
-        [blue]crawlo stats[/blue]                  # View statistics
+        [bold]📚 了解更多:[/bold]
+        [blue]crawlo list[/blue]                    # 列出所有爬虫
+        [blue]crawlo check[/blue] example          # 检查爬虫有效性
+        [blue]crawlo stats[/blue]                  # 查看统计信息
         """.strip()
         console.print(next_steps)
 
@@ -405,12 +404,12 @@ def main(args):
 
     except Exception as e:
         show_error_panel(
-            "Creation Failed",
-            f"Failed to create project: {e}"
+            "创建失败",
+            f"创建项目失败: {e}"
         )
         if project_dir.exists():
             shutil.rmtree(project_dir, ignore_errors=True)
-            console.print("[red]:cross_mark: Cleaned up partially created project.[/red]")
+            console.print("[red]:cross_mark: 已清理部分创建的项目。[/red]")
         return 1
 
 if __name__ == "__main__":

crawlo/commands/stats.py

@@ -61,9 +61,9 @@ def record_stats(crawler):
                 "timestamp": datetime.now().isoformat(),
                 "stats": stats
             }, f, ensure_ascii=False, indent=2, default=str)
-        logger.info(f"📊 Stats saved for spider '{spider_name}' → {filename}")
+        logger.info(f"📊 爬虫 '{spider_name}' 的统计信息已保存 → {filename}")
     except Exception as e:
-        logger.error(f"Failed to save stats for '{spider_name}': {e}")
+        logger.error(f"保存 '{spider_name}' 的统计信息失败: {e}")
 
 
 def load_all_stats() -> Dict[str, list]:
@@ -85,7 +85,7 @@ def load_all_stats() -> Dict[str, list]:
             spider_name = data.get("spider", "unknown")
             result.setdefault(spider_name, []).append(data)
         except Exception as e:
-            logger.warning(f"Failed to load stats file {file}: {e}")
+            logger.warning(f"加载统计文件 {file} 失败: {e}")
     return result
 
 
@@ -99,11 +99,11 @@ def format_value(v: Any) -> str:
     return s
 
 
-def display_stats_table(stats_data: dict, title: str = "Statistics"):
+def display_stats_table(stats_data: dict, title: str = "统计信息"):
     """通用函数：用 rich.table 展示统计数据"""
     table = Table(title=title, box=box.ROUNDED, show_header=True, header_style="bold magenta")
-    table.add_column("Key", style="cyan", no_wrap=True)
-    table.add_column("Value", style="green")
+    table.add_column("键", style="cyan", no_wrap=True)
+    table.add_column("值", style="green")
 
     for k in sorted(stats_data.keys()):
         table.add_row(k, format_value(stats_data[k]))
@@ -120,7 +120,7 @@ def main(args):
         crawlo stats myspider --all  → 显示所有历史（同上）
     """
     if len(args) > 2:
-        console.print("[bold red]Error:[/bold red] Usage: [blue]crawlo stats[/blue] [spider_name] [--all]")
+        console.print("[bold red]错误:[/bold red] 用法: [blue]crawlo stats[/blue] [爬虫名称] [--all]")
         return 1
 
     spider_name = None
@@ -135,11 +135,11 @@ def main(args):
     if not all_stats:
         console.print(Panel(
             Text.from_markup(
-                ":chart_with_upwards_trend: [bold]No stats found.[/bold]\n"
-                "💡 Run a spider first to generate statistics.\n"
-                f"📁 Stats directory: [cyan]{get_stats_dir()}[/cyan]"
+                ":chart_with_upwards_trend: [bold]未找到统计信息。[/bold]\n"
+                "💡 先运行一个爬虫以生成统计信息。\n"
+                f"📁 统计目录: [cyan]{get_stats_dir()}[/cyan]"
             ),
-            title="📊 Statistics",
+            title="📊 统计信息",
             border_style="yellow",
             padding=(1, 2)
         ))
@@ -148,8 +148,8 @@ def main(args):
     # 显示所有爬虫最近一次运行
     if not spider_name:
         console.print(Panel(
-            "[bold]Recent Spider Statistics (last run)[/bold]",
-            title="📊 Spider Stats Overview",
+            "[bold]最近的爬虫统计信息（上次运行）[/bold]",
+            title="📊 爬虫统计概览",
             border_style="green",
             padding=(0, 1)
         ))
@@ -158,31 +158,31 @@ def main(args):
             latest = runs[0]
             ts = latest['timestamp'][:19]
             console.print(f"🕷️  [bold cyan]{name}[/bold cyan] ([green]{ts}[/green])")
-            display_stats_table(latest["stats"], title=f"Stats for {name}")
+            display_stats_table(latest["stats"], title=f"{name} 的统计信息")
             console.print()  # 空行分隔
 
         return 0
 
     # 显示指定爬虫的历史
     if spider_name not in all_stats:
-        console.print(f"[bold red]:cross_mark: No stats found for spider '[cyan]{spider_name}[/cyan]'[/bold red]")
+        console.print(f"[bold red]:cross_mark: 未找到爬虫 '[cyan]{spider_name}[/cyan]' 的统计信息[/bold red]")
         available = ', '.join(all_stats.keys())
         if available:
-            console.print(f":bulb: Available spiders: [green]{available}[/green]")
+            console.print(f":bulb: 可用爬虫: [green]{available}[/green]")
         return 1
 
     runs = all_stats[spider_name]
     if show_all:
-        console.print(f":bar_chart: [bold]All runs for '[cyan]{spider_name}[/cyan]' ({len(runs)} runs):[/bold]")
+        console.print(f":bar_chart: [bold]'[cyan]{spider_name}[/cyan]' 的所有运行记录 ({len(runs)} 次):[/bold]")
     else:
         runs = runs[:1]
-        console.print(f":bar_chart: [bold]Last run for '[cyan]{spider_name}[/cyan]':[/bold]")
+        console.print(f":bar_chart: [bold]'[cyan]{spider_name}[/cyan]' 的上次运行:[/bold]")
 
     for i, run in enumerate(runs, 1):
         ts = run['timestamp']
-        subtitle = f"Run #{i} · {ts}" if show_all else f"Last Run · {ts}"
-        display_stats_table(run["stats"], title=f"Stats for {spider_name} — {subtitle}")
+        subtitle = f"运行 #{i} · {ts}" if show_all else f"上次运行 · {ts}"
+        display_stats_table(run["stats"], title=f"{spider_name} 的统计信息 — {subtitle}")
         if i < len(runs):
             console.print("─" * 60)
 
-    return 0
+    return 0

crawlo/commands/utils.py

@@ -46,7 +46,7 @@ def validate_project_environment() -> Tuple[bool, Optional[str], Optional[str]]:
     # 1. 查找项目根目录
     project_root = get_project_root()
     if not project_root:
-        return False, None, "Cannot find 'crawlo.cfg'. Run this command inside your project directory."
+        return False, None, "找不到 'crawlo.cfg'。请在项目目录中运行此命令。"
     
     # 2. 将项目根加入 Python 路径
     project_root_str = str(project_root)
@@ -60,10 +60,10 @@ def validate_project_environment() -> Tuple[bool, Optional[str], Optional[str]]:
     try:
         config.read(cfg_file, encoding="utf-8")
     except Exception as e:
-        return False, None, f"Failed to read crawlo.cfg: {e}"
+        return False, None, f"读取 crawlo.cfg 失败: {e}"
     
     if not config.has_section("settings") or not config.has_option("settings", "default"):
-        return False, None, "Invalid crawlo.cfg: missing [settings] section or 'default' option"
+        return False, None, "无效的 crawlo.cfg：缺少 [settings] 部分或 'default' 选项"
     
     # 4. 获取项目包名
     settings_module = config.get("settings", "default")
@@ -73,7 +73,7 @@ def validate_project_environment() -> Tuple[bool, Optional[str], Optional[str]]:
     try:
         import_module(project_package)
     except ImportError as e:
-        return False, None, f"Failed to import project package '{project_package}': {e}"
+        return False, None, f"导入项目包 '{project_package}' 失败: {e}"
     
     return True, project_package, None
 

crawlo/config.py

@@ -19,11 +19,11 @@ Crawlo 配置工厂
     config = CrawloConfig.from_env()
 """
 
-from typing import Dict, Any, Optional, Union
-import os
-from crawlo.mode_manager import ModeManager, standalone_mode, distributed_mode, auto_mode, from_env
-from crawlo.utils.log import get_logger
+from typing import Dict, Any, Optional
+
 from crawlo.config_validator import validate_config
+from crawlo.mode_manager import standalone_mode, distributed_mode, auto_mode, from_env
+from crawlo.utils.log import get_logger
 
 
 class CrawloConfig:

crawlo/config_validator.py

@@ -5,9 +5,8 @@
 ==========
 提供配置项的验证和默认值设置功能，确保配置的合理性和一致性。
 """
+from typing import Dict, Any, List, Tuple
 
-from typing import Dict, Any, List, Tuple, Optional
-import re
 from crawlo.utils.log import get_logger
 
 

crawlo/data/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Crawlo 数据模块
+包含用户代理、字典等数据文件
+"""