crawlo 1.4.6__py3-none-any.whl → 1.4.7__py3-none-any.whl

crawlo/commands/genspider.py

@@ -1,187 +1,187 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-08-31 22:36
-# @Author  : crawl-coder
-# @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
-"""
-import sys
-import re
-from pathlib import Path
-import configparser
-import importlib
-from rich.console import Console
-
-from .utils import (
-    get_project_root,
-    validate_project_environment,
-    show_error_panel,
-    show_success_panel,
-    validate_spider_name,
-    is_valid_domain
-)
-
-# 初始化 rich 控制台
-console = Console()
-
-TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
-
-
-def _render_template(tmpl_path, context):
-    """读取模板文件，替换 {{key}} 为 context 中的值"""
-    with open(tmpl_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-    for key, value in context.items():
-        content = content.replace(f'{{{{{key}}}}}', str(value))
-    return content
-
-
-def generate_class_name(spider_name):
-    """
-    根据爬虫名称生成类名
-    规则：蛇形命名 → 大驼峰命名 + 'Spider'
-    示例：
-        'news_spider' → 'NewsSpider'
-        'ofweek_standalone' → 'OfweekStandaloneSpider'
-        'baidu' → 'BaiduSpider'
-    """
-    # 如果名称已包含 'spider' 后缀，先去除
-    name_clean = spider_name
-    
-    # 定义要移除的后缀列表
-    spider_suffixes = ['_spider', 'spider']
-    
-    # 检查并移除后缀
-    for suffix in spider_suffixes:
-        if spider_name.endswith(suffix):
-            name_clean = spider_name[:-len(suffix)]
-            break
-
-    # 按分隔符拆分单词
-    words = re.split(r'[_-]', name_clean)
-
-    # 将每个单词首字母大写
-    capitalized_words = [word.capitalize() for word in words if word]
-
-    # 组合成类名
-    class_name = ''.join(capitalized_words) + 'Spider'
-
-    return class_name
-
-
-def main(args):
-    if len(args) < 2:
-        console.print("[bold red]错误:[/bold red] 用法: [blue]crawlo genspider[/blue] <爬虫名称> <域名>")
-        console.print("示例:")
-        console.print("   [blue]crawlo genspider[/blue] news_spider news.example.com")
-        console.print("   [blue]crawlo genspider[/blue] product_spider shop.example.com")
-        return 1
-
-    spider_name = args[0]
-    domain = args[1]
-
-    # 验证爬虫名称
-    if not validate_spider_name(spider_name):
-        show_error_panel(
-            "无效的爬虫名称",
-            f"爬虫名称 '[cyan]{spider_name}[/cyan]' 无效。\n"
-            "爬虫名称应:\n"
-            "  • 以小写字母开头\n"
-            "  • 只能包含小写字母、数字和下划线\n"
-            "  • 是有效的Python标识符"
-        )
-        return 1
-
-    # 验证域名格式
-    if not is_valid_domain(domain):
-        show_error_panel(
-            "无效的域名",
-            f"域名 '[cyan]{domain}[/cyan]' 格式无效。\n"
-            "请提供有效的域名，如 'example.com'"
-        )
-        return 1
-
-    # 验证项目环境
-    is_valid, project_package, error_msg = validate_project_environment()
-    if not is_valid:
-        show_error_panel("非Crawlo项目", error_msg)
-        return 1
-
-    project_root = get_project_root()
-
-    # 确定 items 模块的路径
-    items_module_path = f"{project_package}.items"
-
-    # 尝试导入 items 模块
-    default_item_class = "ExampleItem"  # 默认回退
-    try:
-        items_module = importlib.import_module(items_module_path)
-        # 获取模块中所有大写开头的类
-        item_classes = [
-            cls for cls in items_module.__dict__.values()
-            if isinstance(cls, type) and cls.__name__[0].isupper()  # 首字母大写
-        ]
-
-        if item_classes:
-            default_item_class = item_classes[0].__name__
-        else:
-            console.print(
-                "[yellow]警告:[/yellow] 在 [cyan]items.py[/cyan] 中未找到项目类，使用 [green]ExampleItem[/green]。")
-
-    except ImportError as e:
-        console.print(f"[yellow]警告:[/yellow] 导入 [cyan]{items_module_path}[/cyan] 失败: {e}")
-        # 仍使用默认 ExampleItem，不中断流程
-
-    # 创建爬虫文件
-    spiders_dir = project_root / project_package / 'spiders'
-    spiders_dir.mkdir(parents=True, exist_ok=True)
-
-    spider_file = spiders_dir / f'{spider_name}.py'
-    if spider_file.exists():
-        show_error_panel(
-            "爬虫已存在",
-            f"爬虫 '[cyan]{spider_name}[/cyan]' 已存在于\n[green]{spider_file}[/green]"
-        )
-        return 1
-
-    # 模板路径
-    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
-    if not tmpl_path.exists():
-        show_error_panel(
-            "模板未找到",
-            f"模板文件未找到于 [cyan]{tmpl_path}[/cyan]"
-        )
-        return 1
-
-    # 生成类名（使用新的转换函数）
-    class_name = generate_class_name(spider_name)
-
-    context = {
-        'spider_name': spider_name,
-        'domain': domain,
-        'project_name': project_package,
-        'item_class': default_item_class,
-        'class_name': class_name
-    }
-
-    try:
-        content = _render_template(tmpl_path, context)
-        with open(spider_file, 'w', encoding='utf-8') as f:
-            f.write(content)
-
-        console.print(f"[green]爬虫 '[bold]{spider_name}[/bold]' 创建成功！[/green]")
-        console.print(f"  → 位置: [cyan]{spider_file}[/cyan]")
-        console.print(f"  → 类名: [yellow]{class_name}[/yellow]")
-        console.print(f"  → 域名: [blue]{domain}[/blue]")
-        console.print("\n[bold]下一步操作:[/bold]")
-        console.print(f"  [blue]crawlo run[/blue] {spider_name}")
-        console.print(f"  [blue]crawlo check[/blue] {spider_name}")
-
-        return 0
-
-    except Exception as e:
-        show_error_panel(
-            "创建失败",
-            f"创建爬虫失败: {e}"
-        )
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-08-31 22:36
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo genspider baidu，创建爬虫。
+"""
+import sys
+import re
+from pathlib import Path
+import configparser
+import importlib
+from rich.console import Console
+
+from .utils import (
+    get_project_root,
+    validate_project_environment,
+    show_error_panel,
+    show_success_panel,
+    validate_spider_name,
+    is_valid_domain
+)
+
+# 初始化 rich 控制台
+console = Console()
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def generate_class_name(spider_name):
+    """
+    根据爬虫名称生成类名
+    规则：蛇形命名 → 大驼峰命名 + 'Spider'
+    示例：
+        'news_spider' → 'NewsSpider'
+        'ofweek_standalone' → 'OfweekStandaloneSpider'
+        'baidu' → 'BaiduSpider'
+    """
+    # 如果名称已包含 'spider' 后缀，先去除
+    name_clean = spider_name
+    
+    # 定义要移除的后缀列表
+    spider_suffixes = ['_spider', 'spider']
+    
+    # 检查并移除后缀
+    for suffix in spider_suffixes:
+        if spider_name.endswith(suffix):
+            name_clean = spider_name[:-len(suffix)]
+            break
+
+    # 按分隔符拆分单词
+    words = re.split(r'[_-]', name_clean)
+
+    # 将每个单词首字母大写
+    capitalized_words = [word.capitalize() for word in words if word]
+
+    # 组合成类名
+    class_name = ''.join(capitalized_words) + 'Spider'
+
+    return class_name
+
+
+def main(args):
+    if len(args) < 2:
+        console.print("[bold red]错误:[/bold red] 用法: [blue]crawlo genspider[/blue] <爬虫名称> <域名>")
+        console.print("示例:")
+        console.print("   [blue]crawlo genspider[/blue] news_spider news.example.com")
+        console.print("   [blue]crawlo genspider[/blue] product_spider shop.example.com")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 验证爬虫名称
+    if not validate_spider_name(spider_name):
+        show_error_panel(
+            "无效的爬虫名称",
+            f"爬虫名称 '[cyan]{spider_name}[/cyan]' 无效。\n"
+            "爬虫名称应:\n"
+            "  • 以小写字母开头\n"
+            "  • 只能包含小写字母、数字和下划线\n"
+            "  • 是有效的Python标识符"
+        )
+        return 1
+
+    # 验证域名格式
+    if not is_valid_domain(domain):
+        show_error_panel(
+            "无效的域名",
+            f"域名 '[cyan]{domain}[/cyan]' 格式无效。\n"
+            "请提供有效的域名，如 'example.com'"
+        )
+        return 1
+
+    # 验证项目环境
+    is_valid, project_package, error_msg = validate_project_environment()
+    if not is_valid:
+        show_error_panel("非Crawlo项目", error_msg)
+        return 1
+
+    project_root = get_project_root()
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    default_item_class = "ExampleItem"  # 默认回退
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [
+            cls for cls in items_module.__dict__.values()
+            if isinstance(cls, type) and cls.__name__[0].isupper()  # 首字母大写
+        ]
+
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            console.print(
+                "[yellow]警告:[/yellow] 在 [cyan]items.py[/cyan] 中未找到项目类，使用 [green]ExampleItem[/green]。")
+
+    except ImportError as e:
+        console.print(f"[yellow]警告:[/yellow] 导入 [cyan]{items_module_path}[/cyan] 失败: {e}")
+        # 仍使用默认 ExampleItem，不中断流程
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    spiders_dir.mkdir(parents=True, exist_ok=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        show_error_panel(
+            "爬虫已存在",
+            f"爬虫 '[cyan]{spider_name}[/cyan]' 已存在于\n[green]{spider_file}[/green]"
+        )
+        return 1
+
+    # 模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+    if not tmpl_path.exists():
+        show_error_panel(
+            "模板未找到",
+            f"模板文件未找到于 [cyan]{tmpl_path}[/cyan]"
+        )
+        return 1
+
+    # 生成类名（使用新的转换函数）
+    class_name = generate_class_name(spider_name)
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name
+    }
+
+    try:
+        content = _render_template(tmpl_path, context)
+        with open(spider_file, 'w', encoding='utf-8') as f:
+            f.write(content)
+
+        console.print(f"[green]爬虫 '[bold]{spider_name}[/bold]' 创建成功！[/green]")
+        console.print(f"  → 位置: [cyan]{spider_file}[/cyan]")
+        console.print(f"  → 类名: [yellow]{class_name}[/yellow]")
+        console.print(f"  → 域名: [blue]{domain}[/blue]")
+        console.print("\n[bold]下一步操作:[/bold]")
+        console.print(f"  [blue]crawlo run[/blue] {spider_name}")
+        console.print(f"  [blue]crawlo check[/blue] {spider_name}")
+
+        return 0
+
+    except Exception as e:
+        show_error_panel(
+            "创建失败",
+            f"创建爬虫失败: {e}"
+        )
         return 1

crawlo/commands/help.py

@@ -1,139 +1,141 @@
-#!/usr/bin/python
-# -*- coding: UTF-8 -*-
-"""
-# @Time    : 2025-09-12
-# @Author  : crawl-coder
-# @Desc    : 命令行入口：crawlo -h|--help，显示帮助信息。
-"""
-import sys
-from rich.console import Console
-from rich.table import Table
-from rich.panel import Panel
-from rich.text import Text
-from rich import box
-from crawlo.utils.env_config import get_version
-
-# 获取框架版本号
-VERSION = get_version()
-
-console = Console()
-
-
-def main(args):
-    """
-    主函数：显示帮助信息
-    用法:
-        crawlo -h|--help
-    """
-    # 检查是否有无效参数
-    if args and args[0] not in ['-h', '--help', 'help']:
-        console.print("[bold red]无效参数:[/bold red] [yellow]{}[/yellow]".format(args[0]))
-        console.print("[bold blue]提示:[/bold blue] 使用 [green]crawlo -h[/green] 或 [green]crawlo --help[/green] 查看帮助信息")
-        return 1
-
-    # 显示帮助信息
-    show_help()
-    return 0
-
-
-def show_help():
-    """显示完整的帮助信息"""
-    # 显示框架标题和版本
-    console.print(Panel(
-        Text.from_markup(f"[bold blue]Crawlo[/bold blue] [bold white]v{VERSION}[/bold white] - 异步爬虫框架"),
-        expand=False,
-        border_style="blue"
-    ))
-    
-    # 显示基本用法
-    console.print("[bold green]基本用法:[/bold green]")
-    console.print("  [blue]crawlo[/blue] [cyan]<command>[/cyan] [options]")
-    console.print()
-    
-    # 显示可用命令
-    console.print("[bold green]可用命令:[/bold green]")
-    table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
-    table.add_column("命令", style="cyan", width=15)
-    table.add_column("描述", style="white")
-    table.add_column("用法", style="yellow")
-    
-    table.add_row("startproject", "创建新项目", "crawlo startproject <project_name>")
-    table.add_row("genspider", "生成爬虫模板", "crawlo genspider <spider_name> [domain]")
-    table.add_row("run", "运行爬虫", "crawlo run <spider_name>|all [options]")
-    table.add_row("check", "检查爬虫代码", "crawlo check [options]")
-    table.add_row("list", "列出所有爬虫", "crawlo list")
-    table.add_row("stats", "查看统计信息", "crawlo stats [spider_name]")
-    table.add_row("help", "显示帮助信息", "crawlo -h|--help")
-    
-    console.print(table)
-    console.print()
-    
-    # 显示全局选项
-    console.print("[bold green]全局选项:[/bold green]")
-    table = Table(box=box.SIMPLE, show_header=False)
-    table.add_column("选项", style="cyan", width=15)
-    table.add_column("描述", style="white")
-    
-    table.add_row("-h, --help", "显示帮助信息")
-    table.add_row("-v, --version", "显示版本信息")
-    
-    console.print(table)
-    console.print()
-    
-    # 显示各命令的详细用法
-    console.print("[bold green]命令详细用法:[/bold green]")
-    
-    # run 命令
-    console.print("[bold cyan]run[/bold cyan] - 运行爬虫")
-    console.print("  用法: crawlo run <spider_name>|all [--json] [--no-stats]")
-    console.print("  示例:")
-    console.print("    crawlo run myspider")
-    console.print("    crawlo run all")
-    console.print("    crawlo run all --json --no-stats")
-    console.print()
-    
-
-    
-    # check 命令
-    console.print("[bold cyan]check[/bold cyan] - 检查爬虫代码")
-    console.print("  用法: crawlo check [--fix] [--ci] [--json] [--watch]")
-    console.print("  示例:")
-    console.print("    crawlo check")
-    console.print("    crawlo check --fix")
-    console.print("    crawlo check --ci --json")
-    console.print()
-    
-    # startproject 命令
-    console.print("[bold cyan]startproject[/bold cyan] - 创建新项目")
-    console.print("  用法: crawlo startproject <project_name>")
-    console.print("  示例:")
-    console.print("    crawlo startproject myproject")
-    console.print()
-    
-    # genspider 命令
-    console.print("[bold cyan]genspider[/bold cyan] - 生成爬虫模板")
-    console.print("  用法: crawlo genspider <spider_name> [domain]")
-    console.print("  示例:")
-    console.print("    crawlo genspider myspider example.com")
-    console.print()
-    
-    # list 命令
-    console.print("[bold cyan]list[/bold cyan] - 列出所有爬虫")
-    console.print("  用法: crawlo list")
-    console.print("  示例:")
-    console.print("    crawlo list")
-    console.print()
-    
-    # stats 命令
-    console.print("[bold cyan]stats[/bold cyan] - 查看统计信息")
-    console.print("  用法: crawlo stats [spider_name]")
-    console.print("  示例:")
-    console.print("    crawlo stats")
-    console.print("    crawlo stats myspider")
-    console.print()
-    
-    # 显示更多信息
-    # console.print("[bold green]更多信息:[/bold green]")
-    # console.print("  文档: https://crawlo.readthedocs.io/")
-    # console.print("  源码: https://github.com/crawl-coder/Crawlo")
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+# @Time    : 2025-09-12
+# @Author  : crawl-coder
+# @Desc    : 命令行入口：crawlo -h|--help，显示帮助信息。
+"""
+import sys
+from rich.console import Console
+from rich.table import Table
+from rich.panel import Panel
+from rich.text import Text
+from rich import box
+from crawlo.utils.config_manager import EnvConfigManager
+
+# 获取框架版本号
+VERSION = EnvConfigManager.get_version()
+
+console = Console()
+
+
+def main(args):
+    """
+    主函数：显示帮助信息
+    用法:
+        crawlo -h|--help
+    """
+    # 检查是否有无效参数
+    if args and args[0] not in ['-h', '--help', 'help']:
+        console.print("[bold red]无效参数:[/bold red] [yellow]{}[/yellow]".format(args[0]))
+        console.print("[bold blue]提示:[/bold blue] 使用 [green]crawlo -h[/green] 或 [green]crawlo --help[/green] 查看帮助信息")
+        return 1
+
+    # 显示帮助信息
+    show_help()
+    return 0
+
+
+def show_help():
+    """显示完整的帮助信息"""
+    # 显示框架标题和版本
+    console.print(Panel(
+        Text.from_markup(f"[bold blue]Crawlo[/bold blue] [bold white]v{VERSION}[/bold white] - 异步爬虫框架"),
+        expand=False,
+        border_style="blue"
+    ))
+    
+    # 显示基本用法
+    console.print("[bold green]基本用法:[/bold green]")
+    console.print("  [blue]crawlo[/blue] [cyan]<command>[/cyan] [options]")
+    console.print()
+    
+    # 显示可用命令
+    console.print("[bold green]可用命令:[/bold green]")
+    table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta")
+    table.add_column("命令", style="cyan", width=15)
+    table.add_column("描述", style="white")
+    table.add_column("用法", style="yellow")
+    
+    table.add_row("startproject", "创建新项目", "crawlo startproject <project_name>")
+    table.add_row("genspider", "生成爬虫模板", "crawlo genspider <spider_name> [domain]")
+    table.add_row("run", "运行爬虫", "crawlo run <spider_name>|all [options]")
+    table.add_row("check", "检查爬虫代码", "crawlo check [options]")
+    table.add_row("list", "列出所有爬虫", "crawlo list")
+    table.add_row("stats", "查看统计信息", "crawlo stats [spider_name]")
+    table.add_row("help", "显示帮助信息", "crawlo -h|--help")
+    
+    console.print(table)
+    console.print()
+    
+    # 显示全局选项
+    console.print("[bold green]全局选项:[/bold green]")
+    table = Table(box=box.SIMPLE, show_header=False)
+    table.add_column("选项", style="cyan", width=15)
+    table.add_column("描述", style="white")
+    
+    table.add_row("-h, --help", "显示帮助信息")
+    table.add_row("-v, --version", "显示版本信息")
+    
+    console.print(table)
+    console.print()
+    
+    # 显示各命令的详细用法
+    console.print("[bold green]命令详细用法:[/bold green]")
+    
+    # run 命令
+    console.print("[bold cyan]run[/bold cyan] - 运行爬虫")
+    console.print("  用法: crawlo run <spider_name>|all [--json] [--no-stats] [--log-level LEVEL] [--config CONFIG] [--concurrency NUM]")
+    console.print("  示例:")
+    console.print("    crawlo run myspider")
+    console.print("    crawlo run all")
+    console.print("    crawlo run all --json --no-stats")
+    console.print("    crawlo run myspider --log-level DEBUG")
+    console.print("    crawlo run myspider --concurrency 32")
+    console.print()
+    
+
+    
+    # check 命令
+    console.print("[bold cyan]check[/bold cyan] - 检查爬虫代码")
+    console.print("  用法: crawlo check [--fix] [--ci] [--json] [--watch]")
+    console.print("  示例:")
+    console.print("    crawlo check")
+    console.print("    crawlo check --fix")
+    console.print("    crawlo check --ci --json")
+    console.print()
+    
+    # startproject 命令
+    console.print("[bold cyan]startproject[/bold cyan] - 创建新项目")
+    console.print("  用法: crawlo startproject <project_name>")
+    console.print("  示例:")
+    console.print("    crawlo startproject myproject")
+    console.print()
+    
+    # genspider 命令
+    console.print("[bold cyan]genspider[/bold cyan] - 生成爬虫模板")
+    console.print("  用法: crawlo genspider <spider_name> [domain]")
+    console.print("  示例:")
+    console.print("    crawlo genspider myspider example.com")
+    console.print()
+    
+    # list 命令
+    console.print("[bold cyan]list[/bold cyan] - 列出所有爬虫")
+    console.print("  用法: crawlo list")
+    console.print("  示例:")
+    console.print("    crawlo list")
+    console.print()
+    
+    # stats 命令
+    console.print("[bold cyan]stats[/bold cyan] - 查看统计信息")
+    console.print("  用法: crawlo stats [spider_name]")
+    console.print("  示例:")
+    console.print("    crawlo stats")
+    console.print("    crawlo stats myspider")
+    console.print()
+    
+    # 显示更多信息
+    # console.print("[bold green]更多信息:[/bold green]")
+    # console.print("  文档: https://crawlo.readthedocs.io/")
+    # console.print("  源码: https://github.com/crawl-coder/Crawlo")
     # console.print("  问题: https://github.com/crawl-coder/Crawlo/issues")