crawlo 1.0.4__py3-none-any.whl → 1.0.6__py3-none-any.whl

crawlo/__init__.py

@@ -1,9 +1,25 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-# from crawlo.spider import Spider
-from crawlo.items.items import Item
-from crawlo.network.request import Request
-from crawlo.network.response import Response
-from crawlo.downloader import DownloaderBase
-from crawlo.middleware import BaseMiddleware
-from .__version__ import __version__
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo - 一个异步爬虫框架
+"""
+from crawlo.spider import Spider
+from crawlo.items.items import Item
+from crawlo.network.request import Request
+from crawlo.network.response import Response
+from crawlo.downloader import DownloaderBase
+from crawlo.middleware import BaseMiddleware
+
+# 版本号
+from crawlo.__version__ import __version__
+
+# 可选：定义对外暴露的接口
+__all__ = [
+    'Spider',
+    'Item',
+    'Request',
+    'Response',
+    'DownloaderBase',
+    'BaseMiddleware',
+    '__version__',
+]

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.4"
+__version__ = "1.0.6"

crawlo/cli.py

@@ -0,0 +1,41 @@
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+
+
+def main():
+    # 获取所有可用命令
+    commands = get_commands()
+
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]"
+    )
+    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
+    # 注意：这里不添加具体参数，由子命令解析
+
+    # 只解析命令
+    args, unknown = parser.parse_known_args()
+
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

crawlo/commands/__init__.py

@@ -0,0 +1,10 @@
+# crawlo/commands/__init__.py
+# 定义可用的命令
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+}
+
+def get_commands():
+    return _commands

crawlo/commands/genspider.py

@@ -0,0 +1,111 @@
+import os
+import sys
+from pathlib import Path
+import configparser
+import importlib
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        print("Usage: crawlo genspider <spider_name> <domain>")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 查找项目根目录
+    project_root = None
+    current = Path.cwd()
+    while True:
+        cfg_file = current / 'crawlo.cfg'
+        if cfg_file.exists():
+            project_root = current
+            break
+        parent = current.parent
+        if parent == current:
+            break
+        current = parent
+
+    if not project_root:
+        print("Error: Not a crawlo project. crawlo.cfg not found.")
+        return 1
+
+    # 将项目根目录加入 sys.path
+    if str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
+
+    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_file, encoding='utf-8')
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
+    except Exception as e:
+        print(f"Error reading crawlo.cfg: {e}")
+        return 1
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [cls for cls in items_module.__dict__.values()
+                        if isinstance(cls, type) and cls.__name__.isupper()]
+
+        # 如果找到了类，使用第一个作为默认
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            default_item_class = "ExampleItem"  # 回退到示例
+    except ImportError as e:
+        print(f"Error importing items module '{items_module_path}': {e}")
+        default_item_class = "ExampleItem"
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    if not spiders_dir.exists():
+        spiders_dir.mkdir(parents=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        print(f"Error: Spider '{spider_name}' already exists.")
+        return 1
+
+    # ✅ 修正模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+
+    if not tmpl_path.exists():
+        print(f"Error: Template file not found at {tmpl_path}")
+        return 1
+
+    # ✅ 生成正确的类名
+    class_name = f"{spider_name.capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name  # ✅ 添加处理好的类名
+    }
+
+    content = _render_template(tmpl_path, context)
+
+    with open(spider_file, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    print(f"Spider '{spider_name}' created in {spider_file}")
+    return 0

crawlo/commands/run.py

@@ -0,0 +1,149 @@
+# crawlo/commands/run.py
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+from crawlo.utils.spider_loader import SpiderLoader
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    """
+    运行指定爬虫的主函数
+    用法: crawlo run <spider_name>
+    """
+    if len(args) < 1:
+        print("Usage: crawlo run <spider_name>")
+        print("Example: crawlo run baidu")
+        return 1
+
+    spider_name = args[0]
+
+    try:
+        # 1. 获取项目根目录
+        project_root = get_settings()
+
+        # 将项目根目录添加到 Python 路径
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+        # 2. 读取配置文件获取项目包名
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 3. 查找并加载指定名称的 Spider
+        spider_class = find_spider_by_name(project_package, spider_name)
+        if spider_class is None:
+            return 1
+
+        # 4. 创建 CrawlerProcess 并运行单个爬虫
+        settings = get_settings()
+        process = CrawlerProcess(settings)
+
+        print(f"🚀 Starting spider: {spider_class.name}")
+        print(f"📁 Project: {project_package}")
+        print(f"🕷️  Class: {spider_class.__name__}")
+        print("-" * 50)
+
+        # 运行单个爬虫
+        asyncio.run(process.crawl(spider_class))
+
+        print("-" * 50)
+        print("✅ Spider completed successfully!")
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error running spider: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+def find_spider_by_name(project_package: str, target_spider_name: str):
+    """使用 SpiderLoader 查找爬虫"""
+    loader = SpiderLoader(project_package)
+    spider_class = loader.load(target_spider_name)
+
+    if spider_class is None:
+        print(f"❌ Error: Spider with name '{target_spider_name}' not found")
+        print("💡 Available spiders:")
+        available_spiders = loader.list()
+        for spider_name in available_spiders:
+            print(f"   - {spider_name}")
+        return None
+
+    return spider_class
+
+
+def list_available_spiders(project_package: str):
+    """
+    列出所有可用的爬虫
+    """
+    spiders_dir = Path.cwd() / project_package / 'spiders'
+    if not spiders_dir.exists():
+        print("   No spiders directory found")
+        return
+
+    spider_count = 0
+    for py_file in spiders_dir.glob("*.py"):
+        if py_file.name.startswith('_'):
+            continue
+
+        module_name = py_file.stem
+        spider_module_path = f"{project_package}.spiders.{module_name}"
+
+        try:
+            module = importlib.import_module(spider_module_path)
+        except ImportError:
+            continue
+
+        # 查找模块中所有 Spider 子类
+        from crawlo.spider import Spider
+        for attr_name in dir(module):
+            attr_value = getattr(module, attr_name)
+            if (isinstance(attr_value, type) and
+                    issubclass(attr_value, Spider) and
+                    attr_value != Spider and
+                    hasattr(attr_value, 'name')):
+                print(f"   - {attr_value.name} (class: {attr_value.__name__}, module: {module_name})")
+                spider_count += 1
+
+    if spider_count == 0:
+        print("   No spiders found")
+
+
+def run_spider_by_name(spider_name: str, project_root: Path = None):
+    """
+    直接在代码中通过 spider name 运行爬虫
+    """
+    if project_root:
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+    args = [spider_name]
+    return main(args)
+
+
+if __name__ == '__main__':
+    # 允许直接运行: python -m crawlo.commands.run <spider_name>
+    import sys
+
+    sys.exit(main(sys.argv[1:]))

crawlo/commands/startproject.py

@@ -0,0 +1,101 @@
+# crawlo/commands/startproject.py
+import os
+import shutil
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            # 创建目标目录
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                # 渲染模板文件，并去掉 .tmpl 后缀
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
+                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                # 普通文件，直接复制
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        print("Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        print(f"Error: Directory '{project_dir}' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+        else:
+            print("Warning: crawlo.cfg.tmpl not found in templates.")
+
+        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
+        package_dir = project_dir / project_name
+        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
+        # 并将它们渲染为 .py 文件
+        _copytree_with_templates(template_dir, package_dir, context)
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+
+        print(f"""
+        ✔ 项目 '{project_name}' 创建成功！
+
+        进入项目目录:
+            cd {project_name}
+
+        创建一个爬虫:
+            crawlo genspider example example.com
+
+        运行爬虫:
+            crawlo run example
+        """)
+        return 0
+
+    except Exception as e:
+        print(f"Error creating project: {e}")
+        # 如果出错，尝试清理已创建的目录
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+        return 1

crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-