crawlo 1.0.5__tar.gz → 1.0.7__tar.gz

{crawlo-1.0.5/crawlo.egg-info → crawlo-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlo
-Version: 1.0.5
+Version: 1.0.7
 Summary: Crawlo 是一款基于异步IO的高性能Python爬虫框架，支持分布式抓取。
 Home-page: https://github.com/crawl-coder/Crawlo.git
 Author: crawl-coder

crawlo-1.0.7/crawlo/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.7"

crawlo-1.0.7/crawlo/cli.py

@@ -0,0 +1,41 @@
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+
+
+def main():
+    # 获取所有可用命令
+    commands = get_commands()
+
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]"
+    )
+    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
+    # 注意：这里不添加具体参数，由子命令解析
+
+    # 只解析命令
+    args, unknown = parser.parse_known_args()
+
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

crawlo-1.0.7/crawlo/commands/__init__.py

@@ -0,0 +1,10 @@
+# crawlo/commands/__init__.py
+# 定义可用的命令
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+}
+
+def get_commands():
+    return _commands

crawlo-1.0.7/crawlo/commands/genspider.py

@@ -0,0 +1,111 @@
+import os
+import sys
+from pathlib import Path
+import configparser
+import importlib
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        print("Usage: crawlo genspider <spider_name> <domain>")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 查找项目根目录
+    project_root = None
+    current = Path.cwd()
+    while True:
+        cfg_file = current / 'crawlo.cfg'
+        if cfg_file.exists():
+            project_root = current
+            break
+        parent = current.parent
+        if parent == current:
+            break
+        current = parent
+
+    if not project_root:
+        print("Error: Not a crawlo project. crawlo.cfg not found.")
+        return 1
+
+    # 将项目根目录加入 sys.path
+    if str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
+
+    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_file, encoding='utf-8')
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
+    except Exception as e:
+        print(f"Error reading crawlo.cfg: {e}")
+        return 1
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [cls for cls in items_module.__dict__.values()
+                        if isinstance(cls, type) and cls.__name__.isupper()]
+
+        # 如果找到了类，使用第一个作为默认
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            default_item_class = "ExampleItem"  # 回退到示例
+    except ImportError as e:
+        print(f"Error importing items module '{items_module_path}': {e}")
+        default_item_class = "ExampleItem"
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    if not spiders_dir.exists():
+        spiders_dir.mkdir(parents=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        print(f"Error: Spider '{spider_name}' already exists.")
+        return 1
+
+    # ✅ 修正模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+
+    if not tmpl_path.exists():
+        print(f"Error: Template file not found at {tmpl_path}")
+        return 1
+
+    # ✅ 生成正确的类名
+    class_name = f"{spider_name.capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name  # ✅ 添加处理好的类名
+    }
+
+    content = _render_template(tmpl_path, context)
+
+    with open(spider_file, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    print(f"Spider '{spider_name}' created in {spider_file}")
+    return 0

crawlo-1.0.7/crawlo/commands/run.py

@@ -0,0 +1,149 @@
+# crawlo/commands/run.py
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+from crawlo.utils.spider_loader import SpiderLoader
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    """
+    运行指定爬虫的主函数
+    用法: crawlo run <spider_name>
+    """
+    if len(args) < 1:
+        print("Usage: crawlo run <spider_name>")
+        print("Example: crawlo run baidu")
+        return 1
+
+    spider_name = args[0]
+
+    try:
+        # 1. 获取项目根目录
+        project_root = get_settings()
+
+        # 将项目根目录添加到 Python 路径
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+        # 2. 读取配置文件获取项目包名
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 3. 查找并加载指定名称的 Spider
+        spider_class = find_spider_by_name(project_package, spider_name)
+        if spider_class is None:
+            return 1
+
+        # 4. 创建 CrawlerProcess 并运行单个爬虫
+        settings = get_settings()
+        process = CrawlerProcess(settings)
+
+        print(f"🚀 Starting spider: {spider_class.name}")
+        print(f"📁 Project: {project_package}")
+        print(f"🕷️  Class: {spider_class.__name__}")
+        print("-" * 50)
+
+        # 运行单个爬虫
+        asyncio.run(process.crawl(spider_class))
+
+        print("-" * 50)
+        print("✅ Spider completed successfully!")
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error running spider: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+def find_spider_by_name(project_package: str, target_spider_name: str):
+    """使用 SpiderLoader 查找爬虫"""
+    loader = SpiderLoader(project_package)
+    spider_class = loader.load(target_spider_name)
+
+    if spider_class is None:
+        print(f"❌ Error: Spider with name '{target_spider_name}' not found")
+        print("💡 Available spiders:")
+        available_spiders = loader.list()
+        for spider_name in available_spiders:
+            print(f"   - {spider_name}")
+        return None
+
+    return spider_class
+
+
+def list_available_spiders(project_package: str):
+    """
+    列出所有可用的爬虫
+    """
+    spiders_dir = Path.cwd() / project_package / 'spiders'
+    if not spiders_dir.exists():
+        print("   No spiders directory found")
+        return
+
+    spider_count = 0
+    for py_file in spiders_dir.glob("*.py"):
+        if py_file.name.startswith('_'):
+            continue
+
+        module_name = py_file.stem
+        spider_module_path = f"{project_package}.spiders.{module_name}"
+
+        try:
+            module = importlib.import_module(spider_module_path)
+        except ImportError:
+            continue
+
+        # 查找模块中所有 Spider 子类
+        from crawlo.spider import Spider
+        for attr_name in dir(module):
+            attr_value = getattr(module, attr_name)
+            if (isinstance(attr_value, type) and
+                    issubclass(attr_value, Spider) and
+                    attr_value != Spider and
+                    hasattr(attr_value, 'name')):
+                print(f"   - {attr_value.name} (class: {attr_value.__name__}, module: {module_name})")
+                spider_count += 1
+
+    if spider_count == 0:
+        print("   No spiders found")
+
+
+def run_spider_by_name(spider_name: str, project_root: Path = None):
+    """
+    直接在代码中通过 spider name 运行爬虫
+    """
+    if project_root:
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+    args = [spider_name]
+    return main(args)
+
+
+if __name__ == '__main__':
+    # 允许直接运行: python -m crawlo.commands.run <spider_name>
+    import sys
+
+    sys.exit(main(sys.argv[1:]))

crawlo-1.0.7/crawlo/commands/startproject.py

@@ -0,0 +1,101 @@
+# crawlo/commands/startproject.py
+import os
+import shutil
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            # 创建目标目录
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                # 渲染模板文件，并去掉 .tmpl 后缀
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
+                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                # 普通文件，直接复制
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        print("Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        print(f"Error: Directory '{project_dir}' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+        else:
+            print("Warning: crawlo.cfg.tmpl not found in templates.")
+
+        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
+        package_dir = project_dir / project_name
+        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
+        # 并将它们渲染为 .py 文件
+        _copytree_with_templates(template_dir, package_dir, context)
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+
+        print(f"""
+        ✔ 项目 '{project_name}' 创建成功！
+
+        进入项目目录:
+            cd {project_name}
+
+        创建一个爬虫:
+            crawlo genspider example example.com
+
+        运行爬虫:
+            crawlo run example
+        """)
+        return 0
+
+    except Exception as e:
+        print(f"Error creating project: {e}")
+        # 如果出错，尝试清理已创建的目录
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+        return 1

crawlo-1.0.7/crawlo/crawler.py

@@ -0,0 +1,219 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+import asyncio
+import signal
+from typing import Type, Optional, Set, List
+
+from crawlo.spider import Spider
+from crawlo.core.engine import Engine
+from crawlo.utils.log import get_logger
+from crawlo.subscriber import Subscriber
+from crawlo.extension import ExtensionManager
+from crawlo.exceptions import SpiderTypeError
+from crawlo.stats_collector import StatsCollector
+from crawlo.event import spider_opened, spider_closed
+from crawlo.settings.setting_manager import SettingManager
+from crawlo.utils.project import merge_settings, get_settings
+
+
+logger = get_logger(__name__)
+
+
+class Crawler:
+    """单个爬虫运行实例，绑定 Spider 与引擎"""
+
+    def __init__(self, spider_cls: Type[Spider], settings: SettingManager):
+        self.spider_cls = spider_cls
+        self.spider: Optional[Spider] = None
+        self.engine: Optional[Engine] = None
+        self.stats: Optional[StatsCollector] = None
+        self.subscriber: Optional[Subscriber] = None
+        self.extension: Optional[ExtensionManager] = None
+        self.settings: SettingManager = settings.copy()
+        self._closed = False  # 新增状态
+        self._close_lock = asyncio.Lock()
+
+    async def crawl(self):
+        """启动爬虫核心流程"""
+        self.subscriber = self._create_subscriber()
+        self.spider = self._create_spider()
+        self.engine = self._create_engine()
+        self.stats = self._create_stats()
+        self.extension = self._create_extension()
+
+        await self.engine.start_spider(self.spider)
+
+    @staticmethod
+    def _create_subscriber() -> Subscriber:
+        return Subscriber()
+
+    def _create_spider(self) -> Spider:
+        spider = self.spider_cls.create_instance(self)
+
+        # --- 关键属性检查 ---
+        if not getattr(spider, 'name', None):
+            raise AttributeError(f"爬虫类 '{self.spider_cls.__name__}' 必须定义 'name' 属性。")
+
+        if not callable(getattr(spider, 'start_requests', None)):
+            raise AttributeError(f"爬虫 '{spider.name}' 必须实现可调用的 'start_requests' 方法。")
+
+        start_urls = getattr(spider, 'start_urls', [])
+        if isinstance(start_urls, str):
+            raise TypeError(f"爬虫 '{spider.name}' 的 'start_urls' 必须是列表或元组，不能是字符串。")
+
+        if not callable(getattr(spider, 'parse', None)):
+            logger.warning(
+                f"爬虫 '{spider.name}' 未定义 'parse' 方法。请确保所有 Request 都指定了回调函数，否则响应将被忽略。")
+
+        self._set_spider(spider)
+        return spider
+
+    def _create_engine(self) -> Engine:
+        engine = Engine(self)
+        engine.engine_start()
+        return engine
+
+    def _create_stats(self) -> StatsCollector:
+        return StatsCollector(self)
+
+    def _create_extension(self) -> ExtensionManager:
+        return ExtensionManager.create_instance(self)
+
+    def _set_spider(self, spider: Spider):
+        self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
+        self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
+        merge_settings(spider, self.settings)
+
+    async def close(self, reason='finished') -> None:
+        async with self._close_lock:
+            if self._closed:
+                return
+            self._closed = True
+            await self.subscriber.notify(spider_closed)
+            if self.stats and self.spider:
+                self.stats.close_spider(spider=self.spider, reason=reason)
+
+
+class CrawlerProcess:
+    """
+    爬虫进程管理器，支持多爬虫并发调度、信号量控制、实时日志与优雅关闭
+    """
+
+    def __init__(self, settings: Optional[SettingManager] = None, max_concurrency: Optional[int] = None):
+        self.settings: SettingManager = settings or self._get_default_settings()
+        self.crawlers: Set[Crawler] = set()
+        self._active_tasks: Set[asyncio.Task] = set()
+
+        # 使用专用配置，降级使用 CONCURRENCY
+        self.max_concurrency: int = (
+            max_concurrency
+            or self.settings.get('MAX_RUNNING_SPIDERS')
+            or self.settings.get('CONCURRENCY', 3)
+        )
+        self.semaphore = asyncio.Semaphore(self.max_concurrency)
+
+        # 注册信号量
+        signal.signal(signal.SIGINT, self._shutdown)
+        signal.signal(signal.SIGTERM, self._shutdown)
+        logger.info(f"CrawlerProcess 初始化完成，最大并行爬虫数: {self.max_concurrency}")
+
+    async def crawl(self, spiders):
+        """
+        启动一个或多个爬虫，流式调度，支持实时进度反馈
+        """
+        spider_classes = self._normalize_spiders(spiders)
+        total = len(spider_classes)
+
+        if total == 0:
+            raise ValueError("至少需要提供一个爬虫类")
+
+        # 按名称排序
+        spider_classes.sort(key=lambda cls: cls.__name__.lower())
+
+        logger.info(f"启动 {total} 个爬虫.")
+
+        # 流式启动所有爬虫任务
+        tasks = [
+            asyncio.create_task(self._run_spider_with_limit(spider_cls, index + 1, total))
+            for index, spider_cls in enumerate(spider_classes)
+        ]
+
+        # 等待所有任务完成（失败不中断）
+        results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # 统计异常
+        failed = [i for i, r in enumerate(results) if isinstance(r, Exception)]
+        if failed:
+            logger.error(f"共 {len(failed)} 个爬虫执行异常: {[spider_classes[i].__name__ for i in failed]}")
+
+    @staticmethod
+    def _normalize_spiders(spiders) -> List[Type[Spider]]:
+        """标准化输入为爬虫类列表"""
+        if isinstance(spiders, type) and issubclass(spiders, Spider):
+            return [spiders]
+        elif isinstance(spiders, (list, tuple)):
+            return list(spiders)
+        else:
+            raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
+
+    async def _run_spider_with_limit(self, spider_cls: Type[Spider], seq: int, total: int):
+        """
+        受信号量限制的爬虫运行函数，带进度日志
+        """
+        task = asyncio.current_task()
+        self._active_tasks.add(task)
+
+        try:
+            # 获取并发许可
+            await self.semaphore.acquire()
+
+            start_msg = f"[{seq}/{total}] 启动爬虫: {spider_cls.__name__}"
+            logger.info(start_msg)
+
+            # 创建并运行爬虫
+            crawler = self._create_crawler(spider_cls)
+            self.crawlers.add(crawler)
+            await crawler.crawl()
+
+            end_msg = f"[{seq}/{total}] 爬虫完成: {spider_cls.__name__}"
+            logger.info(end_msg)
+
+        except Exception as e:
+            logger.error(f"爬虫 {spider_cls.__name__} 执行失败: {e}", exc_info=True)
+            raise
+        finally:
+            if task in self._active_tasks:
+                self._active_tasks.remove(task)
+            self.semaphore.release()  # 必须释放
+
+    def _create_crawler(self, spider_cls: Type[Spider]) -> Crawler:
+        """创建爬虫实例"""
+        if isinstance(spider_cls, str):
+            raise SpiderTypeError(f"不支持字符串形式的爬虫: {spider_cls}")
+        return Crawler(spider_cls, self.settings)
+
+    def _shutdown(self, _signum, _frame):
+        """优雅关闭信号处理"""
+        logger.warning("收到关闭信号，正在停止所有爬虫...")
+        for crawler in list(self.crawlers):
+            if crawler.engine:
+                crawler.engine.running = False
+                crawler.engine.normal = False
+        asyncio.create_task(self._wait_for_shutdown())
+
+    async def _wait_for_shutdown(self):
+        """等待所有活跃任务完成"""
+        pending = [t for t in self._active_tasks if not t.done()]
+        if pending:
+            logger.info(f"等待 {len(pending)} 个活跃任务完成...")
+            await asyncio.gather(*pending, return_exceptions=True)
+        logger.info("所有爬虫已优雅关闭")
+
+    @classmethod
+    def _get_default_settings(cls) -> SettingManager:
+        """加载默认配置"""
+        try:
+            return get_settings()
+        except Exception as e:
+            logger.warning(f"无法加载默认配置: {e}")
+            return SettingManager()

{crawlo-1.0.5 → crawlo-1.0.7}/crawlo/exceptions.py

@@ -71,3 +71,8 @@ class ReceiverTypeError(Exception):
 class SpiderCreationError(Exception):
     """爬虫实例化失败异常"""
     pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass

crawlo-1.0.7/crawlo/items/__init__.py

@@ -0,0 +1,23 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+crawlo.items 包
+===============
+提供 Item 和 Field 类用于数据定义和验证。
+"""
+from .items import Item
+from .fields import Field
+from .base import ItemMeta
+
+from crawlo.exceptions import ItemInitError, ItemAttributeError
+
+__all__ = [
+    'Item',
+    'Field',
+    'ItemMeta',
+    'ItemInitError',
+    'ItemAttributeError'
+]
+
+
+