crawlo 1.0.5__py3-none-any.whl → 1.0.7__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.5"
+__version__ = "1.0.7"

crawlo/cli.py

@@ -0,0 +1,41 @@
+# crawlo/cli.py
+# !/usr/bin/python
+# -*- coding: UTF-8 -*-
+import sys
+import argparse
+from crawlo.commands import get_commands
+
+
+def main():
+    # 获取所有可用命令
+    commands = get_commands()
+
+    parser = argparse.ArgumentParser(
+        description="Crawlo: A lightweight web crawler framework.",
+        usage="crawlo <command> [options]"
+    )
+    parser.add_argument('command', help='Available commands: ' + ', '.join(commands.keys()))
+    # 注意：这里不添加具体参数，由子命令解析
+
+    # 只解析命令
+    args, unknown = parser.parse_known_args()
+
+    if args.command not in commands:
+        print(f"Unknown command: {args.command}")
+        print(f"Available commands: {', '.join(commands.keys())}")
+        sys.exit(1)
+
+    # 动态导入并执行命令
+    try:
+        module = __import__(commands[args.command], fromlist=['main'])
+        sys.exit(module.main(unknown))
+    except ImportError as e:
+        print(f"Failed to load command '{args.command}': {e}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"Command '{args.command}' failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

crawlo/commands/__init__.py

@@ -0,0 +1,10 @@
+# crawlo/commands/__init__.py
+# 定义可用的命令
+_commands = {
+    'startproject': 'crawlo.commands.startproject',
+    'genspider': 'crawlo.commands.genspider',
+    'run': 'crawlo.commands.run',
+}
+
+def get_commands():
+    return _commands

crawlo/commands/genspider.py

@@ -0,0 +1,111 @@
+import os
+import sys
+from pathlib import Path
+import configparser
+import importlib
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def main(args):
+    if len(args) < 2:
+        print("Usage: crawlo genspider <spider_name> <domain>")
+        return 1
+
+    spider_name = args[0]
+    domain = args[1]
+
+    # 查找项目根目录
+    project_root = None
+    current = Path.cwd()
+    while True:
+        cfg_file = current / 'crawlo.cfg'
+        if cfg_file.exists():
+            project_root = current
+            break
+        parent = current.parent
+        if parent == current:
+            break
+        current = parent
+
+    if not project_root:
+        print("Error: Not a crawlo project. crawlo.cfg not found.")
+        return 1
+
+    # 将项目根目录加入 sys.path
+    if str(project_root) not in sys.path:
+        sys.path.insert(0, str(project_root))
+
+    # 从 crawlo.cfg 读取 settings 模块，获取项目包名
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_file, encoding='utf-8')
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]  # e.g., myproject.settings -> myproject
+    except Exception as e:
+        print(f"Error reading crawlo.cfg: {e}")
+        return 1
+
+    # 确定 items 模块的路径
+    items_module_path = f"{project_package}.items"
+
+    # 尝试导入 items 模块
+    try:
+        items_module = importlib.import_module(items_module_path)
+        # 获取模块中所有大写开头的类
+        item_classes = [cls for cls in items_module.__dict__.values()
+                        if isinstance(cls, type) and cls.__name__.isupper()]
+
+        # 如果找到了类，使用第一个作为默认
+        if item_classes:
+            default_item_class = item_classes[0].__name__
+        else:
+            default_item_class = "ExampleItem"  # 回退到示例
+    except ImportError as e:
+        print(f"Error importing items module '{items_module_path}': {e}")
+        default_item_class = "ExampleItem"
+
+    # 创建爬虫文件
+    spiders_dir = project_root / project_package / 'spiders'
+    if not spiders_dir.exists():
+        spiders_dir.mkdir(parents=True)
+
+    spider_file = spiders_dir / f'{spider_name}.py'
+    if spider_file.exists():
+        print(f"Error: Spider '{spider_name}' already exists.")
+        return 1
+
+    # ✅ 修正模板路径
+    tmpl_path = TEMPLATES_DIR / 'spider' / 'spider.py.tmpl'
+
+    if not tmpl_path.exists():
+        print(f"Error: Template file not found at {tmpl_path}")
+        return 1
+
+    # ✅ 生成正确的类名
+    class_name = f"{spider_name.capitalize()}Spider"
+
+    context = {
+        'spider_name': spider_name,
+        'domain': domain,
+        'project_name': project_package,
+        'item_class': default_item_class,
+        'class_name': class_name  # ✅ 添加处理好的类名
+    }
+
+    content = _render_template(tmpl_path, context)
+
+    with open(spider_file, 'w', encoding='utf-8') as f:
+        f.write(content)
+
+    print(f"Spider '{spider_name}' created in {spider_file}")
+    return 0

crawlo/commands/run.py

@@ -0,0 +1,149 @@
+# crawlo/commands/run.py
+import asyncio
+import importlib
+import sys
+from pathlib import Path
+import configparser
+
+from crawlo.crawler import CrawlerProcess
+from crawlo.utils.project import get_settings
+from crawlo.utils.log import get_logger
+from crawlo.utils.spider_loader import SpiderLoader
+
+logger = get_logger(__name__)
+
+
+def main(args):
+    """
+    运行指定爬虫的主函数
+    用法: crawlo run <spider_name>
+    """
+    if len(args) < 1:
+        print("Usage: crawlo run <spider_name>")
+        print("Example: crawlo run baidu")
+        return 1
+
+    spider_name = args[0]
+
+    try:
+        # 1. 获取项目根目录
+        project_root = get_settings()
+
+        # 将项目根目录添加到 Python 路径
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+        # 2. 读取配置文件获取项目包名
+        cfg_file = project_root / 'crawlo.cfg'
+        if not cfg_file.exists():
+            print(f"❌ Error: crawlo.cfg not found in {project_root}")
+            return 1
+
+        config = configparser.ConfigParser()
+        config.read(cfg_file, encoding='utf-8')
+
+        if not config.has_section('settings') or not config.has_option('settings', 'default'):
+            print("❌ Error: Missing [settings] section or 'default' option in crawlo.cfg")
+            return 1
+
+        settings_module = config.get('settings', 'default')
+        project_package = settings_module.split('.')[0]
+
+        # 3. 查找并加载指定名称的 Spider
+        spider_class = find_spider_by_name(project_package, spider_name)
+        if spider_class is None:
+            return 1
+
+        # 4. 创建 CrawlerProcess 并运行单个爬虫
+        settings = get_settings()
+        process = CrawlerProcess(settings)
+
+        print(f"🚀 Starting spider: {spider_class.name}")
+        print(f"📁 Project: {project_package}")
+        print(f"🕷️  Class: {spider_class.__name__}")
+        print("-" * 50)
+
+        # 运行单个爬虫
+        asyncio.run(process.crawl(spider_class))
+
+        print("-" * 50)
+        print("✅ Spider completed successfully!")
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error running spider: {e}")
+        import traceback
+        traceback.print_exc()
+        return 1
+
+
+def find_spider_by_name(project_package: str, target_spider_name: str):
+    """使用 SpiderLoader 查找爬虫"""
+    loader = SpiderLoader(project_package)
+    spider_class = loader.load(target_spider_name)
+
+    if spider_class is None:
+        print(f"❌ Error: Spider with name '{target_spider_name}' not found")
+        print("💡 Available spiders:")
+        available_spiders = loader.list()
+        for spider_name in available_spiders:
+            print(f"   - {spider_name}")
+        return None
+
+    return spider_class
+
+
+def list_available_spiders(project_package: str):
+    """
+    列出所有可用的爬虫
+    """
+    spiders_dir = Path.cwd() / project_package / 'spiders'
+    if not spiders_dir.exists():
+        print("   No spiders directory found")
+        return
+
+    spider_count = 0
+    for py_file in spiders_dir.glob("*.py"):
+        if py_file.name.startswith('_'):
+            continue
+
+        module_name = py_file.stem
+        spider_module_path = f"{project_package}.spiders.{module_name}"
+
+        try:
+            module = importlib.import_module(spider_module_path)
+        except ImportError:
+            continue
+
+        # 查找模块中所有 Spider 子类
+        from crawlo.spider import Spider
+        for attr_name in dir(module):
+            attr_value = getattr(module, attr_name)
+            if (isinstance(attr_value, type) and
+                    issubclass(attr_value, Spider) and
+                    attr_value != Spider and
+                    hasattr(attr_value, 'name')):
+                print(f"   - {attr_value.name} (class: {attr_value.__name__}, module: {module_name})")
+                spider_count += 1
+
+    if spider_count == 0:
+        print("   No spiders found")
+
+
+def run_spider_by_name(spider_name: str, project_root: Path = None):
+    """
+    直接在代码中通过 spider name 运行爬虫
+    """
+    if project_root:
+        if str(project_root) not in sys.path:
+            sys.path.insert(0, str(project_root))
+
+    args = [spider_name]
+    return main(args)
+
+
+if __name__ == '__main__':
+    # 允许直接运行: python -m crawlo.commands.run <spider_name>
+    import sys
+
+    sys.exit(main(sys.argv[1:]))

crawlo/commands/startproject.py

@@ -0,0 +1,101 @@
+# crawlo/commands/startproject.py
+import os
+import shutil
+from pathlib import Path
+
+TEMPLATES_DIR = Path(__file__).parent.parent / 'templates'
+
+
+def _render_template(tmpl_path, context):
+    """读取模板文件，替换 {{key}} 为 context 中的值"""
+    with open(tmpl_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    for key, value in context.items():
+        content = content.replace(f'{{{{{key}}}}}', str(value))
+    return content
+
+
+def _copytree_with_templates(src, dst, context):
+    """
+    递归复制目录，将 .tmpl 文件渲染后复制（去除 .tmpl 后缀），其他文件直接复制。
+    """
+    src_path = Path(src)
+    dst_path = Path(dst)
+    dst_path.mkdir(parents=True, exist_ok=True)
+
+    for item in src_path.rglob('*'):
+        rel_path = item.relative_to(src_path)
+        dst_item = dst_path / rel_path
+
+        if item.is_dir():
+            # 创建目标目录
+            dst_item.mkdir(parents=True, exist_ok=True)
+        else:
+            if item.suffix == '.tmpl':
+                # 渲染模板文件，并去掉 .tmpl 后缀
+                rendered_content = _render_template(item, context)
+                final_dst = dst_item.with_suffix('') # 去掉 .tmpl
+                final_dst.parent.mkdir(parents=True, exist_ok=True) # 确保父目录存在
+                with open(final_dst, 'w', encoding='utf-8') as f:
+                    f.write(rendered_content)
+            else:
+                # 普通文件，直接复制
+                shutil.copy2(item, dst_item)
+
+
+def main(args):
+    if len(args) != 1:
+        print("Usage: crawlo startproject <project_name>")
+        return 1
+
+    project_name = args[0]
+    project_dir = Path(project_name)
+
+    if project_dir.exists():
+        print(f"Error: Directory '{project_dir}' already exists.")
+        return 1
+
+    context = {'project_name': project_name}
+    template_dir = TEMPLATES_DIR / 'project'
+
+    try:
+        # 1. 创建项目根目录
+        project_dir.mkdir()
+
+        # 2. 处理 crawlo.cfg.tmpl：单独渲染并写入项目根目录
+        cfg_template = TEMPLATES_DIR / 'crawlo.cfg.tmpl'  # ✅ 使用 templates/ 目录下的模板
+        if cfg_template.exists():
+            cfg_content = _render_template(cfg_template, context)
+            (project_dir / 'crawlo.cfg').write_text(cfg_content, encoding='utf-8')
+        else:
+            print("Warning: crawlo.cfg.tmpl not found in templates.")
+
+        # 3. 复制所有其他模板文件到项目包内 (project_dir / project_name)
+        package_dir = project_dir / project_name
+        # 这会复制 __init__.py.tmpl, items.py.tmpl, settings.py.tmpl, spiders/ 等
+        # 并将它们渲染为 .py 文件
+        _copytree_with_templates(template_dir, package_dir, context)
+
+        # 4. 创建 logs 目录
+        (project_dir / 'logs').mkdir(exist_ok=True)
+
+        print(f"""
+        ✔ 项目 '{project_name}' 创建成功！
+
+        进入项目目录:
+            cd {project_name}
+
+        创建一个爬虫:
+            crawlo genspider example example.com
+
+        运行爬虫:
+            crawlo run example
+        """)
+        return 0
+
+    except Exception as e:
+        print(f"Error creating project: {e}")
+        # 如果出错，尝试清理已创建的目录
+        if project_dir.exists():
+            shutil.rmtree(project_dir, ignore_errors=True)
+        return 1

crawlo/crawler.py

@@ -216,209 +216,4 @@ class CrawlerProcess:
             return get_settings()
         except Exception as e:
             logger.warning(f"无法加载默认配置: {e}")
-            return SettingManager()
-
-
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*
-# import signal
-# import asyncio
-# from typing import Final, Set, Optional
-#
-# from crawlo.spider import Spider
-# from crawlo.core.engine import Engine
-# from crawlo.utils.log import get_logger
-# from crawlo.subscriber import Subscriber
-# from crawlo.extension import ExtensionManager
-# from crawlo.exceptions import SpiderTypeError
-# from crawlo.stats_collector import StatsCollector
-# from crawlo.event import spider_opened, spider_closed
-# from crawlo.settings.setting_manager import SettingManager
-# from crawlo.utils.project import merge_settings, get_settings
-#
-# logger = get_logger(__name__)
-#
-#
-# class Crawler:
-#
-#     def __init__(self, spider_cls, settings):
-#         self.spider_cls = spider_cls
-#         self.spider: Optional[Spider] = None
-#         self.engine: Optional[Engine] = None
-#         self.stats: Optional[StatsCollector] = None
-#         self.subscriber: Optional[Subscriber] = None
-#         self.extension: Optional[ExtensionManager] = None
-#         self.settings: SettingManager = settings.copy()
-#
-#     async def crawl(self):
-#         self.subscriber = self._create_subscriber()
-#         self.spider = self._create_spider()
-#         self.engine = self._create_engine()
-#         self.stats = self._create_stats()
-#         self.extension = self._create_extension()
-#
-#         await self.engine.start_spider(self.spider)
-#
-#     @staticmethod
-#     def _create_subscriber():
-#         return Subscriber()
-#
-#     def _create_spider(self) -> Spider:
-#         spider = self.spider_cls.create_instance(self)
-#
-#         # --- 关键属性检查 ---
-#         # 1. 检查 name
-#         if not getattr(spider, 'name', None):
-#             raise AttributeError(f"Spider class '{self.spider_cls.__name__}' must have a 'name' attribute.")
-#
-#         # 2. 检查 start_requests 是否可调用
-#         if not callable(getattr(spider, 'start_requests', None)):
-#             raise AttributeError(f"Spider '{spider.name}' must have a callable 'start_requests' method.")
-#
-#         # 3. 检查 start_urls 类型
-#         start_urls = getattr(spider, 'start_urls', [])
-#         if isinstance(start_urls, str):
-#             raise TypeError(f"'{spider.name}.start_urls' must be a list or tuple, not a string.")
-#
-#         # --- 日志提示 ---
-#         # 提醒用户定义 parse 方法
-#         if not callable(getattr(spider, 'parse', None)):
-#             logger.warning(f"Spider '{spider.name}' lacks a 'parse' method. Ensure all Requests have callbacks.")
-#
-#         self._set_spider(spider)
-#         return spider
-#
-#     def _create_engine(self) -> Engine:
-#         engine = Engine(self)
-#         engine.engine_start()
-#         return engine
-#
-#     def _create_stats(self) -> StatsCollector:
-#         stats = StatsCollector(self)
-#         return stats
-#
-#     def _create_extension(self) -> ExtensionManager:
-#         extension = ExtensionManager.create_instance(self)
-#         return extension
-#
-#     def _set_spider(self, spider):
-#         self.subscriber.subscribe(spider.spider_opened, event=spider_opened)
-#         self.subscriber.subscribe(spider.spider_closed, event=spider_closed)
-#         merge_settings(spider, self.settings)
-#
-#     async def close(self, reason='finished') -> None:
-#         await asyncio.create_task(self.subscriber.notify(spider_closed))
-#         self.stats.close_spider(spider=self.spider, reason=reason)
-#
-#
-# class CrawlerProcess:
-#     """爬虫处理类，支持跨平台动态并发控制和精细化日志"""
-#
-#     def __init__(self, settings=None, max_concurrency: Optional[int] = None, batch_size: int = 10):
-#         self.crawlers: Final[Set] = set()
-#         self._active_spiders: Final[Set] = set()
-#         self.settings = settings or self._get_default_settings()
-#         self.batch_size = batch_size
-#
-#         # 优先使用专用配置，降级使用 CONCURRENCY，最后用默认值
-#         self.max_concurrency = (
-#                 max_concurrency or
-#                 self.settings.get('MAX_RUNNING_SPIDERS') or
-#                 self.settings.get('CONCURRENCY', 5)
-#         )
-#         self.semaphore = asyncio.Semaphore(self.max_concurrency)
-#
-#         signal.signal(signal.SIGINT, self._shutdown)
-#         logger.debug(f"初始化爬虫处理进程，最大并发数: {self.max_concurrency}")
-#
-#     async def crawl(self, spiders):
-#         """支持单个或多个爬虫的批量处理，优化日志输出"""
-#         if not spiders:
-#             raise ValueError("至少需要提供一个爬虫类")
-#
-#         # 统一转换为列表
-#         if isinstance(spiders, type) and issubclass(spiders, Spider):
-#             spiders = [spiders]
-#         elif isinstance(spiders, (list, tuple)):
-#             spiders = list(spiders)
-#         else:
-#             raise TypeError("spiders 必须是爬虫类或爬虫类列表/元组")
-#
-#         # 按爬虫类名首字母排序（升序）
-#         spiders.sort(key=lambda x: x.__name__.lower())
-#
-#         if len(spiders) == 1:
-#             logger.info(f"启动爬虫: {spiders[0].__name__}")
-#         else:
-#             logger.info(f"启动{len(spiders)}个爬虫，按名称排序后分批处理中")
-#
-#         batches = [spiders[i:i + self.batch_size] for i in range(0, len(spiders), self.batch_size)]
-#
-#         for batch_idx, batch in enumerate(batches):
-#             batch_tasks = set()
-#
-#             for spider_cls in batch:
-#                 crawler = self._create_crawler(spider_cls)
-#                 self.crawlers.add(crawler)
-#
-#                 await self.semaphore.acquire()
-#                 task = asyncio.create_task(self._run_crawler_with_semaphore(crawler))
-#                 batch_tasks.add(task)
-#                 self._active_spiders.add(task)
-#
-#             if len(spiders) > 1:  # 仅对多爬虫显示批次信息
-#                 logger.info(f"启动第 {batch_idx + 1}/{len(batches)} 批爬虫，共 {len(batch)} 个")
-#
-#             await asyncio.gather(*batch_tasks)
-#
-#             if len(spiders) > 1:  # 仅对多爬虫显示批次完成信息
-#                 logger.info(f"第 {batch_idx + 1} 批爬虫处理完成")
-#
-#     async def _run_crawler_with_semaphore(self, crawler):
-#         """使用信号量控制的爬虫运行函数"""
-#         try:
-#             await crawler.crawl()
-#         finally:
-#             self.semaphore.release()  # 确保资源释放
-#
-#     async def start(self):
-#         """启动所有爬虫任务"""
-#         if self._active_spiders:
-#             logger.info(f"启动 {len(self._active_spiders)} 个爬虫任务，计算得知当前设备最大并发限制: {self.max_concurrency}")
-#             await asyncio.gather(*self._active_spiders)
-#
-#     def _create_crawler(self, spider_cls) -> Crawler:
-#         """创建爬虫实例"""
-#         if isinstance(spider_cls, str):
-#             raise SpiderTypeError(f"{type(self)}.crawl args: String is not supported.")
-#         crawler: Crawler = Crawler(spider_cls, self.settings)
-#         return crawler
-#
-#     def _shutdown(self, _signum, _frame):
-#         """优雅关闭所有爬虫"""
-#         logger.warning(f"收到关闭信号，正在优雅关闭 {len(self.crawlers)} 个爬虫...")
-#         for crawler in self.crawlers:
-#             if crawler.engine:
-#                 crawler.engine.running = False
-#                 crawler.engine.normal = False
-#                 crawler.stats.close_spider(crawler.spider, 'shutdown signal')
-#
-#         # 等待所有任务完成
-#         asyncio.create_task(self._wait_for_tasks())
-#
-#     async def _wait_for_tasks(self):
-#         """等待所有活跃任务完成"""
-#         pending = [task for task in self._active_spiders if not task.done()]
-#         if pending:
-#             logger.info(f"等待 {len(pending)} 个活跃任务完成...")
-#             await asyncio.gather(*pending)
-#         logger.info("所有爬虫已优雅关闭")
-#
-#     @classmethod
-#     def _get_default_settings(cls):
-#         """框架自动获取默认配置"""
-#         try:
-#             return get_settings()
-#         except ImportError:
-#             return {}
+            return SettingManager()

crawlo/exceptions.py

@@ -71,3 +71,8 @@ class ReceiverTypeError(Exception):
 class SpiderCreationError(Exception):
     """爬虫实例化失败异常"""
     pass
+
+
+class ItemValidationError(Exception):
+    """Item 字段验证错误"""
+    pass