crawlo 1.3.3__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/framework.py

@@ -0,0 +1,292 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架统一入口
+================
+
+提供简洁、一致的API接口，隐藏内部复杂性
+"""
+
+import os
+import sys
+from typing import Type, Optional, List, Union
+
+from .crawler import ModernCrawler, CrawlerProcess
+from .initialization import initialize_framework
+from .logging import get_logger
+from .utils.env_config import get_version
+
+
+class CrawloFramework:
+    """
+    Crawlo框架门面类
+    
+    提供统一的框架入口点，简化使用复杂度
+    """
+
+    def __init__(self, settings=None, **kwargs):
+        """
+        初始化框架
+        
+        Args:
+            settings: 配置对象
+            **kwargs: 额外配置参数
+        """
+        # 合并配置
+        config = {}
+        if settings:
+            if hasattr(settings, '__dict__'):
+                config.update(settings.__dict__)
+            elif isinstance(settings, dict):
+                config.update(settings)
+        config.update(kwargs)
+        
+        # 如果没有提供配置，尝试自动加载项目配置
+        if not config:
+            config = self._load_project_config()
+
+        # 初始化框架
+        self._settings = initialize_framework(config)
+        self._logger = get_logger('crawlo.framework')
+
+        # 获取版本号
+        version = get_version()
+
+        # 创建进程管理器
+        self._process = CrawlerProcess(self._settings)
+
+        self._logger.info(f"Crawlo Framework Started {version}")
+        
+        # 获取运行模式并记录日志
+        run_mode = self._settings.get('RUN_MODE', 'unknown')
+        self._logger.info(f"Run mode: {run_mode}")
+        
+        # 记录项目名称
+        project_name = self._settings.get('PROJECT_NAME', 'unknown')
+        self._logger.info(f"Project: {project_name}")
+
+    def _load_project_config(self):
+        """
+        自动加载项目配置
+        """
+        try:
+            # 查找项目根目录
+            project_root = self._find_project_root()
+            if not project_root:
+                print("警告: 未找到项目根目录，使用默认配置")
+                return {}
+            
+            # 添加项目根目录到Python路径
+            if project_root not in sys.path:
+                sys.path.insert(0, project_root)
+            
+            # 读取crawlo.cfg配置文件
+            cfg_file = os.path.join(project_root, "crawlo.cfg")
+            if not os.path.exists(cfg_file):
+                print(f"警告: 未找到配置文件 {cfg_file}，使用默认配置")
+                return {}
+            
+            import configparser
+            config_parser = configparser.ConfigParser()
+            config_parser.read(cfg_file, encoding="utf-8")
+            
+            if not config_parser.has_section("settings") or not config_parser.has_option("settings", "default"):
+                print("警告: 配置文件缺少 [settings] 部分或 'default' 选项，使用默认配置")
+                return {}
+            
+            # 获取settings模块路径
+            settings_module_path = config_parser.get("settings", "default")
+            project_package = settings_module_path.split(".")[0]
+            
+            # 导入项目配置模块
+            import importlib
+            settings_module = importlib.import_module(settings_module_path)
+            
+            # 创建配置字典
+            project_config = {}
+            for key in dir(settings_module):
+                if key.isupper():
+                    project_config[key] = getattr(settings_module, key)
+            
+            # print(f"已加载项目配置: {settings_module_path}")
+            return project_config
+            
+        except Exception as e:
+            print(f"加载项目配置时出错: {e}")
+            return {}
+
+    def _find_project_root(self):
+        """
+        查找项目根目录（包含crawlo.cfg的目录）
+        """
+        current_path = os.getcwd()
+        
+        # 向上查找直到找到crawlo.cfg
+        checked_paths = set()
+        path = current_path
+        
+        while path not in checked_paths:
+            checked_paths.add(path)
+            
+            # 检查crawlo.cfg
+            cfg_file = os.path.join(path, "crawlo.cfg")
+            if os.path.exists(cfg_file):
+                return path
+            
+            # 向上一级目录
+            parent = os.path.dirname(path)
+            if parent == path:
+                break
+            path = parent
+        
+        return None
+
+    @property
+    def settings(self):
+        """获取配置"""
+        return self._settings
+
+    @property
+    def logger(self):
+        """获取框架日志器"""
+        return self._logger
+
+    async def run(self, spider_cls_or_name, settings=None):
+        """
+        运行单个爬虫
+        
+        Args:
+            spider_cls_or_name: Spider类或名称
+            settings: 额外配置
+            
+        Returns:
+            Crawler实例
+        """
+        # 记录启动的爬虫名称
+        if isinstance(spider_cls_or_name, str):
+            spider_name = spider_cls_or_name
+        else:
+            spider_name = getattr(spider_cls_or_name, 'name', spider_cls_or_name.__name__)
+        
+        self._logger.info(f"Starting spider: {spider_name}")
+        
+        return await self._process.crawl(spider_cls_or_name, settings)
+
+    async def run_multiple(self, spider_classes_or_names: List[Union[Type, str]],
+                           settings=None):
+        """
+        运行多个爬虫
+        
+        Args:
+            spider_classes_or_names: Spider类或名称列表
+            settings: 额外配置
+            
+        Returns:
+            结果列表
+        """
+        # 记录启动的爬虫名称
+        spider_names = []
+        for spider_cls_or_name in spider_classes_or_names:
+            if isinstance(spider_cls_or_name, str):
+                spider_names.append(spider_cls_or_name)
+            else:
+                spider_names.append(getattr(spider_cls_or_name, 'name', spider_cls_or_name.__name__))
+        
+        self._logger.info(f"Starting spiders: {', '.join(spider_names)}")
+        
+        return await self._process.crawl_multiple(spider_classes_or_names, settings)
+
+    def create_crawler(self, spider_cls: Type, settings=None) -> ModernCrawler:
+        """
+        创建Crawler实例
+        
+        Args:
+            spider_cls: Spider类
+            settings: 额外配置
+            
+        Returns:
+            ModernCrawler实例
+        """
+        merged_settings = self._merge_settings(settings)
+        return ModernCrawler(spider_cls, merged_settings)
+
+    def _merge_settings(self, additional_settings):
+        """合并配置"""
+        if not additional_settings:
+            return self._settings
+
+        from .settings.setting_manager import SettingManager
+        merged = SettingManager()
+
+        # 复制基础配置
+        if self._settings:
+            merged.update_attributes(self._settings.__dict__)
+
+        # 应用额外配置
+        if isinstance(additional_settings, dict):
+            merged.update_attributes(additional_settings)
+        elif hasattr(additional_settings, '__dict__'):
+            merged.update_attributes(additional_settings.__dict__)
+
+        return merged
+
+    def get_metrics(self) -> dict:
+        """获取框架指标"""
+        return self._process.get_metrics()
+
+
+# 全局框架实例
+_global_framework: Optional[CrawloFramework] = None
+
+
+def get_framework(settings=None, **kwargs) -> CrawloFramework:
+    """
+    获取全局框架实例（单例模式）
+    
+    Args:
+        settings: 配置对象
+        **kwargs: 额外配置参数
+        
+    Returns:
+        CrawloFramework实例
+    """
+    global _global_framework
+
+    if _global_framework is None:
+        _global_framework = CrawloFramework(settings, **kwargs)
+
+    return _global_framework
+
+
+def reset_framework():
+    """重置全局框架实例（主要用于测试）"""
+    global _global_framework
+    _global_framework = None
+
+
+# 便捷函数
+async def run_spider(spider_cls_or_name, settings=None, **kwargs):
+    """运行单个爬虫的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return await framework.run(spider_cls_or_name)
+
+
+async def run_spiders(spider_classes_or_names: List[Union[Type, str]],
+                      settings=None, **kwargs):
+    """运行多个爬虫的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return await framework.run_multiple(spider_classes_or_names)
+
+
+def create_crawler(spider_cls: Type, settings=None, **kwargs) -> ModernCrawler:
+    """创建Crawler的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return framework.create_crawler(spider_cls)
+
+
+# 配置相关便捷函数
+def configure_framework(settings=None, **kwargs):
+    """配置框架的便捷函数"""
+    if settings or kwargs:
+        reset_framework()  # 重置以应用新配置
+    return get_framework(settings, **kwargs)

crawlo/initialization/__init__.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架统一初始化系统
+=======================
+
+设计原则：
+1. 单一职责 - 每个初始化器负责特定领域
+2. 依赖清晰 - 明确的初始化顺序和依赖关系
+3. 可测试性 - 支持依赖注入和模拟
+4. 健壮性 - 优雅的错误处理和降级策略
+"""
+
+from .registry import InitializerRegistry
+from .context import InitializationContext
+from .core import CoreInitializer
+from .phases import InitializationPhase
+
+# 公共接口
+def initialize_framework(settings=None, **kwargs):
+    """初始化框架的主要入口"""
+    return CoreInitializer().initialize(settings, **kwargs)
+
+def is_framework_ready():
+    """检查框架是否已准备就绪"""
+    return CoreInitializer().is_ready
+
+def get_framework_context():
+    """获取框架初始化上下文"""
+    return CoreInitializer().context
+
+__all__ = [
+    'InitializerRegistry',
+    'InitializationContext', 
+    'CoreInitializer',
+    'InitializationPhase',
+    'initialize_framework',
+    'is_framework_ready',
+    'get_framework_context'
+]