crawlo 1.3.2__py3-none-any.whl → 1.3.4__py3-none-any.whl

crawlo/downloader/aiohttp_downloader.py

@@ -39,8 +39,8 @@ class AioHttpDownloader(DownloaderBase):
         # 读取配置
         timeout_secs = self.crawler.settings.get_int("DOWNLOAD_TIMEOUT", 30)
         verify_ssl = self.crawler.settings.get_bool("VERIFY_SSL", True)
-        pool_limit = self.crawler.settings.get_int("CONNECTION_POOL_LIMIT", 100)
-        pool_per_host = self.crawler.settings.get_int("CONNECTION_POOL_LIMIT_PER_HOST", 20)
+        pool_limit = self.crawler.settings.get_int("CONNECTION_POOL_LIMIT", 300)  # 从200增加到300
+        pool_per_host = self.crawler.settings.get_int("CONNECTION_POOL_LIMIT_PER_HOST", 100)  # 从50增加到100
         self.max_download_size = self.crawler.settings.get_int("DOWNLOAD_MAXSIZE", 10 * 1024 * 1024)  # 10MB
 
         # 创建连接器
@@ -51,6 +51,8 @@ class AioHttpDownloader(DownloaderBase):
             ttl_dns_cache=300,
             keepalive_timeout=15,
             force_close=False,
+            use_dns_cache=True,  # 启用DNS缓存
+            family=0,  # 允许IPv4和IPv6
         )
 
         # 超时控制

crawlo/extension/__init__.py

@@ -4,7 +4,7 @@ from typing import List, Any
 from pprint import pformat
 
 from crawlo.utils.log import get_logger
-from crawlo.project import load_class
+from crawlo.utils.class_loader import load_class
 from crawlo.exceptions import ExtensionInitError
 
 

crawlo/extension/logging_extension.py

@@ -1,9 +1,16 @@
 from typing import Any
 from crawlo.exceptions import NotConfigured
-from crawlo.utils.log import LoggerManager
+from crawlo.utils.log import get_logger
 
-# 使用全局logger，避免每个模块都创建自己的延迟初始化函数
-logger = LoggerManager.get_logger(__name__)
+# 延迟获取logger，确保在日志系统配置之后获取
+_logger = None
+
+def logger():
+    """延迟获取logger实例，确保在日志系统配置之后获取"""
+    global _logger
+    if _logger is None:
+        _logger = get_logger(__name__)
+    return _logger
 
 
 class CustomLoggerExtension:
@@ -14,8 +21,17 @@ class CustomLoggerExtension:
 
     def __init__(self, settings: Any):
         self.settings = settings
-        # 初始化全局日志配置
-        LoggerManager.configure(settings)
+        # 使用新的日志系统，但要简化配置传递
+        try:
+            from crawlo.logging import configure_logging
+            # 直接传递settings对象，让日志系统内部处理
+            configure_logging(settings)
+        except Exception as e:
+            # 如果日志系统配置失败，不应该阻止扩展加载
+            # 使用基本日志输出错误信息
+            import logging
+            logging.getLogger(__name__).warning(f"Failed to configure logging system: {e}")
+            # 不抛出异常，让扩展继续加载
 
     @classmethod
     def create_instance(cls, crawler: Any, *args: Any, **kwargs: Any) -> 'CustomLoggerExtension':
@@ -34,9 +50,9 @@ class CustomLoggerExtension:
         return cls(crawler.settings)
 
     def spider_opened(self, spider: Any) -> None:
-        logger = LoggerManager.get_logger(__name__)
+        logger_instance = logger()
         try:
-            logger.info(
+            logger_instance.info(
                 f"CustomLoggerExtension: Logging initialized. "
                 f"LOG_FILE={self.settings.get('LOG_FILE')}, "
                 f"LOG_LEVEL={self.settings.get('LOG_LEVEL')}"

crawlo/factories/__init__.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo组件工厂系统
+==================
+
+提供统一的组件创建和依赖注入机制
+"""
+
+from .registry import ComponentRegistry, get_component_registry
+from .base import ComponentFactory, ComponentSpec
+from .crawler import CrawlerComponentFactory
+
+# 公共接口
+register_component = get_component_registry().register
+get_component = get_component_registry().get
+create_component = get_component_registry().create
+
+__all__ = [
+    'ComponentRegistry',
+    'ComponentFactory', 
+    'ComponentSpec',
+    'CrawlerComponentFactory',
+    'get_component_registry',
+    'register_component',
+    'get_component',
+    'create_component'
+]

crawlo/factories/base.py

@@ -0,0 +1,69 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件工厂基类和规范
+"""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Type, Any, Dict, Optional, Callable
+
+
+@dataclass
+class ComponentSpec:
+    """组件规范 - 定义如何创建组件"""
+    
+    name: str
+    component_type: Type
+    factory_func: Callable[..., Any]
+    dependencies: list = None
+    singleton: bool = False
+    config_key: str = None
+    
+    def __post_init__(self):
+        if self.dependencies is None:
+            self.dependencies = []
+
+
+class ComponentFactory(ABC):
+    """组件工厂基类"""
+    
+    @abstractmethod
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        pass
+    
+    @abstractmethod
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型的组件"""
+        pass
+
+
+class DefaultComponentFactory(ComponentFactory):
+    """默认组件工厂实现"""
+    
+    def __init__(self):
+        self._instances: Dict[str, Any] = {}
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建组件实例"""
+        # 单例模式检查
+        if spec.singleton and spec.name in self._instances:
+            return self._instances[spec.name]
+        
+        # 调用工厂函数创建实例
+        instance = spec.factory_func(**kwargs)
+        
+        # 保存单例实例
+        if spec.singleton:
+            self._instances[spec.name] = instance
+        
+        return instance
+    
+    def supports(self, component_type: Type) -> bool:
+        """支持所有类型"""
+        return True
+    
+    def clear_singletons(self):
+        """清除单例实例（测试用）"""
+        self._instances.clear()

crawlo/factories/crawler.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawler组件工厂 - 专门用于创建Crawler相关组件
+"""
+
+from typing import Any, Type
+
+from .base import ComponentFactory, ComponentSpec
+from .registry import get_component_registry
+
+
+class CrawlerComponentFactory(ComponentFactory):
+    """Crawler组件工厂"""
+    
+    def create(self, spec: ComponentSpec, **kwargs) -> Any:
+        """创建Crawler相关组件"""
+        # 检查是否需要crawler依赖
+        if 'crawler' in spec.dependencies and 'crawler' not in kwargs:
+            raise ValueError(f"Crawler instance required for component {spec.name}")
+        
+        return spec.factory_func(**kwargs)
+    
+    def supports(self, component_type: Type) -> bool:
+        """检查是否支持指定类型"""
+        # 这里可以根据需要定义支持的组件类型
+        supported_types = [
+            'Engine', 'Scheduler', 'StatsCollector', 
+            'Subscriber', 'ExtensionManager'
+        ]
+        return component_type.__name__ in supported_types
+
+
+def register_crawler_components():
+    """注册Crawler相关组件"""
+    registry = get_component_registry()
+    
+    # 注册工厂
+    registry.register_factory(CrawlerComponentFactory())
+    
+    # 注册组件规范
+    
+    # Engine组件
+    def create_engine(crawler, **kwargs):
+        from crawlo.core.engine import Engine
+        return Engine(crawler)
+    
+    registry.register(ComponentSpec(
+        name='engine',
+        component_type=type('Engine', (), {}),
+        factory_func=create_engine,
+        dependencies=['crawler']
+    ))
+    
+    # Scheduler组件
+    def create_scheduler(crawler, **kwargs):
+        from crawlo.core.scheduler import Scheduler
+        return Scheduler.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='scheduler',
+        component_type=type('Scheduler', (), {}),
+        factory_func=create_scheduler,
+        dependencies=['crawler']
+    ))
+    
+    # StatsCollector组件
+    def create_stats(crawler, **kwargs):
+        from crawlo.stats_collector import StatsCollector
+        return StatsCollector(crawler)
+    
+    registry.register(ComponentSpec(
+        name='stats',
+        component_type=type('StatsCollector', (), {}),
+        factory_func=create_stats,
+        dependencies=['crawler']
+    ))
+    
+    # Subscriber组件
+    def create_subscriber(**kwargs):
+        from crawlo.subscriber import Subscriber
+        return Subscriber()
+    
+    registry.register(ComponentSpec(
+        name='subscriber',
+        component_type=type('Subscriber', (), {}),
+        factory_func=create_subscriber
+    ))
+    
+    # ExtensionManager组件
+    def create_extension_manager(crawler, **kwargs):
+        from crawlo.extension import ExtensionManager
+        return ExtensionManager.create_instance(crawler)
+    
+    registry.register(ComponentSpec(
+        name='extension_manager',
+        component_type=type('ExtensionManager', (), {}),
+        factory_func=create_extension_manager,
+        dependencies=['crawler']
+    ))
+
+
+# 自动注册
+register_crawler_components()

crawlo/factories/registry.py

@@ -0,0 +1,85 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+组件注册表 - 管理所有组件的注册和创建
+"""
+
+import threading
+from typing import Dict, List, Type, Any, Optional
+
+from .base import ComponentFactory, ComponentSpec, DefaultComponentFactory
+
+
+class ComponentRegistry:
+    """
+    组件注册表
+    
+    职责：
+    1. 管理组件规范的注册
+    2. 根据类型查找合适的工厂
+    3. 处理依赖关系
+    4. 创建组件实例
+    """
+    
+    def __init__(self):
+        self._specs: Dict[str, ComponentSpec] = {}
+        self._factories: List[ComponentFactory] = []
+        self._default_factory = DefaultComponentFactory()
+        self._lock = threading.RLock()
+    
+    def register(self, spec: ComponentSpec):
+        """注册组件规范"""
+        with self._lock:
+            self._specs[spec.name] = spec
+    
+    def register_factory(self, factory: ComponentFactory):
+        """注册组件工厂"""
+        with self._lock:
+            self._factories.append(factory)
+    
+    def get_spec(self, name: str) -> Optional[ComponentSpec]:
+        """获取组件规范"""
+        with self._lock:
+            return self._specs.get(name)
+    
+    def get_factory(self, component_type: Type) -> ComponentFactory:
+        """获取支持指定类型的工厂"""
+        with self._lock:
+            for factory in self._factories:
+                if factory.supports(component_type):
+                    return factory
+            return self._default_factory
+    
+    def create(self, name: str, **kwargs) -> Any:
+        """创建组件实例"""
+        spec = self.get_spec(name)
+        if not spec:
+            raise ValueError(f"Component spec '{name}' not found")
+        
+        factory = self.get_factory(spec.component_type)
+        return factory.create(spec, **kwargs)
+    
+    def get(self, name: str, **kwargs) -> Any:
+        """获取组件实例（create的别名）"""
+        return self.create(name, **kwargs)
+    
+    def list_components(self) -> List[str]:
+        """列出所有已注册的组件"""
+        with self._lock:
+            return list(self._specs.keys())
+    
+    def clear(self):
+        """清空注册表"""
+        with self._lock:
+            self._specs.clear()
+            self._factories.clear()
+            self._default_factory.clear_singletons()
+
+
+# 全局组件注册表
+_global_registry = ComponentRegistry()
+
+
+def get_component_registry() -> ComponentRegistry:
+    """获取全局组件注册表"""
+    return _global_registry

crawlo/filters/aioredis_filter.py

@@ -206,9 +206,32 @@ class AioRedisFilter(BaseFilter):
             self.logger.error(f"添加指纹失败: {fp[:20]}... - {e}")
             return False
 
-    async def __contains__(self, fp: str) -> bool:
+    def __contains__(self, fp: str) -> bool:
         """
-        检查指纹是否存在于Redis集合中
+        检查指纹是否存在于Redis集合中（同步方法）
+        
+        注意：Python的魔术方法__contains__不能是异步的，
+        所以这个方法提供同步接口，仅用于基本的存在性检查。
+        对于需要异步检查的场景，请使用 contains_async() 方法。
+        
+        :param fp: 请求指纹字符串
+        :return: 是否存在
+        """
+        # 由于__contains__不能是异步的，我们只能提供一个基本的同步检查
+        # 如果Redis客户端未初始化，返回False
+        if self.redis is None:
+            return False
+            
+        # 对于同步场景，我们无法进行真正的Redis查询
+        # 所以返回False，避免阻塞调用
+        # 真正的异步检查应该使用 contains_async() 方法
+        return False
+    
+    async def contains_async(self, fp: str) -> bool:
+        """
+        异步检查指纹是否存在于Redis集合中
+        
+        这是真正的异步检查方法，应该优先使用这个方法而不是__contains__
         
         :param fp: 请求指纹字符串
         :return: 是否存在

crawlo/framework.py

@@ -0,0 +1,292 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo框架统一入口
+================
+
+提供简洁、一致的API接口，隐藏内部复杂性
+"""
+
+import os
+import sys
+from typing import Type, Optional, List, Union
+
+from .crawler import ModernCrawler, CrawlerProcess
+from .initialization import initialize_framework
+from .logging import get_logger
+from .utils.env_config import get_version
+
+
+class CrawloFramework:
+    """
+    Crawlo框架门面类
+    
+    提供统一的框架入口点，简化使用复杂度
+    """
+
+    def __init__(self, settings=None, **kwargs):
+        """
+        初始化框架
+        
+        Args:
+            settings: 配置对象
+            **kwargs: 额外配置参数
+        """
+        # 合并配置
+        config = {}
+        if settings:
+            if hasattr(settings, '__dict__'):
+                config.update(settings.__dict__)
+            elif isinstance(settings, dict):
+                config.update(settings)
+        config.update(kwargs)
+        
+        # 如果没有提供配置，尝试自动加载项目配置
+        if not config:
+            config = self._load_project_config()
+
+        # 初始化框架
+        self._settings = initialize_framework(config)
+        self._logger = get_logger('crawlo.framework')
+
+        # 获取版本号
+        version = get_version()
+
+        # 创建进程管理器
+        self._process = CrawlerProcess(self._settings)
+
+        self._logger.info(f"Crawlo Framework Started {version}")
+        
+        # 获取运行模式并记录日志
+        run_mode = self._settings.get('RUN_MODE', 'unknown')
+        self._logger.info(f"Run mode: {run_mode}")
+        
+        # 记录项目名称
+        project_name = self._settings.get('PROJECT_NAME', 'unknown')
+        self._logger.info(f"Project: {project_name}")
+
+    def _load_project_config(self):
+        """
+        自动加载项目配置
+        """
+        try:
+            # 查找项目根目录
+            project_root = self._find_project_root()
+            if not project_root:
+                print("警告: 未找到项目根目录，使用默认配置")
+                return {}
+            
+            # 添加项目根目录到Python路径
+            if project_root not in sys.path:
+                sys.path.insert(0, project_root)
+            
+            # 读取crawlo.cfg配置文件
+            cfg_file = os.path.join(project_root, "crawlo.cfg")
+            if not os.path.exists(cfg_file):
+                print(f"警告: 未找到配置文件 {cfg_file}，使用默认配置")
+                return {}
+            
+            import configparser
+            config_parser = configparser.ConfigParser()
+            config_parser.read(cfg_file, encoding="utf-8")
+            
+            if not config_parser.has_section("settings") or not config_parser.has_option("settings", "default"):
+                print("警告: 配置文件缺少 [settings] 部分或 'default' 选项，使用默认配置")
+                return {}
+            
+            # 获取settings模块路径
+            settings_module_path = config_parser.get("settings", "default")
+            project_package = settings_module_path.split(".")[0]
+            
+            # 导入项目配置模块
+            import importlib
+            settings_module = importlib.import_module(settings_module_path)
+            
+            # 创建配置字典
+            project_config = {}
+            for key in dir(settings_module):
+                if key.isupper():
+                    project_config[key] = getattr(settings_module, key)
+            
+            # print(f"已加载项目配置: {settings_module_path}")
+            return project_config
+            
+        except Exception as e:
+            print(f"加载项目配置时出错: {e}")
+            return {}
+
+    def _find_project_root(self):
+        """
+        查找项目根目录（包含crawlo.cfg的目录）
+        """
+        current_path = os.getcwd()
+        
+        # 向上查找直到找到crawlo.cfg
+        checked_paths = set()
+        path = current_path
+        
+        while path not in checked_paths:
+            checked_paths.add(path)
+            
+            # 检查crawlo.cfg
+            cfg_file = os.path.join(path, "crawlo.cfg")
+            if os.path.exists(cfg_file):
+                return path
+            
+            # 向上一级目录
+            parent = os.path.dirname(path)
+            if parent == path:
+                break
+            path = parent
+        
+        return None
+
+    @property
+    def settings(self):
+        """获取配置"""
+        return self._settings
+
+    @property
+    def logger(self):
+        """获取框架日志器"""
+        return self._logger
+
+    async def run(self, spider_cls_or_name, settings=None):
+        """
+        运行单个爬虫
+        
+        Args:
+            spider_cls_or_name: Spider类或名称
+            settings: 额外配置
+            
+        Returns:
+            Crawler实例
+        """
+        # 记录启动的爬虫名称
+        if isinstance(spider_cls_or_name, str):
+            spider_name = spider_cls_or_name
+        else:
+            spider_name = getattr(spider_cls_or_name, 'name', spider_cls_or_name.__name__)
+        
+        self._logger.info(f"Starting spider: {spider_name}")
+        
+        return await self._process.crawl(spider_cls_or_name, settings)
+
+    async def run_multiple(self, spider_classes_or_names: List[Union[Type, str]],
+                           settings=None):
+        """
+        运行多个爬虫
+        
+        Args:
+            spider_classes_or_names: Spider类或名称列表
+            settings: 额外配置
+            
+        Returns:
+            结果列表
+        """
+        # 记录启动的爬虫名称
+        spider_names = []
+        for spider_cls_or_name in spider_classes_or_names:
+            if isinstance(spider_cls_or_name, str):
+                spider_names.append(spider_cls_or_name)
+            else:
+                spider_names.append(getattr(spider_cls_or_name, 'name', spider_cls_or_name.__name__))
+        
+        self._logger.info(f"Starting spiders: {', '.join(spider_names)}")
+        
+        return await self._process.crawl_multiple(spider_classes_or_names, settings)
+
+    def create_crawler(self, spider_cls: Type, settings=None) -> ModernCrawler:
+        """
+        创建Crawler实例
+        
+        Args:
+            spider_cls: Spider类
+            settings: 额外配置
+            
+        Returns:
+            ModernCrawler实例
+        """
+        merged_settings = self._merge_settings(settings)
+        return ModernCrawler(spider_cls, merged_settings)
+
+    def _merge_settings(self, additional_settings):
+        """合并配置"""
+        if not additional_settings:
+            return self._settings
+
+        from .settings.setting_manager import SettingManager
+        merged = SettingManager()
+
+        # 复制基础配置
+        if self._settings:
+            merged.update_attributes(self._settings.__dict__)
+
+        # 应用额外配置
+        if isinstance(additional_settings, dict):
+            merged.update_attributes(additional_settings)
+        elif hasattr(additional_settings, '__dict__'):
+            merged.update_attributes(additional_settings.__dict__)
+
+        return merged
+
+    def get_metrics(self) -> dict:
+        """获取框架指标"""
+        return self._process.get_metrics()
+
+
+# 全局框架实例
+_global_framework: Optional[CrawloFramework] = None
+
+
+def get_framework(settings=None, **kwargs) -> CrawloFramework:
+    """
+    获取全局框架实例（单例模式）
+    
+    Args:
+        settings: 配置对象
+        **kwargs: 额外配置参数
+        
+    Returns:
+        CrawloFramework实例
+    """
+    global _global_framework
+
+    if _global_framework is None:
+        _global_framework = CrawloFramework(settings, **kwargs)
+
+    return _global_framework
+
+
+def reset_framework():
+    """重置全局框架实例（主要用于测试）"""
+    global _global_framework
+    _global_framework = None
+
+
+# 便捷函数
+async def run_spider(spider_cls_or_name, settings=None, **kwargs):
+    """运行单个爬虫的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return await framework.run(spider_cls_or_name)
+
+
+async def run_spiders(spider_classes_or_names: List[Union[Type, str]],
+                      settings=None, **kwargs):
+    """运行多个爬虫的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return await framework.run_multiple(spider_classes_or_names)
+
+
+def create_crawler(spider_cls: Type, settings=None, **kwargs) -> ModernCrawler:
+    """创建Crawler的便捷函数"""
+    framework = get_framework(settings, **kwargs)
+    return framework.create_crawler(spider_cls)
+
+
+# 配置相关便捷函数
+def configure_framework(settings=None, **kwargs):
+    """配置框架的便捷函数"""
+    if settings or kwargs:
+        reset_framework()  # 重置以应用新配置
+    return get_framework(settings, **kwargs)