crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/pipelines/pipeline_manager.py

@@ -1,56 +1,56 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import List
-from pprint import pformat
-from asyncio import create_task
-
-
-from crawlo.utils.log import get_logger
-from crawlo.event import item_successful, item_discard
-from crawlo.utils.project import load_class, common_call
-from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
-
-
-class PipelineManager:
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.pipelines: List = []
-        self.methods: List = []
-
-        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
-        pipelines = self.crawler.settings.get_list('PIPELINES')
-        self._add_pipelines(pipelines)
-        self._add_methods()
-
-    @classmethod
-    def from_crawler(cls, *args, **kwargs):
-        o = cls(*args, **kwargs)
-        return o
-
-    def _add_pipelines(self, pipelines):
-        for pipeline in pipelines:
-            pipeline_cls = load_class(pipeline)
-            if not hasattr(pipeline_cls, 'from_crawler'):
-                raise PipelineInitError(
-                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
-                )
-            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
-        if pipelines:
-            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
-
-    def _add_methods(self):
-        for pipeline in self.pipelines:
-            if hasattr(pipeline, 'process_item'):
-                self.methods.append(pipeline.process_item)
-
-    async def process_item(self, item):
-        try:
-            for method in self.methods:
-                item = await common_call(method, item, self.crawler.spider)
-                if item is None:
-                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
-        except ItemDiscard as exc:
-            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
-        else:
-            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import List
+from pprint import pformat
+from asyncio import create_task
+
+
+from crawlo.utils.log import get_logger
+from crawlo.event import item_successful, item_discard
+from crawlo.project import load_class, common_call
+from crawlo.exceptions import PipelineInitError, ItemDiscard, InvalidOutputError
+
+
+class PipelineManager:
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.pipelines: List = []
+        self.methods: List = []
+
+        self.logger = get_logger(self.__class__.__name__, self.crawler.settings.get('LOG_LEVEL'))
+        pipelines = self.crawler.settings.get_list('PIPELINES')
+        self._add_pipelines(pipelines)
+        self._add_methods()
+
+    @classmethod
+    def from_crawler(cls, *args, **kwargs):
+        o = cls(*args, **kwargs)
+        return o
+
+    def _add_pipelines(self, pipelines):
+        for pipeline in pipelines:
+            pipeline_cls = load_class(pipeline)
+            if not hasattr(pipeline_cls, 'from_crawler'):
+                raise PipelineInitError(
+                    f"Pipeline init failed, must inherit from `BasePipeline` or have a `create_instance` method"
+                )
+            self.pipelines.append(pipeline_cls.from_crawler(self.crawler))
+        if pipelines:
+            self.logger.info(f"enabled pipelines: \n {pformat(pipelines)}")
+
+    def _add_methods(self):
+        for pipeline in self.pipelines:
+            if hasattr(pipeline, 'process_item'):
+                self.methods.append(pipeline.process_item)
+
+    async def process_item(self, item):
+        try:
+            for method in self.methods:
+                item = await common_call(method, item, self.crawler.spider)
+                if item is None:
+                    raise InvalidOutputError(f"{method.__qualname__} return None is not supported.")
+        except ItemDiscard as exc:
+            create_task(self.crawler.subscriber.notify(item_discard, item, exc, self.crawler.spider))
+        else:
+            create_task(self.crawler.subscriber.notify(item_successful, item, self.crawler.spider))

crawlo/project.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo 项目初始化模块
+
+负责：
+1. 向上搜索项目根目录（通过 crawlo.cfg 或 settings.py）
+2. 将项目根目录加入 sys.path
+3. 加载 settings 模块
+4. 返回 SettingManager 实例
+"""
+import os
+import sys
+import configparser
+from importlib import import_module
+from inspect import iscoroutinefunction
+from typing import Callable, Optional, Tuple
+
+from crawlo.utils.log import get_logger
+from crawlo.settings.setting_manager import SettingManager
+
+logger = get_logger(__name__)
+
+
+def _find_project_root(start_path: str = ".") -> Optional[str]:
+    """
+    从指定路径向上查找项目根目录。
+    识别依据：
+        1. 存在 'crawlo.cfg'
+        2. 存在 '__init__.py' 和 'settings.py'（即 Python 包）
+    """
+    path = os.path.abspath(start_path)
+    while True:
+        cfg_file = os.path.join(path, "crawlo.cfg")
+        if os.path.isfile(cfg_file):
+            logger.info(f"✅ 找到项目配置文件: {cfg_file}")
+            return path
+
+        settings_file = os.path.join(path, "settings.py")
+        init_file = os.path.join(path, "__init__.py")
+        if os.path.isfile(settings_file) and os.path.isfile(init_file):
+            logger.info(f"✅ 找到项目模块: {path}")
+            return path
+
+        parent = os.path.dirname(path)
+        if parent == path:
+            break
+        path = parent
+
+    logger.warning("❌ 未找到 Crawlo 项目根目录。请确保在包含 'crawlo.cfg' 或 'settings.py' 的目录运行。")
+    return None
+
+
+def _get_settings_module_from_cfg(cfg_path: str) -> str:
+    """从 crawlo.cfg 读取 settings 模块路径"""
+    config = configparser.ConfigParser()
+    try:
+        config.read(cfg_path, encoding="utf-8")
+        if config.has_section("settings") and config.has_option("settings", "default"):
+            module_path = config.get("settings", "default")
+            logger.info(f"📄 从 crawlo.cfg 加载 settings 模块: {module_path}")
+            return module_path
+        else:
+            raise RuntimeError(f"配置文件缺少 [settings] 或 default 选项: {cfg_path}")
+    except Exception as e:
+        raise RuntimeError(f"解析 crawlo.cfg 失败: {e}")
+
+
+def get_settings(custom_settings: Optional[dict] = None) -> SettingManager:
+    """
+    获取配置管理器实例（主入口函数）
+
+    Args:
+        custom_settings: 运行时自定义配置，会覆盖 settings.py
+
+    Returns:
+        SettingManager: 已加载配置的实例
+    """
+    logger.info("🚀 正在初始化 Crawlo 项目配置...")
+
+    # 1. 查找项目根
+    project_root = _find_project_root()
+    if not project_root:
+        raise RuntimeError("未找到 Crawlo 项目，请检查项目结构")
+
+    # 2. 确定 settings 模块
+    settings_module_path = None
+    cfg_file = os.path.join(project_root, "crawlo.cfg")
+
+    if os.path.isfile(cfg_file):
+        settings_module_path = _get_settings_module_from_cfg(cfg_file)
+    else:
+        # 推断：项目目录名.settings
+        project_name = os.path.basename(project_root)
+        settings_module_path = f"{project_name}.settings"
+        logger.warning(f"⚠️ 未找到 crawlo.cfg，推断 settings 模块为: {settings_module_path}")
+
+    # 3. 注入 sys.path
+    project_root_str = os.path.abspath(project_root)
+    if project_root_str not in sys.path:
+        sys.path.insert(0, project_root_str)
+        logger.info(f"📁 项目根目录已加入 sys.path: {project_root_str}")
+
+    # 4. 加载 SettingManager
+    logger.info(f"⚙️ 正在加载配置模块: {settings_module_path}")
+    settings = SettingManager()
+
+    try:
+        settings.set_settings(settings_module_path)
+        logger.info("✅ settings 模块加载成功")
+    except Exception as e:
+        raise ImportError(f"加载 settings 模块失败 '{settings_module_path}': {e}")
+
+    # 5. 合并运行时配置
+    if custom_settings:
+        settings.update_attributes(custom_settings)
+        logger.info(f"🔧 已应用运行时自定义配置: {list(custom_settings.keys())}")
+
+    logger.info("🎉 Crawlo 项目配置初始化完成！")
+    return settings
+
+
+def load_class(_path):
+    if not isinstance(_path, str):
+        if callable(_path):
+            return _path
+        else:
+            raise TypeError(f"args expect str or object, got {_path}")
+
+    module_name, class_name = _path.rsplit('.', 1)
+    module = import_module(module_name)
+
+    try:
+        cls = getattr(module, class_name)
+    except AttributeError:
+        raise NameError(f"Module {module_name!r} has no class named {class_name!r}")
+    return cls
+
+
+def merge_settings(spider, settings):
+    spider_name = getattr(spider, 'name', 'UnknownSpider')
+    if hasattr(spider, 'custom_settings'):
+        custom_settings = getattr(spider, 'custom_settings')
+        settings.update_attributes(custom_settings)
+    else:
+        logger.debug(f"爬虫 '{spider_name}' 无 custom_settings，跳过合并")  # 添加日志
+
+
+async def common_call(func: Callable, *args, **kwargs):
+    if iscoroutinefunction(func):
+        return await func(*args, **kwargs)
+    else:
+        return func(*args, **kwargs)

crawlo/queue/pqueue.py

@@ -0,0 +1,37 @@
+# -*- coding:UTF-8 -*-
+import json
+import sys
+import asyncio
+from asyncio import PriorityQueue
+from typing import Optional
+
+
+from crawlo import Request
+
+
+class SpiderPriorityQueue(PriorityQueue):
+    """带超时功能的异步优先级队列"""
+
+    def __init__(self, maxsize: int = 0) -> None:
+        """初始化队列，maxsize为0表示无大小限制"""
+        super().__init__(maxsize)
+
+    async def get(self, timeout: float = 0.1) -> Optional[Request]:
+        """
+        异步获取队列元素，带超时功能
+
+        Args:
+            timeout: 超时时间（秒），默认0.1秒
+
+        Returns:
+            队列元素(优先级, 值)或None(超时)
+        """
+        try:
+            # 根据Python版本选择超时实现方式
+            if sys.version_info >= (3, 11):
+                async with asyncio.timeout(timeout):
+                    return await super().get()
+            else:
+                return await asyncio.wait_for(super().get(), timeout=timeout)
+        except asyncio.TimeoutError:
+            return None

crawlo/queue/queue_manager.py

@@ -0,0 +1,304 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+统一的队列管理器
+提供简洁、一致的队列接口，自动处理不同队列类型的差异
+"""
+from typing import Optional, Dict, Any, Union
+from enum import Enum
+import asyncio
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.queue.pqueue import SpiderPriorityQueue
+from crawlo import Request
+
+try:
+    from crawlo.queue.redis_priority_queue import RedisPriorityQueue
+    REDIS_AVAILABLE = True
+except ImportError:
+    RedisPriorityQueue = None
+    REDIS_AVAILABLE = False
+
+
+class QueueType(Enum):
+    """队列类型枚举"""
+    MEMORY = "memory"
+    REDIS = "redis"
+    AUTO = "auto"  # 自动选择
+
+
+class QueueConfig:
+    """队列配置类"""
+    
+    def __init__(
+        self,
+        queue_type: Union[QueueType, str] = QueueType.AUTO,
+        redis_url: Optional[str] = None,
+        redis_host: str = "127.0.0.1",
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        redis_db: int = 0,
+        queue_name: str = "crawlo:requests",
+        max_queue_size: int = 1000,
+        max_retries: int = 3,
+        timeout: int = 300,
+        **kwargs
+    ):
+        self.queue_type = QueueType(queue_type) if isinstance(queue_type, str) else queue_type
+        
+        # Redis 配置
+        if redis_url:
+            self.redis_url = redis_url
+        else:
+            if redis_password:
+                self.redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+            else:
+                self.redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
+        
+        self.queue_name = queue_name
+        self.max_queue_size = max_queue_size
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self.extra_config = kwargs
+    
+    @classmethod
+    def from_settings(cls, settings) -> 'QueueConfig':
+        """从 settings 创建配置"""
+        return cls(
+            queue_type=settings.get('QUEUE_TYPE', QueueType.AUTO),
+            redis_url=settings.get('REDIS_URL'),
+            redis_host=settings.get('REDIS_HOST', '127.0.0.1'),
+            redis_port=settings.get_int('REDIS_PORT', 6379),
+            redis_password=settings.get('REDIS_PASSWORD'),
+            redis_db=settings.get_int('REDIS_DB', 0),
+            queue_name=settings.get('SCHEDULER_QUEUE_NAME', 'crawlo:requests'),
+            max_queue_size=settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 1000),
+            max_retries=settings.get_int('QUEUE_MAX_RETRIES', 3),
+            timeout=settings.get_int('QUEUE_TIMEOUT', 300)
+        )
+
+
+class QueueManager:
+    """统一的队列管理器"""
+    
+    def __init__(self, config: QueueConfig):
+        self.config = config
+        self.logger = get_logger(self.__class__.__name__)
+        self.request_serializer = RequestSerializer()
+        self._queue = None
+        self._queue_semaphore = None
+        self._queue_type = None
+        self._health_status = "unknown"
+    
+    async def initialize(self) -> bool:
+        """初始化队列"""
+        try:
+            queue_type = await self._determine_queue_type()
+            self._queue = await self._create_queue(queue_type)
+            self._queue_type = queue_type
+            
+            # 测试队列健康状态
+            await self._health_check()
+            
+            self.logger.info(f"✅ 队列初始化成功: {queue_type.value}")
+            self.logger.info(f"📊 队列配置: {self._get_queue_info()}")
+            return True
+            
+        except Exception as e:
+            self.logger.error(f"❌ 队列初始化失败: {e}")
+            self._health_status = "error"
+            return False
+    
+    async def put(self, request: Request, priority: int = 0) -> bool:
+        """统一的入队接口"""
+        if not self._queue:
+            raise RuntimeError("队列未初始化")
+        
+        try:
+            # 序列化处理（仅对 Redis 队列）
+            if self._queue_type == QueueType.REDIS:
+                request = self.request_serializer.prepare_for_serialization(request)
+            
+            # 背压控制（仅对内存队列）
+            if self._queue_semaphore:
+                # 对于大量请求，使用非阻塞式检查
+                if not self._queue_semaphore.locked():
+                    await self._queue_semaphore.acquire()
+                else:
+                    # 如果队列已满，返回 False 而不是阻塞
+                    self.logger.warning("队列已满，跳过当前请求")
+                    return False
+            
+            # 统一的入队操作
+            if hasattr(self._queue, 'put'):
+                if self._queue_type == QueueType.REDIS:
+                    success = await self._queue.put(request, priority)
+                else:
+                    await self._queue.put(request)
+                    success = True
+            else:
+                raise RuntimeError(f"队列类型 {self._queue_type} 不支持 put 操作")
+            
+            if success:
+                self.logger.debug(f"✅ 请求入队成功: {request.url}")
+            
+            return success
+            
+        except Exception as e:
+            self.logger.error(f"❌ 请求入队失败: {e}")
+            if self._queue_semaphore:
+                self._queue_semaphore.release()
+            return False
+    
+    async def get(self, timeout: float = 5.0) -> Optional[Request]:
+        """统一的出队接口"""
+        if not self._queue:
+            raise RuntimeError("队列未初始化")
+        
+        try:
+            request = await self._queue.get(timeout=timeout)
+            
+            # 释放信号量（仅对内存队列）
+            if self._queue_semaphore and request:
+                self._queue_semaphore.release()
+            
+            # 反序列化处理（仅对 Redis 队列）
+            if request and self._queue_type == QueueType.REDIS:
+                # 这里需要 spider 实例，暂时返回原始请求
+                # 实际的 callback 恢复在 scheduler 中处理
+                pass
+            
+            return request
+            
+        except Exception as e:
+            self.logger.error(f"❌ 请求出队失败: {e}")
+            return None
+    
+    async def size(self) -> int:
+        """获取队列大小"""
+        if not self._queue:
+            return 0
+        
+        try:
+            if hasattr(self._queue, 'qsize'):
+                if asyncio.iscoroutinefunction(self._queue.qsize):
+                    return await self._queue.qsize()
+                else:
+                    return self._queue.qsize()
+            return 0
+        except Exception as e:
+            self.logger.warning(f"获取队列大小失败: {e}")
+            return 0
+    
+    def empty(self) -> bool:
+        """检查队列是否为空"""
+        try:
+            # 对于内存队列，可以同步检查
+            if self._queue_type == QueueType.MEMORY:
+                return self._queue.qsize() == 0
+            # 对于 Redis 队列，需要异步操作，这里返回近似值
+            return False
+        except Exception:
+            return True
+    
+    async def close(self) -> None:
+        """关闭队列"""
+        if self._queue and hasattr(self._queue, 'close'):
+            try:
+                await self._queue.close()
+                self.logger.info("✅ 队列已关闭")
+            except Exception as e:
+                self.logger.warning(f"关闭队列时发生错误: {e}")
+    
+    def get_status(self) -> Dict[str, Any]:
+        """获取队列状态信息"""
+        return {
+            "type": self._queue_type.value if self._queue_type else "unknown",
+            "health": self._health_status,
+            "config": self._get_queue_info(),
+            "initialized": self._queue is not None
+        }
+    
+    async def _determine_queue_type(self) -> QueueType:
+        """确定队列类型"""
+        if self.config.queue_type == QueueType.AUTO:
+            # 自动选择：优先使用 Redis（如果可用）
+            if REDIS_AVAILABLE and self.config.redis_url:
+                # 测试 Redis 连接
+                try:
+                    test_queue = RedisPriorityQueue(self.config.redis_url)
+                    await test_queue.connect()
+                    await test_queue.close()
+                    self.logger.info("🔍 自动检测: Redis 可用，使用分布式队列")
+                    return QueueType.REDIS
+                except Exception as e:
+                    self.logger.warning(f"🔍 自动检测: Redis 不可用 ({e})，使用内存队列")
+                    return QueueType.MEMORY
+            else:
+                self.logger.info("🔍 自动检测: Redis 未配置，使用内存队列")
+                return QueueType.MEMORY
+        
+        elif self.config.queue_type == QueueType.REDIS:
+            if not REDIS_AVAILABLE:
+                raise RuntimeError("Redis 队列不可用：未安装 redis 依赖")
+            if not self.config.redis_url:
+                raise RuntimeError("Redis 队列不可用：未配置 REDIS_URL")
+            return QueueType.REDIS
+        
+        elif self.config.queue_type == QueueType.MEMORY:
+            return QueueType.MEMORY
+        
+        else:
+            raise ValueError(f"不支持的队列类型: {self.config.queue_type}")
+    
+    async def _create_queue(self, queue_type: QueueType):
+        """创建队列实例"""
+        if queue_type == QueueType.REDIS:
+            queue = RedisPriorityQueue(
+                redis_url=self.config.redis_url,
+                queue_name=self.config.queue_name,
+                max_retries=self.config.max_retries,
+                timeout=self.config.timeout
+            )
+            # 不需要立即连接，使用 lazy connect
+            return queue
+        
+        elif queue_type == QueueType.MEMORY:
+            queue = SpiderPriorityQueue()
+            # 为内存队列设置背压控制
+            self._queue_semaphore = asyncio.Semaphore(self.config.max_queue_size)
+            return queue
+        
+        else:
+            raise ValueError(f"不支持的队列类型: {queue_type}")
+    
+    async def _health_check(self) -> None:
+        """健康检查"""
+        try:
+            if self._queue_type == QueueType.REDIS:
+                # 测试 Redis 连接
+                await self._queue.connect()
+                self._health_status = "healthy"
+            else:
+                # 内存队列总是健康的
+                self._health_status = "healthy"
+        except Exception as e:
+            self.logger.warning(f"队列健康检查失败: {e}")
+            self._health_status = "unhealthy"
+    
+    def _get_queue_info(self) -> Dict[str, Any]:
+        """获取队列配置信息"""
+        info = {
+            "queue_name": self.config.queue_name,
+            "max_queue_size": self.config.max_queue_size
+        }
+        
+        if self._queue_type == QueueType.REDIS:
+            info.update({
+                "redis_url": self.config.redis_url,
+                "max_retries": self.config.max_retries,
+                "timeout": self.config.timeout
+            })
+        
+        return info