crawlo 1.1.0__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/config.py

@@ -0,0 +1,280 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+Crawlo 配置工厂
+===============
+提供优雅的配置方式，让用户能够轻松选择运行模式。
+
+使用示例：
+    # 单机模式（默认）
+    config = CrawloConfig.standalone()
+    
+    # 分布式模式
+    config = CrawloConfig.distributed(redis_host='192.168.1.100')
+    
+    # 自动检测模式
+    config = CrawloConfig.auto()
+    
+    # 从环境变量
+    config = CrawloConfig.from_env()
+"""
+
+from typing import Dict, Any, Optional, Union
+import os
+from crawlo.mode_manager import ModeManager, standalone_mode, distributed_mode, auto_mode, from_env
+from crawlo.utils.log import get_logger
+
+
+class CrawloConfig:
+    """Crawlo 配置工厂类"""
+    
+    def __init__(self, settings: Dict[str, Any]):
+        self.settings = settings
+        self.logger = get_logger(self.__class__.__name__)
+    
+    def get(self, key: str, default: Any = None) -> Any:
+        """获取配置项"""
+        return self.settings.get(key, default)
+    
+    def set(self, key: str, value: Any) -> 'CrawloConfig':
+        """设置配置项（链式调用）"""
+        self.settings[key] = value
+        return self
+    
+    def update(self, settings: Dict[str, Any]) -> 'CrawloConfig':
+        """更新配置（链式调用）"""
+        self.settings.update(settings)
+        return self
+    
+    def set_concurrency(self, concurrency: int) -> 'CrawloConfig':
+        """设置并发数"""
+        return self.set('CONCURRENCY', concurrency)
+    
+    def set_delay(self, delay: float) -> 'CrawloConfig':
+        """设置请求延迟"""
+        return self.set('DOWNLOAD_DELAY', delay)
+    
+    def enable_debug(self) -> 'CrawloConfig':
+        """启用调试模式"""
+        return self.set('LOG_LEVEL', 'DEBUG')
+    
+    def enable_mysql(self) -> 'CrawloConfig':
+        """启用 MySQL 存储"""
+        pipelines = self.get('PIPELINES', [])
+        if 'crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline' not in pipelines:
+            pipelines.append('crawlo.pipelines.mysql_pipeline.AsyncmyMySQLPipeline')
+        return self.set('PIPELINES', pipelines)
+    
+    def set_redis_host(self, host: str) -> 'CrawloConfig':
+        """设置 Redis 主机"""
+        return self.set('REDIS_HOST', host)
+    
+    def to_dict(self) -> Dict[str, Any]:
+        """转换为字典"""
+        return self.settings.copy()
+    
+    def print_summary(self) -> 'CrawloConfig':
+        """打印配置摘要"""
+        mode_info = {
+            'memory': '🏠 单机模式',
+            'redis': '🌐 分布式模式', 
+            'auto': '🤖 自动检测模式'
+        }
+        
+        queue_type = self.settings.get('QUEUE_TYPE', 'memory')
+        filter_class = self.settings.get('FILTER_CLASS', '').split('.')[-1]
+        concurrency = self.settings.get('CONCURRENCY', 8)
+        
+        print("=" * 50)
+        print(f"📋 Crawlo 配置摘要")
+        print("=" * 50)
+        print(f"运行模式: {mode_info.get(queue_type, queue_type)}")
+        print(f"队列类型: {queue_type}")
+        print(f"去重方式: {filter_class}")
+        print(f"并发数量: {concurrency}")
+        
+        if queue_type == 'redis':
+            redis_host = self.settings.get('REDIS_HOST', 'localhost')
+            print(f"Redis 服务器: {redis_host}")
+        
+        print("=" * 50)
+        return self
+    
+    # ==================== 静态工厂方法 ====================
+    
+    @staticmethod
+    def standalone(
+        concurrency: int = 8,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建单机模式配置
+        
+        Args:
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = standalone_mode(
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def distributed(
+        redis_host: str = '127.0.0.1',
+        redis_port: int = 6379,
+        redis_password: Optional[str] = None,
+        project_name: str = 'crawlo',
+        concurrency: int = 16,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建分布式模式配置
+        
+        Args:
+            redis_host: Redis 服务器地址
+            redis_port: Redis 端口
+            redis_password: Redis 密码
+            project_name: 项目名称（用于命名空间）
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = distributed_mode(
+            redis_host=redis_host,
+            redis_port=redis_port,
+            redis_password=redis_password,
+            project_name=project_name,
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def auto(
+        concurrency: int = 12,
+        download_delay: float = 1.0,
+        **kwargs
+    ) -> 'CrawloConfig':
+        """
+        创建自动检测模式配置
+        
+        Args:
+            concurrency: 并发数
+            download_delay: 下载延迟
+            **kwargs: 其他配置项
+        """
+        settings = auto_mode(
+            CONCURRENCY=concurrency,
+            DOWNLOAD_DELAY=download_delay,
+            **kwargs
+        )
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def from_env(default_mode: str = 'standalone') -> 'CrawloConfig':
+        """
+        从环境变量创建配置
+        
+        支持的环境变量：
+        - CRAWLO_MODE: 运行模式 (standalone/distributed/auto)
+        - REDIS_HOST: Redis 主机
+        - REDIS_PORT: Redis 端口
+        - REDIS_PASSWORD: Redis 密码
+        - CONCURRENCY: 并发数
+        - PROJECT_NAME: 项目名称
+        """
+        settings = from_env(default_mode)
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def custom(settings: Dict[str, Any]) -> 'CrawloConfig':
+        """
+        创建自定义配置
+        
+        Args:
+            settings: 自定义配置字典
+        """
+        return CrawloConfig(settings)
+    
+    @staticmethod
+    def presets() -> 'Presets':
+        """获取预设配置对象"""
+        return Presets()
+
+
+# ==================== 便利函数 ====================
+
+def create_config(
+    mode: str = 'standalone',
+    **kwargs
+) -> CrawloConfig:
+    """
+    便利函数：创建配置
+    
+    Args:
+        mode: 运行模式 ('standalone', 'distributed', 'auto')
+        **kwargs: 配置参数
+    """
+    if mode.lower() == 'standalone':
+        return CrawloConfig.standalone(**kwargs)
+    elif mode.lower() == 'distributed':
+        return CrawloConfig.distributed(**kwargs)
+    elif mode.lower() == 'auto':
+        return CrawloConfig.auto(**kwargs)
+    else:
+        raise ValueError(f"不支持的运行模式: {mode}")
+
+
+# ==================== 预设配置 ====================
+
+class Presets:
+    """预设配置类"""
+    
+    @staticmethod
+    def development() -> CrawloConfig:
+        """开发环境配置"""
+        return CrawloConfig.standalone(
+            concurrency=4,
+            download_delay=2.0,
+            LOG_LEVEL='DEBUG',
+            STATS_DUMP=True
+        )
+    
+    @staticmethod
+    def production() -> CrawloConfig:
+        """生产环境配置"""
+        return CrawloConfig.auto(
+            concurrency=16,
+            download_delay=1.0,
+            LOG_LEVEL='INFO',
+            RETRY_TIMES=5
+        )
+    
+    @staticmethod
+    def large_scale(redis_host: str, project_name: str) -> CrawloConfig:
+        """大规模分布式配置"""
+        return CrawloConfig.distributed(
+            redis_host=redis_host,
+            project_name=project_name,
+            concurrency=32,
+            download_delay=0.5,
+            SCHEDULER_MAX_QUEUE_SIZE=10000,
+            LARGE_SCALE_BATCH_SIZE=2000
+        )
+    
+    @staticmethod
+    def gentle() -> CrawloConfig:
+        """温和模式配置（避免被封）"""
+        return CrawloConfig.standalone(
+            concurrency=2,
+            download_delay=3.0,
+            RANDOMNESS=True,
+            RANDOM_RANGE=(2.0, 5.0)
+        )

crawlo/core/__init__.py

@@ -1,2 +1,2 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-

crawlo/core/engine.py

@@ -1,159 +1,172 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-import asyncio
-from inspect import iscoroutine
-from typing import Optional, Generator, Callable
-
-from crawlo import Request, Item
-from crawlo.spider import Spider
-from crawlo.utils.log import get_logger
-from crawlo.exceptions import OutputError
-from crawlo.core.scheduler import Scheduler
-from crawlo.core.processor import Processor
-from crawlo.task_manager import TaskManager
-from crawlo.utils.project import load_class
-from crawlo.downloader import DownloaderBase
-from crawlo.utils.func_tools import transform
-from crawlo.event import spider_opened, spider_error, request_scheduled
-
-
-class Engine(object):
-
-    def __init__(self, crawler):
-        self.running = False
-        self.normal = True
-        self.crawler = crawler
-        self.settings = crawler.settings
-        self.spider: Optional[Spider] = None
-        self.downloader: Optional[DownloaderBase] = None
-        self.scheduler: Optional[Scheduler] = None
-        self.processor: Optional[Processor] = None
-        self.start_requests: Optional[Generator] = None
-        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
-
-        self.logger = get_logger(name=self.__class__.__name__)
-
-    def _get_downloader_cls(self):
-        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
-        if not issubclass(downloader_cls, DownloaderBase):
-            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
-        return downloader_cls
-
-    def engine_start(self):
-        self.running = True
-        self.logger.info(
-            f"Crawlo (version {self.settings.get_float('VERSION')}) started. "
-            f"(project name : {self.settings.get('PROJECT_NAME')})"
-        )
-
-    async def start_spider(self, spider):
-        self.spider = spider
-
-        self.scheduler = Scheduler.create_instance(self.crawler)
-        if hasattr(self.scheduler, 'open'):
-            self.scheduler.open()
-
-        downloader_cls = self._get_downloader_cls()
-        self.downloader = downloader_cls(self.crawler)
-        if hasattr(self.downloader, 'open'):
-            self.downloader.open()
-
-        self.processor = Processor(self.crawler)
-        if hasattr(self.processor, 'open'):
-            self.processor.open()
-
-        self.start_requests = iter(spider.start_requests())
-        await self._open_spider()
-
-    async def crawl(self):
-        """
-        Crawl the spider
-        """
-        while self.running:
-            if request := await self._get_next_request():
-                await self._crawl(request)
-            try:
-                start_request = next(self.start_requests)
-            except StopIteration:
-                self.start_requests = None
-            except Exception as exp:
-                # 1、发去请求的request全部运行完毕
-                # 2、调度器是否空闲
-                # 3、下载器是否空闲
-                if not await self._exit():
-                    continue
-                self.running = False
-                if self.start_requests is not None:
-                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
-            else:
-                # 请求入队
-                await self.enqueue_request(start_request)
-
-        if not self.running:
-            await self.close_spider()
-
-    async def _open_spider(self):
-        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
-        crawling = asyncio.create_task(self.crawl())
-        await crawling
-
-    async def _crawl(self, request):
-        # TODO 实现并发
-        async def crawl_task():
-            outputs = await self._fetch(request)
-            # TODO 处理output
-            if outputs:
-                await self._handle_spider_output(outputs)
-
-        # asyncio.create_task(crawl_task())
-        self.task_manager.create_task(crawl_task())
-
-    async def _fetch(self, request):
-        async def _successful(_response):
-            callback: Callable = request.callback or self.spider.parse
-            if _outputs := callback(_response):
-                if iscoroutine(_outputs):
-                    await _outputs
-                else:
-                    return transform(_outputs, _response)
-
-        _response = await self.downloader.fetch(request)
-        if _response is None:
-            return None
-        output = await _successful(_response)
-        return output
-
-    async def enqueue_request(self, start_request):
-        await self._schedule_request(start_request)
-
-    async def _schedule_request(self, request):
-        # TODO 去重
-        if await self.scheduler.enqueue_request(request):
-            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
-
-    async def _get_next_request(self):
-        return await self.scheduler.next_request()
-
-    async def _handle_spider_output(self, outputs):
-        async for spider_output in outputs:
-            if isinstance(spider_output, (Request, Item)):
-                await self.processor.enqueue(spider_output)
-            elif isinstance(spider_output, Exception):
-                asyncio.create_task(
-                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
-                )
-                raise spider_output
-            else:
-                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
-
-    async def _exit(self):
-        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
-            return True
-        return False
-
-    async def close_spider(self):
-        await asyncio.gather(*self.task_manager.current_task)
-        await self.scheduler.close()
-        await self.downloader.close()
-        if self.normal:
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+import asyncio
+from inspect import iscoroutine
+from typing import Optional, Generator, Callable
+
+from crawlo import Request, Item
+from crawlo.spider import Spider
+from crawlo.utils.log import get_logger
+from crawlo.exceptions import OutputError
+from crawlo.core.scheduler import Scheduler
+from crawlo.core.processor import Processor
+from crawlo.task_manager import TaskManager
+from crawlo.project import load_class
+from crawlo.downloader import DownloaderBase
+from crawlo.utils.func_tools import transform
+from crawlo.event import spider_opened, spider_error, request_scheduled
+
+
+class Engine(object):
+
+    def __init__(self, crawler):
+        self.running = False
+        self.normal = True
+        self.crawler = crawler
+        self.settings = crawler.settings
+        self.spider: Optional[Spider] = None
+        self.downloader: Optional[DownloaderBase] = None
+        self.scheduler: Optional[Scheduler] = None
+        self.processor: Optional[Processor] = None
+        self.start_requests: Optional[Generator] = None
+        self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
+
+        self.logger = get_logger(name=self.__class__.__name__)
+
+    def _get_downloader_cls(self):
+        """获取下载器类，支持多种配置方式"""
+        # 方式1: 使用 DOWNLOADER_TYPE 简化名称（推荐）
+        downloader_type = self.settings.get('DOWNLOADER_TYPE')
+        if downloader_type:
+            try:
+                from crawlo.downloader import get_downloader_class
+                downloader_cls = get_downloader_class(downloader_type)
+                self.logger.debug(f"使用下载器类型: {downloader_type} -> {downloader_cls.__name__}")
+                return downloader_cls
+            except (ImportError, ValueError) as e:
+                self.logger.warning(f"无法使用下载器类型 '{downloader_type}': {e}，回退到默认配置")
+        
+        # 方式2: 使用 DOWNLOADER 完整类路径（兼容旧版本）
+        downloader_cls = load_class(self.settings.get('DOWNLOADER'))
+        if not issubclass(downloader_cls, DownloaderBase):
+            raise TypeError(f'Downloader {downloader_cls.__name__} is not subclass of DownloaderBase.')
+        return downloader_cls
+
+    def engine_start(self):
+        self.running = True
+        self.logger.info(
+            f"Crawlo (version {self.settings.get_float('VERSION')}) started. "
+            f"(project name : {self.settings.get('PROJECT_NAME')})"
+        )
+
+    async def start_spider(self, spider):
+        self.spider = spider
+
+        self.scheduler = Scheduler.create_instance(self.crawler)
+        if hasattr(self.scheduler, 'open'):
+            await self.scheduler.open()
+
+        downloader_cls = self._get_downloader_cls()
+        self.downloader = downloader_cls(self.crawler)
+        if hasattr(self.downloader, 'open'):
+            self.downloader.open()
+
+        self.processor = Processor(self.crawler)
+        if hasattr(self.processor, 'open'):
+            self.processor.open()
+
+        self.start_requests = iter(spider.start_requests())
+        await self._open_spider()
+
+    async def crawl(self):
+        """
+        Crawl the spider
+        """
+        while self.running:
+            if request := await self._get_next_request():
+                await self._crawl(request)
+            try:
+                start_request = next(self.start_requests)
+            except StopIteration:
+                self.start_requests = None
+            except Exception as exp:
+                # 1、发去请求的request全部运行完毕
+                # 2、调度器是否空闲
+                # 3、下载器是否空闲
+                if not await self._exit():
+                    continue
+                self.running = False
+                if self.start_requests is not None:
+                    self.logger.error(f"启动请求时发生错误: {str(exp)}")
+            else:
+                # 请求入队
+                await self.enqueue_request(start_request)
+
+        if not self.running:
+            await self.close_spider()
+
+    async def _open_spider(self):
+        asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
+        crawling = asyncio.create_task(self.crawl())
+        await crawling
+
+    async def _crawl(self, request):
+        # TODO 实现并发
+        async def crawl_task():
+            outputs = await self._fetch(request)
+            # TODO 处理output
+            if outputs:
+                await self._handle_spider_output(outputs)
+
+        # 使用异步任务创建，遵守并发限制
+        await self.task_manager.create_task(crawl_task())
+
+    async def _fetch(self, request):
+        async def _successful(_response):
+            callback: Callable = request.callback or self.spider.parse
+            if _outputs := callback(_response):
+                if iscoroutine(_outputs):
+                    await _outputs
+                else:
+                    return transform(_outputs, _response)
+
+        _response = await self.downloader.fetch(request)
+        if _response is None:
+            return None
+        output = await _successful(_response)
+        return output
+
+    async def enqueue_request(self, start_request):
+        await self._schedule_request(start_request)
+
+    async def _schedule_request(self, request):
+        # TODO 去重
+        if await self.scheduler.enqueue_request(request):
+            asyncio.create_task(self.crawler.subscriber.notify(request_scheduled, request, self.crawler.spider))
+
+    async def _get_next_request(self):
+        return await self.scheduler.next_request()
+
+    async def _handle_spider_output(self, outputs):
+        async for spider_output in outputs:
+            if isinstance(spider_output, (Request, Item)):
+                await self.processor.enqueue(spider_output)
+            elif isinstance(spider_output, Exception):
+                asyncio.create_task(
+                    self.crawler.subscriber.notify(spider_error, spider_output, self.spider)
+                )
+                raise spider_output
+            else:
+                raise OutputError(f'{type(self.spider)} must return `Request` or `Item`.')
+
+    async def _exit(self):
+        if self.scheduler.idle() and self.downloader.idle() and self.task_manager.all_done() and self.processor.idle():
+            return True
+        return False
+
+    async def close_spider(self):
+        await asyncio.gather(*self.task_manager.current_task)
+        await self.scheduler.close()
+        await self.downloader.close()
+        if self.normal:
             await self.crawler.close()