crawlo 1.1.1__py3-none-any.whl → 1.1.2__py3-none-any.whl

crawlo/utils/large_scale_helper.py

@@ -0,0 +1,344 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+大规模爬虫优化辅助工具
+"""
+import asyncio
+import json
+import time
+from typing import Generator, Iterator, List, Set, Optional, Dict, Any
+from crawlo.utils.log import get_logger
+
+
+class LargeScaleHelper:
+    """大规模爬虫辅助类"""
+    
+    def __init__(self, batch_size: int = 1000, checkpoint_interval: int = 5000):
+        self.batch_size = batch_size
+        self.checkpoint_interval = checkpoint_interval
+        self.logger = get_logger(self.__class__.__name__)
+        
+    def batch_iterator(self, data_source, start_offset: int = 0) -> Generator[List[Any], None, None]:
+        """
+        批量迭代器，适用于大量数据的分批处理
+        
+        Args:
+            data_source: 数据源（支持多种类型）
+            start_offset: 起始偏移量
+            
+        Yields:
+            每批数据的列表
+        """
+        if hasattr(data_source, '__iter__') and not isinstance(data_source, (str, bytes)):
+            # 可迭代对象
+            yield from self._iterate_batches(data_source, start_offset)
+        elif hasattr(data_source, 'get_batch'):
+            # 支持分批获取的数据源
+            yield from self._get_batches_from_source(data_source, start_offset)
+        elif callable(data_source):
+            # 函数形式的数据源
+            yield from self._get_batches_from_function(data_source, start_offset)
+        else:
+            raise ValueError(f"不支持的数据源类型: {type(data_source)}")
+    
+    def _iterate_batches(self, iterable, start_offset: int) -> Generator[List[Any], None, None]:
+        """从可迭代对象分批获取数据"""
+        iterator = iter(iterable)
+        
+        # 跳过已处理的数据
+        for _ in range(start_offset):
+            try:
+                next(iterator)
+            except StopIteration:
+                return
+        
+        while True:
+            batch = []
+            for _ in range(self.batch_size):
+                try:
+                    batch.append(next(iterator))
+                except StopIteration:
+                    if batch:
+                        yield batch
+                    return
+            
+            if batch:
+                yield batch
+    
+    def _get_batches_from_source(self, data_source, start_offset: int) -> Generator[List[Any], None, None]:
+        """从支持分批获取的数据源获取数据"""
+        offset = start_offset
+        
+        while True:
+            try:
+                batch = data_source.get_batch(offset, self.batch_size)
+                if not batch:
+                    break
+                
+                yield batch
+                offset += len(batch)
+                
+                if len(batch) < self.batch_size:
+                    break  # 已到达数据末尾
+                    
+            except Exception as e:
+                self.logger.error(f"获取批次数据失败: {e}")
+                break
+    
+    def _get_batches_from_function(self, func, start_offset: int) -> Generator[List[Any], None, None]:
+        """从函数获取批次数据"""
+        offset = start_offset
+        
+        while True:
+            try:
+                batch = func(offset, self.batch_size)
+                if not batch:
+                    break
+                
+                yield batch
+                offset += len(batch)
+                
+                if len(batch) < self.batch_size:
+                    break
+                    
+            except Exception as e:
+                self.logger.error(f"函数获取数据失败: {e}")
+                break
+
+
+class ProgressManager:
+    """进度管理器"""
+    
+    def __init__(self, progress_file: str = "spider_progress.json"):
+        self.progress_file = progress_file
+        self.logger = get_logger(self.__class__.__name__)
+        
+    def load_progress(self) -> Dict[str, Any]:
+        """加载进度"""
+        try:
+            with open(self.progress_file, 'r', encoding='utf-8') as f:
+                progress = json.load(f)
+                self.logger.info(f"📊 加载进度: {progress}")
+                return progress
+        except FileNotFoundError:
+            self.logger.info("📄 未找到进度文件，从头开始")
+            return self._get_default_progress()
+        except Exception as e:
+            self.logger.error(f"加载进度失败: {e}")
+            return self._get_default_progress()
+    
+    def save_progress(self, **kwargs):
+        """保存进度"""
+        try:
+            progress = {
+                **kwargs,
+                'timestamp': time.time(),
+                'formatted_time': time.strftime('%Y-%m-%d %H:%M:%S')
+            }
+            
+            with open(self.progress_file, 'w', encoding='utf-8') as f:
+                json.dump(progress, f, indent=2, ensure_ascii=False)
+                
+            self.logger.debug(f"💾 已保存进度: {progress}")
+            
+        except Exception as e:
+            self.logger.error(f"保存进度失败: {e}")
+    
+    def _get_default_progress(self) -> Dict[str, Any]:
+        """获取默认进度"""
+        return {
+            'batch_num': 0,
+            'processed_count': 0,
+            'skipped_count': 0,
+            'timestamp': time.time()
+        }
+
+
+class MemoryOptimizer:
+    """内存优化器"""
+    
+    def __init__(self, max_memory_mb: int = 500):
+        self.max_memory_mb = max_memory_mb
+        self.logger = get_logger(self.__class__.__name__)
+        
+    def check_memory_usage(self) -> Dict[str, float]:
+        """检查内存使用情况"""
+        try:
+            import psutil
+            process = psutil.Process()
+            memory_info = process.memory_info()
+            
+            memory_mb = memory_info.rss / 1024 / 1024
+            memory_percent = process.memory_percent()
+            
+            return {
+                'memory_mb': memory_mb,
+                'memory_percent': memory_percent,
+                'threshold_mb': self.max_memory_mb
+            }
+        except ImportError:
+            self.logger.warning("psutil 未安装，无法监控内存")
+            return {}
+        except Exception as e:
+            self.logger.error(f"检查内存失败: {e}")
+            return {}
+    
+    def should_pause_for_memory(self) -> bool:
+        """检查是否应该因内存不足而暂停"""
+        memory_info = self.check_memory_usage()
+        
+        if not memory_info:
+            return False
+            
+        memory_mb = memory_info.get('memory_mb', 0)
+        
+        if memory_mb > self.max_memory_mb:
+            self.logger.warning(f"⚠️ 内存使用过高: {memory_mb:.1f}MB > {self.max_memory_mb}MB")
+            return True
+            
+        return False
+    
+    def force_garbage_collection(self):
+        """强制垃圾回收"""
+        try:
+            import gc
+            collected = gc.collect()
+            self.logger.debug(f"🗑️ 垃圾回收: 清理了 {collected} 个对象")
+        except Exception as e:
+            self.logger.error(f"垃圾回收失败: {e}")
+
+
+class DataSourceAdapter:
+    """数据源适配器"""
+    
+    @staticmethod
+    def from_redis_queue(queue, batch_size: int = 1000):
+        """从Redis队列创建批量数据源"""
+        def get_batch(offset: int, limit: int) -> List[Dict]:
+            try:
+                # 如果队列支持范围查询
+                if hasattr(queue, 'get_range'):
+                    return queue.get_range(offset, offset + limit - 1)
+                
+                # 如果队列支持批量获取
+                if hasattr(queue, 'get_batch'):
+                    return queue.get_batch(offset, limit)
+                
+                # 模拟批量获取
+                results = []
+                for _ in range(limit):
+                    item = queue.get_nowait() if hasattr(queue, 'get_nowait') else None
+                    if item:
+                        results.append(item)
+                    else:
+                        break
+                
+                return results
+                
+            except Exception as e:
+                print(f"获取批次失败: {e}")
+                return []
+        
+        return get_batch
+    
+    @staticmethod
+    def from_database(db_helper, query: str, batch_size: int = 1000):
+        """从数据库创建批量数据源"""
+        def get_batch(offset: int, limit: int) -> List[Dict]:
+            try:
+                # 添加分页查询
+                paginated_query = f"{query} LIMIT {limit} OFFSET {offset}"
+                return db_helper.execute_query(paginated_query)
+            except Exception as e:
+                print(f"数据库查询失败: {e}")
+                return []
+        
+        return get_batch
+    
+    @staticmethod
+    def from_file(file_path: str, batch_size: int = 1000):
+        """从文件创建批量数据源"""
+        def get_batch(offset: int, limit: int) -> List[str]:
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    # 跳过已处理的行
+                    for _ in range(offset):
+                        f.readline()
+                    
+                    # 读取当前批次
+                    batch = []
+                    for _ in range(limit):
+                        line = f.readline()
+                        if not line:
+                            break
+                        batch.append(line.strip())
+                    
+                    return batch
+            except Exception as e:
+                print(f"读取文件失败: {e}")
+                return []
+        
+        return get_batch
+
+
+class LargeScaleSpiderMixin:
+    """大规模爬虫混入类"""
+    
+    def __init__(self):
+        super().__init__()
+        self.large_scale_helper = LargeScaleHelper(
+            batch_size=getattr(self, 'batch_size', 1000),
+            checkpoint_interval=getattr(self, 'checkpoint_interval', 5000)
+        )
+        self.progress_manager = ProgressManager(
+            progress_file=getattr(self, 'progress_file', f"{self.name}_progress.json")
+        )
+        self.memory_optimizer = MemoryOptimizer(
+            max_memory_mb=getattr(self, 'max_memory_mb', 500)
+        )
+        
+    def create_streaming_start_requests(self, data_source):
+        """创建流式start_requests生成器"""
+        progress = self.progress_manager.load_progress()
+        start_offset = progress.get('processed_count', 0)
+        
+        processed_count = start_offset
+        skipped_count = progress.get('skipped_count', 0)
+        
+        for batch in self.large_scale_helper.batch_iterator(data_source, start_offset):
+            
+            # 内存检查
+            if self.memory_optimizer.should_pause_for_memory():
+                self.memory_optimizer.force_garbage_collection()
+                # 可以添加延迟或其他处理
+                asyncio.sleep(1)
+            
+            for item in batch:
+                processed_count += 1
+                
+                # 检查进度保存
+                if processed_count % self.large_scale_helper.checkpoint_interval == 0:
+                    self.progress_manager.save_progress(
+                        processed_count=processed_count,
+                        skipped_count=skipped_count
+                    )
+                
+                # 生成请求
+                request = self.create_request_from_item(item)
+                if request:
+                    yield request
+                else:
+                    skipped_count += 1
+        
+        # 最终保存进度
+        self.progress_manager.save_progress(
+            processed_count=processed_count,
+            skipped_count=skipped_count,
+            completed=True
+        )
+        
+        self.logger.info(f"🎉 处理完成！总计: {processed_count}, 跳过: {skipped_count}")
+    
+    def create_request_from_item(self, item):
+        """从数据项创建请求（需要子类实现）"""
+        raise NotImplementedError("子类必须实现 create_request_from_item 方法")

crawlo/utils/queue_helper.py

@@ -0,0 +1,176 @@
+#!/usr/bin/python
+# -*- coding: UTF-8 -*-
+"""
+队列配置辅助工具
+为用户提供简洁的队列配置接口
+"""
+from typing import Dict, Any, Optional
+
+
+class QueueHelper:
+    """队列配置辅助类"""
+    
+    @staticmethod
+    def use_memory_queue(max_size: int = 2000) -> Dict[str, Any]:
+        """
+        配置使用内存队列
+        
+        Args:
+            max_size: 队列最大容量
+            
+        Returns:
+            配置字典
+        """
+        return {
+            'QUEUE_TYPE': 'memory',
+            'SCHEDULER_MAX_QUEUE_SIZE': max_size,
+        }
+    
+    @staticmethod
+    def use_redis_queue(
+        host: str = "127.0.0.1",
+        port: int = 6379,
+        password: Optional[str] = None,
+        db: int = 0,
+        queue_name: str = "crawlo:requests",
+        max_retries: int = 3,
+        timeout: int = 300
+    ) -> Dict[str, Any]:
+        """
+        配置使用 Redis 分布式队列
+        
+        Args:
+            host: Redis 主机地址
+            port: Redis 端口
+            password: Redis 密码（可选）
+            db: Redis 数据库编号
+            queue_name: 队列名称
+            max_retries: 最大重试次数
+            timeout: 操作超时时间（秒）
+            
+        Returns:
+            配置字典
+        """
+        if password:
+            redis_url = f"redis://:{password}@{host}:{port}/{db}"
+        else:
+            redis_url = f"redis://{host}:{port}/{db}"
+        
+        return {
+            'QUEUE_TYPE': 'redis',
+            'REDIS_URL': redis_url,
+            'REDIS_HOST': host,
+            'REDIS_PORT': port,
+            'REDIS_PASSWORD': password or '',
+            'REDIS_DB': db,
+            'SCHEDULER_QUEUE_NAME': queue_name,
+            'QUEUE_MAX_RETRIES': max_retries,
+            'QUEUE_TIMEOUT': timeout,
+        }
+    
+    @staticmethod
+    def auto_queue(
+        redis_fallback: bool = True,
+        memory_max_size: int = 2000,
+        **redis_kwargs
+    ) -> Dict[str, Any]:
+        """
+        配置自动选择队列类型
+        
+        Args:
+            redis_fallback: Redis 不可用时是否回退到内存队列
+            memory_max_size: 内存队列最大容量
+            **redis_kwargs: Redis 配置参数
+            
+        Returns:
+            配置字典
+        """
+        config = {
+            'QUEUE_TYPE': 'auto',
+            'SCHEDULER_MAX_QUEUE_SIZE': memory_max_size,
+        }
+        
+        # 添加 Redis 配置（用于自动检测）
+        if redis_kwargs:
+            redis_config = QueueHelper.use_redis_queue(**redis_kwargs)
+            config.update(redis_config)
+            config['QUEUE_TYPE'] = 'auto'  # 确保是自动模式
+        
+        return config
+
+
+# 预定义的常用配置
+class QueuePresets:
+    """预定义的队列配置"""
+    
+    # 开发环境：使用内存队列
+    DEVELOPMENT = QueueHelper.use_memory_queue(max_size=1000)
+    
+    # 生产环境：使用 Redis 分布式队列
+    PRODUCTION = QueueHelper.use_redis_queue(
+        host="127.0.0.1",
+        port=6379,
+        queue_name="crawlo:production",
+        max_retries=5,
+        timeout=600
+    )
+    
+    # 测试环境：自动选择，Redis 不可用时使用内存队列
+    TESTING = QueueHelper.auto_queue(
+        redis_fallback=True,
+        memory_max_size=500,
+        host="127.0.0.1",
+        port=6379,
+        queue_name="crawlo:testing"
+    )
+    
+    # 高性能环境：Redis 集群
+    HIGH_PERFORMANCE = QueueHelper.use_redis_queue(
+        host="redis-cluster.example.com",
+        port=6379,
+        queue_name="crawlo:cluster",
+        max_retries=10,
+        timeout=300
+    )
+
+
+def apply_queue_config(settings_dict: Dict[str, Any], config: Dict[str, Any]) -> None:
+    """
+    将队列配置应用到设置字典
+    
+    Args:
+        settings_dict: 现有的设置字典
+        config: 队列配置字典
+    """
+    settings_dict.update(config)
+
+
+# 使用示例和文档
+USAGE_EXAMPLES = """
+# 使用示例：
+
+# 1. 在 settings.py 中使用内存队列
+from crawlo.utils.queue_helper import QueueHelper
+apply_queue_config(locals(), QueueHelper.use_memory_queue())
+
+# 2. 在 settings.py 中使用 Redis 队列
+apply_queue_config(locals(), QueueHelper.use_redis_queue(
+    host="redis.example.com",
+    password="your_password"
+))
+
+# 3. 使用预定义配置
+from crawlo.utils.queue_helper import QueuePresets
+apply_queue_config(locals(), QueuePresets.PRODUCTION)
+
+# 4. 自动选择队列类型
+apply_queue_config(locals(), QueueHelper.auto_queue(
+    host="127.0.0.1",
+    port=6379
+))
+
+# 5. 直接在 settings 中配置
+QUEUE_TYPE = 'auto'  # 'memory', 'redis', 'auto'
+REDIS_URL = 'redis://127.0.0.1:6379/0'
+SCHEDULER_MAX_QUEUE_SIZE = 2000
+"""