crawlo 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

crawlo/core/enhanced_engine.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+增强的引擎实现
+解决大规模请求生成时的并发控制和背压问题
+"""
+import asyncio
+
+from crawlo.core.engine import Engine as BaseEngine
+from crawlo.utils.log import get_logger
+
+
+class EnhancedEngine(BaseEngine):
+    """
+    增强的引擎实现
+    
+    主要改进：
+    1. 智能的请求生成控制
+    2. 背压感知的调度
+    3. 动态并发调整
+    """
+    
+    def __init__(self, crawler):
+        super().__init__(crawler)
+        
+        # 增强控制参数
+        self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
+        self.generation_batch_size = 10
+        self.generation_interval = 0.05
+        self.backpressure_ratio = 0.8  # 队列达到80%时启动背压
+        
+        # 状态跟踪
+        self._generation_paused = False
+        self._last_generation_time = 0
+        self._generation_stats = {
+            'total_generated': 0,
+            'backpressure_events': 0
+        }
+        
+        self.logger = get_logger(self.__class__.__name__)
+    
+    async def crawl(self):
+        """
+        增强的爬取循环
+        支持智能请求生成和背压控制
+        """
+        generation_task = None
+        
+        try:
+            # 启动请求生成任务
+            if self.start_requests:
+                generation_task = asyncio.create_task(
+                    self._controlled_request_generation()
+                )
+            
+            # 主爬取循环
+            while self.running:
+                # 获取并处理请求
+                if request := await self._get_next_request():
+                    await self._crawl(request)
+                
+                # 检查退出条件
+                if await self._should_exit():
+                    break
+                
+                # 短暂休息避免忙等
+                await asyncio.sleep(0.001)
+        
+        finally:
+            # 清理生成任务
+            if generation_task and not generation_task.done():
+                generation_task.cancel()
+                try:
+                    await generation_task
+                except asyncio.CancelledError:
+                    pass
+            
+            await self.close_spider()
+    
+    async def _controlled_request_generation(self):
+        """受控的请求生成"""
+        self.logger.info("🎛️ 启动受控请求生成")
+        
+        batch = []
+        total_generated = 0
+        
+        try:
+            for request in self.start_requests:
+                batch.append(request)
+                
+                # 批量处理
+                if len(batch) >= self.generation_batch_size:
+                    generated = await self._process_generation_batch(batch)
+                    total_generated += generated
+                    batch = []
+                
+                # 背压检查
+                if await self._should_pause_generation():
+                    await self._wait_for_capacity()
+            
+            # 处理剩余请求
+            if batch:
+                generated = await self._process_generation_batch(batch)
+                total_generated += generated
+        
+        except Exception as e:
+            self.logger.error(f"❌ 请求生成失败: {e}")
+        
+        finally:
+            self.start_requests = None
+            self.logger.info(f"🎉 请求生成完成，总计: {total_generated}")
+    
+    async def _process_generation_batch(self, batch) -> int:
+        """处理一批请求"""
+        generated = 0
+        
+        for request in batch:
+            if not self.running:
+                break
+            
+            # 等待队列有空间
+            while await self._is_queue_full() and self.running:
+                await asyncio.sleep(0.1)
+            
+            if self.running:
+                await self.enqueue_request(request)
+                generated += 1
+                self._generation_stats['total_generated'] += 1
+            
+            # 控制生成速度
+            if self.generation_interval > 0:
+                await asyncio.sleep(self.generation_interval)
+        
+        return generated
+    
+    async def _should_pause_generation(self) -> bool:
+        """判断是否应该暂停生成"""
+        # 检查队列大小
+        if await self._is_queue_full():
+            return True
+        
+        # 检查任务管理器负载
+        if self.task_manager:
+            current_tasks = len(self.task_manager.current_task)
+            if hasattr(self.task_manager, 'semaphore'):
+                max_concurrency = getattr(self.task_manager.semaphore, '_initial_value', 8)
+                if current_tasks >= max_concurrency * self.backpressure_ratio:
+                    return True
+        
+        return False
+    
+    async def _is_queue_full(self) -> bool:
+        """检查队列是否已满"""
+        if not self.scheduler:
+            return False
+        
+        queue_size = len(self.scheduler)
+        return queue_size >= self.max_queue_size * self.backpressure_ratio
+    
+    async def _wait_for_capacity(self):
+        """等待系统有足够容量"""
+        self._generation_stats['backpressure_events'] += 1
+        self.logger.debug("⏸️ 触发背压，暂停请求生成")
+        
+        wait_time = 0.1
+        max_wait = 2.0
+        
+        while await self._should_pause_generation() and self.running:
+            await asyncio.sleep(wait_time)
+            wait_time = min(wait_time * 1.1, max_wait)
+    
+    async def _should_exit(self) -> bool:
+        """检查是否应该退出"""
+        # 没有启动请求，且所有队列都空闲
+        if (self.start_requests is None and 
+            self.scheduler.idle() and 
+            self.downloader.idle() and 
+            self.task_manager.all_done() and 
+            self.processor.idle()):
+            return True
+        
+        return False
+    
+    def get_generation_stats(self) -> dict:
+        """获取生成统计"""
+        return {
+            **self._generation_stats,
+            'queue_size': len(self.scheduler) if self.scheduler else 0,
+            'active_tasks': len(self.task_manager.current_task) if self.task_manager else 0
+        }

crawlo/core/processor.py

@@ -1,40 +1,40 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from asyncio import Queue
-from typing import Union, Optional
-
-from crawlo import Request, Item
-from crawlo.pipelines.pipeline_manager import PipelineManager
-
-
-class Processor(object):
-
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.queue: Queue = Queue()
-        self.pipelines: Optional[PipelineManager] = None
-
-    def open(self):
-        self.pipelines = PipelineManager.from_crawler(self.crawler)
-
-    async def process(self):
-        while not self.idle():
-            result = await self.queue.get()
-            if isinstance(result, Request):
-                await self.crawler.engine.enqueue_request(result)
-            else:
-                assert isinstance(result, Item)
-                await self._process_item(result)
-
-    async def _process_item(self, item):
-        await self.pipelines.process_item(item=item)
-
-    async def enqueue(self, output: Union[Request, Item]):
-        await self.queue.put(output)
-        await self.process()
-
-    def idle(self) -> bool:
-        return len(self) == 0
-
-    def __len__(self):
-        return self.queue.qsize()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from asyncio import Queue
+from typing import Union, Optional
+
+from crawlo import Request, Item
+from crawlo.pipelines.pipeline_manager import PipelineManager
+
+
+class Processor(object):
+
+    def __init__(self, crawler):
+        self.crawler = crawler
+        self.queue: Queue = Queue()
+        self.pipelines: Optional[PipelineManager] = None
+
+    def open(self):
+        self.pipelines = PipelineManager.from_crawler(self.crawler)
+
+    async def process(self):
+        while not self.idle():
+            result = await self.queue.get()
+            if isinstance(result, Request):
+                await self.crawler.engine.enqueue_request(result)
+            else:
+                assert isinstance(result, Item)
+                await self._process_item(result)
+
+    async def _process_item(self, item):
+        await self.pipelines.process_item(item=item)
+
+    async def enqueue(self, output: Union[Request, Item]):
+        await self.queue.put(output)
+        await self.process()
+
+    def idle(self) -> bool:
+        return len(self) == 0
+
+    def __len__(self):
+        return self.queue.qsize()

crawlo/core/scheduler.py

@@ -1,57 +1,166 @@
-#!/usr/bin/python
-# -*- coding:UTF-8 -*-
-from typing import Optional, Callable
-
-from crawlo.utils.log import get_logger
-from crawlo.utils.request import set_request
-from crawlo.utils.pqueue import SpiderPriorityQueue
-from crawlo.project import load_class, common_call
-
-
-class Scheduler:
-    def __init__(self, crawler, dupe_filter, stats, log_level, priority):
-        self.crawler = crawler
-        self.request_queue: Optional[SpiderPriorityQueue] = None
-
-        self.logger = get_logger(name=self.__class__.__name__, level=log_level)
-        self.stats = stats
-        self.dupe_filter = dupe_filter
-        self.priority = priority
-
-    @classmethod
-    def create_instance(cls, crawler):
-        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
-        o = cls(
-            crawler=crawler,
-            dupe_filter=filter_cls.create_instance(crawler),
-            stats=crawler.stats,
-            log_level=crawler.settings.get('LOG_LEVEL'),
-            priority=crawler.settings.get('DEPTH_PRIORITY')
-        )
-        return o
-
-    def open(self):
-        self.request_queue = SpiderPriorityQueue()
-        self.logger.info(f'requesting filter: {self.dupe_filter}')
-
-    async def next_request(self):
-        request = await self.request_queue.get()
-        return request
-
-    async def enqueue_request(self, request):
-        if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
-            self.dupe_filter.log_stats(request)
-            return False
-        set_request(request, self.priority)
-        await self.request_queue.put(request)
-        return True
-
-    def idle(self) -> bool:
-        return len(self) == 0
-
-    async def close(self):
-        if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
-            await closed()
-
-    def __len__(self):
-        return self.request_queue.qsize()
+#!/usr/bin/python
+# -*- coding:UTF-8 -*-
+from typing import Optional, Callable
+
+from crawlo.utils.log import get_logger
+from crawlo.utils.request import set_request
+from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.queue.queue_manager import QueueManager, QueueConfig
+from crawlo.project import load_class, common_call
+
+
+class Scheduler:
+    def __init__(self, crawler, dupe_filter, stats, log_level, priority):
+        self.crawler = crawler
+        self.queue_manager: Optional[QueueManager] = None
+        self.request_serializer = RequestSerializer()  # 专门处理序列化
+
+        self.logger = get_logger(name=self.__class__.__name__, level=log_level)
+        self.stats = stats
+        self.dupe_filter = dupe_filter
+        self.priority = priority
+
+    @classmethod
+    def create_instance(cls, crawler):
+        filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+        o = cls(
+            crawler=crawler,
+            dupe_filter=filter_cls.create_instance(crawler),
+            stats=crawler.stats,
+            log_level=crawler.settings.get('LOG_LEVEL'),
+            priority=crawler.settings.get('DEPTH_PRIORITY')
+        )
+        return o
+
+    async def open(self):
+        """初始化调度器和队列"""
+        try:
+            # 创建队列配置
+            queue_config = QueueConfig.from_settings(self.crawler.settings)
+            
+            # 创建队列管理器
+            self.queue_manager = QueueManager(queue_config)
+            
+            # 初始化队列
+            success = await self.queue_manager.initialize()
+            if not success:
+                raise RuntimeError("队列初始化失败")
+            
+            # 输出队列状态
+            status = self.queue_manager.get_status()
+            self.logger.info(f'队列类型: {status["type"]}, 状态: {status["health"]}')
+            self.logger.info(f'requesting filter: {self.dupe_filter}')
+        except Exception as e:
+            self.logger.error(f"❌ 调度器初始化失败: {e}")
+            raise
+
+    async def next_request(self):
+        """获取下一个请求"""
+        if not self.queue_manager:
+            return None
+            
+        request = await self.queue_manager.get()
+        
+        # 恢复 callback（从 Redis 队列取出时）
+        if request:
+            spider = getattr(self.crawler, 'spider', None)
+            request = self.request_serializer.restore_after_deserialization(request, spider)
+            
+        return request
+
+    async def enqueue_request(self, request):
+        """将请求加入队列"""
+        if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
+            self.dupe_filter.log_stats(request)
+            return False
+
+        if not self.queue_manager:
+            self.logger.error("队列管理器未初始化")
+            return False
+
+        set_request(request, self.priority)
+        
+        # 使用统一的队列接口
+        success = await self.queue_manager.put(request, priority=getattr(request, 'priority', 0))
+        
+        if success:
+            self.logger.debug(f"✅ 请求入队成功: {request.url}")
+        
+        return success
+
+    def idle(self) -> bool:
+        """检查队列是否为空"""
+        return len(self) == 0
+
+    async def close(self):
+        """关闭调度器"""
+        if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
+            await closed()
+        
+        if self.queue_manager:
+            await self.queue_manager.close()
+
+    def __len__(self):
+        """获取队列大小"""
+        if not self.queue_manager:
+            return 0
+        # 返回同步的近似值，实际大小需要异步获取
+        return 0 if self.queue_manager.empty() else 1
+
+# #!/usr/bin/python
+# # -*- coding:UTF-8 -*-
+# from typing import Optional, Callable
+#
+# from crawlo.utils.log import get_logger
+# from crawlo.utils.request import set_request
+# from crawlo.utils.pqueue import SpiderPriorityQueue
+# from crawlo.project import load_class, common_call
+#
+#
+# class Scheduler:
+#     def __init__(self, crawler, dupe_filter, stats, log_level, priority):
+#         self.crawler = crawler
+#         self.request_queue: Optional[SpiderPriorityQueue] = None
+#
+#         self.logger = get_logger(name=self.__class__.__name__, level=log_level)
+#         self.stats = stats
+#         self.dupe_filter = dupe_filter
+#         self.priority = priority
+#
+#     @classmethod
+#     def create_instance(cls, crawler):
+#         filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
+#         o = cls(
+#             crawler=crawler,
+#             dupe_filter=filter_cls.create_instance(crawler),
+#             stats=crawler.stats,
+#             log_level=crawler.settings.get('LOG_LEVEL'),
+#             priority=crawler.settings.get('DEPTH_PRIORITY')
+#         )
+#         return o
+#
+#     def open(self):
+#         self.request_queue = SpiderPriorityQueue()
+#         self.logger.info(f'requesting filter: {self.dupe_filter}')
+#
+#     async def next_request(self):
+#         request = await self.request_queue.get()
+#         return request
+#
+#     async def enqueue_request(self, request):
+#         if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
+#             self.dupe_filter.log_stats(request)
+#             return False
+#         set_request(request, self.priority)
+#         await self.request_queue.put(request)
+#         return True
+#
+#     def idle(self) -> bool:
+#         return len(self) == 0
+#
+#     async def close(self):
+#         if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
+#             await closed()
+#
+#     def __len__(self):
+#         return self.request_queue.qsize()