crawlo 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

crawlo/core/engine.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 # -*- coding:UTF-8 -*-
 import asyncio
+import time
 from inspect import iscoroutine
 from typing import Optional, Generator, Callable
 
@@ -31,6 +32,20 @@ class Engine(object):
         self.start_requests: Optional[Generator] = None
         self.task_manager: Optional[TaskManager] = TaskManager(self.settings.get_int('CONCURRENCY'))
 
+        # 增强控制参数
+        self.max_queue_size = self.settings.get_int('SCHEDULER_MAX_QUEUE_SIZE', 200)
+        self.generation_batch_size = self.settings.get_int('REQUEST_GENERATION_BATCH_SIZE', 10)
+        self.generation_interval = self.settings.get_float('REQUEST_GENERATION_INTERVAL', 0.05)
+        self.backpressure_ratio = self.settings.get_float('BACKPRESSURE_RATIO', 0.8)  # 队列达到80%时启动背压
+        
+        # 状态跟踪
+        self._generation_paused = False
+        self._last_generation_time = 0
+        self._generation_stats = {
+            'total_generated': 0,
+            'backpressure_events': 0
+        }
+
         self.logger = get_logger(name=self.__class__.__name__)
 
     def _get_downloader_cls(self):
@@ -64,16 +79,25 @@ class Engine(object):
 
         self.scheduler = Scheduler.create_instance(self.crawler)
         if hasattr(self.scheduler, 'open'):
-            await self.scheduler.open()
+            if asyncio.iscoroutinefunction(self.scheduler.open):
+                await self.scheduler.open()
+            else:
+                self.scheduler.open()
 
         downloader_cls = self._get_downloader_cls()
         self.downloader = downloader_cls(self.crawler)
         if hasattr(self.downloader, 'open'):
-            self.downloader.open()
+            if asyncio.iscoroutinefunction(self.downloader.open):
+                await self.downloader.open()
+            else:
+                self.downloader.open()
 
         self.processor = Processor(self.crawler)
         if hasattr(self.processor, 'open'):
-            self.processor.open()
+            if asyncio.iscoroutinefunction(self.processor.open):
+                await self.processor.open()
+            else:
+                self.processor.open()
 
         self.start_requests = iter(spider.start_requests())
         await self._open_spider()
@@ -81,14 +105,57 @@ class Engine(object):
     async def crawl(self):
         """
         Crawl the spider
+        增强版本支持智能请求生成和背压控制
         """
+        generation_task = None
+        
+        try:
+            # 启动请求生成任务（如果启用了受控生成）
+            if (self.start_requests and 
+                self.settings.get_bool('ENABLE_CONTROLLED_REQUEST_GENERATION', False)):
+                generation_task = asyncio.create_task(
+                    self._controlled_request_generation()
+                )
+            else:
+                # 传统方式处理启动请求
+                generation_task = asyncio.create_task(
+                    self._traditional_request_generation()
+                )
+            
+            # 主爬取循环
+            while self.running:
+                # 获取并处理请求
+                if request := await self._get_next_request():
+                    await self._crawl(request)
+                
+                # 检查退出条件
+                if await self._should_exit():
+                    break
+                
+                # 短暂休息避免忙等
+                await asyncio.sleep(0.001)
+        
+        finally:
+            # 清理生成任务
+            if generation_task and not generation_task.done():
+                generation_task.cancel()
+                try:
+                    await generation_task
+                except asyncio.CancelledError:
+                    pass
+            
+            await self.close_spider()
+
+    async def _traditional_request_generation(self):
+        """传统的请求生成方式（兼容旧版本）"""
         while self.running:
-            if request := await self._get_next_request():
-                await self._crawl(request)
             try:
                 start_request = next(self.start_requests)
+                # 请求入队
+                await self.enqueue_request(start_request)
             except StopIteration:
                 self.start_requests = None
+                break
             except Exception as exp:
                 # 1、发去请求的request全部运行完毕
                 # 2、调度器是否空闲
@@ -98,12 +165,99 @@ class Engine(object):
                 self.running = False
                 if self.start_requests is not None:
                     self.logger.error(f"启动请求时发生错误: {str(exp)}")
-            else:
-                # 请求入队
-                await self.enqueue_request(start_request)
+            await asyncio.sleep(0.001)
 
-        if not self.running:
-            await self.close_spider()
+    async def _controlled_request_generation(self):
+        """受控的请求生成（增强功能）"""
+        self.logger.info("🎛️ 启动受控请求生成")
+        
+        batch = []
+        total_generated = 0
+        
+        try:
+            for request in self.start_requests:
+                batch.append(request)
+                
+                # 批量处理
+                if len(batch) >= self.generation_batch_size:
+                    generated = await self._process_generation_batch(batch)
+                    total_generated += generated
+                    batch = []
+                
+                # 背压检查
+                if await self._should_pause_generation():
+                    await self._wait_for_capacity()
+            
+            # 处理剩余请求
+            if batch:
+                generated = await self._process_generation_batch(batch)
+                total_generated += generated
+        
+        except Exception as e:
+            self.logger.error(f"❌ 请求生成失败: {e}")
+        
+        finally:
+            self.start_requests = None
+            self.logger.info(f"🎉 请求生成完成，总计: {total_generated}")
+
+    async def _process_generation_batch(self, batch) -> int:
+        """处理一批请求"""
+        generated = 0
+        
+        for request in batch:
+            if not self.running:
+                break
+            
+            # 等待队列有空间
+            while await self._is_queue_full() and self.running:
+                await asyncio.sleep(0.1)
+            
+            if self.running:
+                await self.enqueue_request(request)
+                generated += 1
+                self._generation_stats['total_generated'] += 1
+            
+            # 控制生成速度
+            if self.generation_interval > 0:
+                await asyncio.sleep(self.generation_interval)
+        
+        return generated
+
+    async def _should_pause_generation(self) -> bool:
+        """判断是否应该暂停生成"""
+        # 检查队列大小
+        if await self._is_queue_full():
+            return True
+        
+        # 检查任务管理器负载
+        if self.task_manager:
+            current_tasks = len(self.task_manager.current_task)
+            if hasattr(self.task_manager, 'semaphore'):
+                max_concurrency = getattr(self.task_manager.semaphore, '_initial_value', 8)
+                if current_tasks >= max_concurrency * self.backpressure_ratio:
+                    return True
+        
+        return False
+
+    async def _is_queue_full(self) -> bool:
+        """检查队列是否已满"""
+        if not self.scheduler:
+            return False
+        
+        queue_size = len(self.scheduler)
+        return queue_size >= self.max_queue_size * self.backpressure_ratio
+
+    async def _wait_for_capacity(self):
+        """等待系统有足够容量"""
+        self._generation_stats['backpressure_events'] += 1
+        self.logger.debug("⏸️ 触发背压，暂停请求生成")
+        
+        wait_time = 0.1
+        max_wait = 2.0
+        
+        while await self._should_pause_generation() and self.running:
+            await asyncio.sleep(wait_time)
+            wait_time = min(wait_time * 1.1, max_wait)
 
     async def _open_spider(self):
         asyncio.create_task(self.crawler.subscriber.notify(spider_opened))
@@ -164,9 +318,29 @@ class Engine(object):
             return True
         return False
 
+    async def _should_exit(self) -> bool:
+        """检查是否应该退出（增强版本）"""
+        # 没有启动请求，且所有队列都空闲
+        if (self.start_requests is None and 
+            self.scheduler.idle() and 
+            self.downloader.idle() and 
+            self.task_manager.all_done() and 
+            self.processor.idle()):
+            return True
+        
+        return False
+
     async def close_spider(self):
         await asyncio.gather(*self.task_manager.current_task)
         await self.scheduler.close()
         await self.downloader.close()
         if self.normal:
-            await self.crawler.close()
+            await self.crawler.close()
+    
+    def get_generation_stats(self) -> dict:
+        """获取生成统计"""
+        return {
+            **self._generation_stats,
+            'queue_size': len(self.scheduler) if self.scheduler else 0,
+            'active_tasks': len(self.task_manager.current_task) if self.task_manager else 0
+        }

crawlo/core/scheduler.py

@@ -5,6 +5,7 @@ from typing import Optional, Callable
 from crawlo.utils.log import get_logger
 from crawlo.utils.request import set_request
 from crawlo.utils.request_serializer import RequestSerializer
+from crawlo.utils.error_handler import ErrorHandler
 from crawlo.queue.queue_manager import QueueManager, QueueConfig
 from crawlo.project import load_class, common_call
 
@@ -16,6 +17,7 @@ class Scheduler:
         self.request_serializer = RequestSerializer()  # 专门处理序列化
 
         self.logger = get_logger(name=self.__class__.__name__, level=log_level)
+        self.error_handler = ErrorHandler(self.__class__.__name__, log_level)
         self.stats = stats
         self.dupe_filter = dupe_filter
         self.priority = priority
@@ -34,6 +36,7 @@ class Scheduler:
 
     async def open(self):
         """初始化调度器和队列"""
+        self.logger.info("开始初始化调度器...")
         try:
             # 创建队列配置
             queue_config = QueueConfig.from_settings(self.crawler.settings)
@@ -42,6 +45,7 @@ class Scheduler:
             self.queue_manager = QueueManager(queue_config)
             
             # 初始化队列
+            self.logger.info("开始初始化队列管理器...")
             success = await self.queue_manager.initialize()
             if not success:
                 raise RuntimeError("队列初始化失败")
@@ -50,8 +54,10 @@ class Scheduler:
             status = self.queue_manager.get_status()
             self.logger.info(f'队列类型: {status["type"]}, 状态: {status["health"]}')
             self.logger.info(f'requesting filter: {self.dupe_filter}')
+            self.logger.info("调度器初始化完成")
         except Exception as e:
             self.logger.error(f"❌ 调度器初始化失败: {e}")
+            self.logger.debug(f"详细错误信息:\n{traceback.format_exc()}")
             raise
 
     async def next_request(self):
@@ -59,14 +65,22 @@ class Scheduler:
         if not self.queue_manager:
             return None
             
-        request = await self.queue_manager.get()
-        
-        # 恢复 callback（从 Redis 队列取出时）
-        if request:
-            spider = getattr(self.crawler, 'spider', None)
-            request = self.request_serializer.restore_after_deserialization(request, spider)
+        try:
+            request = await self.queue_manager.get()
+            
+            # 恢复 callback（从 Redis 队列取出时）
+            if request:
+                spider = getattr(self.crawler, 'spider', None)
+                request = self.request_serializer.restore_after_deserialization(request, spider)
             
-        return request
+            return request
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="获取下一个请求失败", 
+                raise_error=False
+            )
+            return None
 
     async def enqueue_request(self, request):
         """将请求加入队列"""
@@ -80,13 +94,21 @@ class Scheduler:
 
         set_request(request, self.priority)
         
-        # 使用统一的队列接口
-        success = await self.queue_manager.put(request, priority=getattr(request, 'priority', 0))
-        
-        if success:
-            self.logger.debug(f"✅ 请求入队成功: {request.url}")
-        
-        return success
+        try:
+            # 使用统一的队列接口
+            success = await self.queue_manager.put(request, priority=getattr(request, 'priority', 0))
+            
+            if success:
+                self.logger.debug(f"✅ 请求入队成功: {request.url}")
+            
+            return success
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="请求入队失败", 
+                raise_error=False
+            )
+            return False
 
     def idle(self) -> bool:
         """检查队列是否为空"""
@@ -94,73 +116,22 @@ class Scheduler:
 
     async def close(self):
         """关闭调度器"""
-        if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
-            await closed()
-        
-        if self.queue_manager:
-            await self.queue_manager.close()
+        try:
+            if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
+                await closed()
+            
+            if self.queue_manager:
+                await self.queue_manager.close()
+        except Exception as e:
+            self.error_handler.handle_error(
+                e, 
+                context="关闭调度器失败", 
+                raise_error=False
+            )
 
     def __len__(self):
         """获取队列大小"""
         if not self.queue_manager:
             return 0
         # 返回同步的近似值，实际大小需要异步获取
-        return 0 if self.queue_manager.empty() else 1
-
-# #!/usr/bin/python
-# # -*- coding:UTF-8 -*-
-# from typing import Optional, Callable
-#
-# from crawlo.utils.log import get_logger
-# from crawlo.utils.request import set_request
-# from crawlo.utils.pqueue import SpiderPriorityQueue
-# from crawlo.project import load_class, common_call
-#
-#
-# class Scheduler:
-#     def __init__(self, crawler, dupe_filter, stats, log_level, priority):
-#         self.crawler = crawler
-#         self.request_queue: Optional[SpiderPriorityQueue] = None
-#
-#         self.logger = get_logger(name=self.__class__.__name__, level=log_level)
-#         self.stats = stats
-#         self.dupe_filter = dupe_filter
-#         self.priority = priority
-#
-#     @classmethod
-#     def create_instance(cls, crawler):
-#         filter_cls = load_class(crawler.settings.get('FILTER_CLASS'))
-#         o = cls(
-#             crawler=crawler,
-#             dupe_filter=filter_cls.create_instance(crawler),
-#             stats=crawler.stats,
-#             log_level=crawler.settings.get('LOG_LEVEL'),
-#             priority=crawler.settings.get('DEPTH_PRIORITY')
-#         )
-#         return o
-#
-#     def open(self):
-#         self.request_queue = SpiderPriorityQueue()
-#         self.logger.info(f'requesting filter: {self.dupe_filter}')
-#
-#     async def next_request(self):
-#         request = await self.request_queue.get()
-#         return request
-#
-#     async def enqueue_request(self, request):
-#         if not request.dont_filter and await common_call(self.dupe_filter.requested, request):
-#             self.dupe_filter.log_stats(request)
-#             return False
-#         set_request(request, self.priority)
-#         await self.request_queue.put(request)
-#         return True
-#
-#     def idle(self) -> bool:
-#         return len(self) == 0
-#
-#     async def close(self):
-#         if isinstance(closed := getattr(self.dupe_filter, 'closed', None), Callable):
-#             await closed()
-#
-#     def __len__(self):
-#         return self.request_queue.qsize()
+        return 0 if self.queue_manager.empty() else 1

crawlo/crawler.py

@@ -139,7 +139,7 @@ class Crawler:
 
     async def crawl(self):
         """
-        启动爬虫核心流程（增强版）
+        启动爬虫核心流程
         
         包含以下阶段:
         1. 初始化阶段: 创建所有组件
@@ -391,7 +391,7 @@ class Crawler:
 
 class CrawlerProcess:
     """
-    爬虫进程管理器（增强版）
+    爬虫进程管理器
     
     支持功能:
     - 多爬虫并发调度和资源管理
@@ -586,7 +586,7 @@ class CrawlerProcess:
 
     async def crawl(self, spiders: Union[Type[Spider], str, List[Union[Type[Spider], str]]]):
         """
-        启动一个或多个爬虫（增强版）
+        启动一个或多个爬虫
         
         增强功能:
         - 智能并发控制
@@ -856,7 +856,7 @@ class CrawlerProcess:
 
     def _shutdown(self, _signum, _frame):
         """
-        优雅关闭信号处理（增强版）
+        优雅关闭信号处理
         
         提供更好的关闭体验和资源清理
         """
@@ -881,7 +881,7 @@ class CrawlerProcess:
 
     async def _wait_for_shutdown(self):
         """
-        等待所有活跃任务完成（增强版）
+        等待所有活跃任务完成
         
         提供更好的关闭时间控制和进度反馈
         """
@@ -935,7 +935,7 @@ class CrawlerProcess:
     @classmethod
     def _get_default_settings(cls) -> SettingManager:
         """
-        加载默认配置（增强版）
+        加载默认配置
         
         提供更好的错误处理和降级策略
         """

crawlo/downloader/__init__.py

@@ -209,6 +209,21 @@ try:
 except ImportError:
     HttpXDownloader = None
 
+try:
+    from .selenium_downloader import SeleniumDownloader
+except ImportError:
+    SeleniumDownloader = None
+
+try:
+    from .playwright_downloader import PlaywrightDownloader
+except ImportError:
+    PlaywrightDownloader = None
+
+try:
+    from .hybrid_downloader import HybridDownloader
+except ImportError:
+    HybridDownloader = None
+
 # 导出所有可用的类
 __all__ = [
     'DownloaderBase',
@@ -223,6 +238,12 @@ if CurlCffiDownloader:
     __all__.append('CurlCffiDownloader')
 if HttpXDownloader:
     __all__.append('HttpXDownloader')
+if SeleniumDownloader:
+    __all__.append('SeleniumDownloader')
+if PlaywrightDownloader:
+    __all__.append('PlaywrightDownloader')
+if HybridDownloader:
+    __all__.append('HybridDownloader')
 
 # 提供便捷的下载器映射
 DOWNLOADER_MAP = {
@@ -230,6 +251,9 @@ DOWNLOADER_MAP = {
     'httpx': HttpXDownloader, 
     'curl_cffi': CurlCffiDownloader,
     'cffi': CurlCffiDownloader,  # 别名
+    'selenium': SeleniumDownloader,
+    'playwright': PlaywrightDownloader,
+    'hybrid': HybridDownloader,
 }
 
 # 过滤掉不可用的下载器

crawlo/downloader/aiohttp_downloader.py

@@ -162,6 +162,14 @@ class AioHttpDownloader(DownloaderBase):
             except Exception as e:
                 raise ValueError(f"Invalid proxy URL: {proxy}") from e
 
+        # 处理通过meta传递的代理认证信息
+        meta_proxy_auth = request.meta.get("proxy_auth")
+        if meta_proxy_auth and isinstance(meta_proxy_auth, dict):
+            username = meta_proxy_auth.get("username")
+            password = meta_proxy_auth.get("password")
+            if username and password:
+                kwargs["proxy_auth"] = BasicAuth(username, password)
+
         # === 处理请求体 ===
         if hasattr(request, "_json_body") and request._json_body is not None:
             kwargs["json"] = request._json_body

crawlo/downloader/cffi_downloader.py

@@ -210,6 +210,11 @@ class CurlCffiDownloader(DownloaderBase):
             else:
                 self.logger.error(f"不支持的 proxy 类型: {type(proxy)}，值: {proxy}")
 
+        # 处理通过meta传递的代理认证信息
+        proxy_auth_header = request.headers.get("Proxy-Authorization") or request.meta.get("proxy_auth_header")
+        if proxy_auth_header:
+            kwargs["headers"]["Proxy-Authorization"] = proxy_auth_header
+
         # 请求体处理
         if hasattr(request, "_json_body") and request._json_body is not None:
             kwargs["json"] = request._json_body