crawlo 1.1.9__py3-none-any.whl → 1.2.0__py3-none-any.whl

crawlo/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.9"
+__version__ = "1.2.0"

crawlo/core/engine.py

@@ -321,12 +321,21 @@ class Engine(object):
     async def _should_exit(self) -> bool:
         """检查是否应该退出（增强版本）"""
         # 没有启动请求，且所有队列都空闲
-        if (self.start_requests is None and 
-            self.scheduler.idle() and 
-            self.downloader.idle() and 
-            self.task_manager.all_done() and 
-            self.processor.idle()):
-            return True
+        if self.start_requests is None:
+            # 使用异步的idle检查方法以获得更精确的结果
+            scheduler_idle = await self.scheduler.async_idle() if hasattr(self.scheduler, 'async_idle') else self.scheduler.idle()
+            
+            if (scheduler_idle and 
+                self.downloader.idle() and 
+                self.task_manager.all_done() and 
+                self.processor.idle()):
+                # 增加额外检查确保所有任务都完成
+                await asyncio.sleep(0.1)  # 短暂等待确保没有新的任务加入
+                if (await self.scheduler.async_idle() and 
+                    self.downloader.idle() and 
+                    self.task_manager.all_done() and 
+                    self.processor.idle()):
+                    return True
         
         return False
 

crawlo/core/scheduler.py

@@ -114,6 +114,13 @@ class Scheduler:
         """检查队列是否为空"""
         return len(self) == 0
 
+    async def async_idle(self) -> bool:
+        """异步检查队列是否为空（更精确）"""
+        if not self.queue_manager:
+            return True
+        # 使用队列管理器的异步empty方法
+        return await self.queue_manager.async_empty()
+
     async def close(self):
         """关闭调度器"""
         try:

crawlo/queue/queue_manager.py

@@ -198,13 +198,28 @@ class QueueManager:
             return 0
     
     def empty(self) -> bool:
-        """检查队列是否为空"""
+        """检查队列是否为空（同步版本，用于兼容性）"""
         try:
             # 对于内存队列，可以同步检查
             if self._queue_type == QueueType.MEMORY:
                 return self._queue.qsize() == 0
-            # 对于 Redis 队列，需要异步操作，这里返回近似值
-            return False
+            # 对于 Redis 队列，由于需要异步操作，这里返回近似值
+            # 为了确保程序能正常退出，我们返回True，让上层通过更精确的异步检查来判断
+            return True
+        except Exception:
+            return True
+    
+    async def async_empty(self) -> bool:
+        """检查队列是否为空（异步版本，更精确）"""
+        try:
+            # 对于内存队列
+            if self._queue_type == QueueType.MEMORY:
+                return self._queue.qsize() == 0
+            # 对于 Redis 队列，使用异步检查
+            elif self._queue_type == QueueType.REDIS:
+                size = await self.size()
+                return size == 0
+            return True
         except Exception:
             return True
     
@@ -261,27 +276,15 @@ class QueueManager:
     async def _create_queue(self, queue_type: QueueType):
         """创建队列实例"""
         if queue_type == QueueType.REDIS:
-            # 从队列名称中提取项目名称，用于module_name
-            # 例如：crawlo:books_distributed:queue:requests -> books_distributed
+            # 简化项目名称提取逻辑
             project_name = "default"
             if ':' in self.config.queue_name:
                 parts = self.config.queue_name.split(':')
-                if len(parts) >= 2:
-                    # 处理可能的双重 crawlo 前缀
-                    if parts[0] == "crawlo" and parts[1] == "crawlo":
-                        # 双重 crawlo 前缀，取第三个部分作为项目名称
-                        if len(parts) >= 3:
-                            project_name = parts[2]
-                        else:
-                            project_name = "default"
-                    elif parts[0] == "crawlo":
-                        # 正常的 crawlo 前缀，取第二个部分作为项目名称
-                        project_name = parts[1]
-                    else:
-                        # 没有 crawlo 前缀，使用第一个部分作为项目名称
-                        project_name = parts[0]
-                else:
-                    project_name = self.config.queue_name or "default"
+                # 跳过所有"crawlo"前缀，取第一个非"crawlo"部分作为项目名称
+                for part in parts:
+                    if part != "crawlo":
+                        project_name = part
+                        break
             else:
                 project_name = self.config.queue_name or "default"
             

crawlo/settings/default_settings.py

@@ -45,7 +45,7 @@ SCHEDULER_MAX_QUEUE_SIZE = 1000
 SCHEDULER_QUEUE_NAME = f"crawlo:{PROJECT_NAME}:queue:requests"
 
 # 队列类型：memory/redis/auto
-QUEUE_TYPE = 'memory'
+QUEUE_TYPE = 'auto'
 
 # 默认去重管道（根据运行模式自动选择）
 DEFAULT_DEDUP_PIPELINE = 'crawlo.pipelines.redis_dedup_pipeline.RedisDedupPipeline'