jettask 0.2.19__py3-none-any.whl → 0.2.20__py3-none-any.whl

jettask/{multi_namespace_consumer.py → webui/namespace_manager/multi.py}

@@ -17,9 +17,9 @@ from datetime import datetime
 # 添加项目路径
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 
-from jettask.pg_consumer import PostgreSQLConsumer
-from jettask.webui_config import PostgreSQLConfig, RedisConfig
-from jettask.core.consumer_manager import ConsumerStrategy
+from jettask.persistence import PostgreSQLConsumer
+from jettask.webui.config import PostgreSQLConfig, RedisConfig
+# ConsumerStrategy 已移除，现在只使用 HEARTBEAT 策略
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy import text
@@ -46,6 +46,7 @@ class NamespaceConsumerProcess:
         """
         self.namespace_info = namespace_info
         self.process: Optional[mp.Process] = None
+        self.shutdown_event = mp.Event()  # 用于优雅退出的事件
         
     def start(self):
         """启动进程"""
@@ -65,12 +66,20 @@ class NamespaceConsumerProcess:
         """停止进程"""
         if self.process and self.process.is_alive():
             logger.info(f"停止命名空间 {self.namespace_info['name']} 的消费进程")
-            self.process.terminate()
-            self.process.join(timeout=10)
+            # 先发送优雅退出信号
+            self.shutdown_event.set()
+            # 等待进程正常退出（7秒：5秒清理超时 + 2秒缓冲）
+            self.process.join(timeout=7)
             if self.process.is_alive():
-                logger.warning(f"强制停止命名空间 {self.namespace_info['name']} 的消费进程")
+                logger.warning(f"等待优雅退出超时（7秒），发送 SIGTERM 信号")
+                self.process.terminate()
+                self.process.join(timeout=3)
+            if self.process.is_alive():
+                logger.warning(f"SIGTERM 超时（3秒），强制 SIGKILL")
                 self.process.kill()
-                self.process.join()
+                self.process.join(timeout=2)
+            if self.process.is_alive():
+                logger.error(f"SIGKILL 后进程仍然存活，可能存在严重问题")
                 
     def is_alive(self) -> bool:
         """检查进程是否存活"""
@@ -81,25 +90,42 @@ class NamespaceConsumerProcess:
         # 设置信号处理
         signal.signal(signal.SIGTERM, self._signal_handler)
         signal.signal(signal.SIGINT, self._signal_handler)
-        
+
         # 创建新的事件循环
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        
+
+        consumer = None
         try:
-            loop.run_until_complete(self._async_run())
+            # 传递 consumer 引用以便在信号处理中使用
+            consumer = loop.run_until_complete(self._async_run())
         except KeyboardInterrupt:
             logger.info(f"命名空间 {self.namespace_info['name']} 的消费进程收到中断信号")
         except Exception as e:
             logger.error(f"命名空间 {self.namespace_info['name']} 的消费进程异常退出: {e}", exc_info=True)
         finally:
+            # 确保执行清理（带超时）
+            if consumer:
+                try:
+                    logger.info(f"执行命名空间 {self.namespace_info['name']} 的消费者清理")
+                    # 创建一个带超时的任务
+                    async def cleanup_with_timeout():
+                        try:
+                            await asyncio.wait_for(consumer.stop(), timeout=5.0)
+                        except asyncio.TimeoutError:
+                            logger.warning(f"外层清理超时（5秒），强制退出")
+
+                    loop.run_until_complete(cleanup_with_timeout())
+                except Exception as e:
+                    logger.error(f"清理消费者时出错: {e}")
             loop.close()
             
     async def _async_run(self):
         """异步运行消费者"""
         namespace_name = self.namespace_info.get('name', 'unknown')
         logger.info(f"开始初始化命名空间 {namespace_name} 的消费者")
-        
+
+        consumer = None
         try:
             # 创建配置
             pg_config_data = self.namespace_info.get('pg_config', {})
@@ -131,7 +157,8 @@ class NamespaceConsumerProcess:
                 filtered_config = {k: v for k, v in redis_config_data.items() if k in valid_fields}
                 redis_config = RedisConfig(**filtered_config)
             
-            
+            print(f'{pg_config=}')
+            print(f'{redis_config=}')
             # 创建消费者实例
             consumer = PostgreSQLConsumer(
                 pg_config=pg_config,
@@ -139,7 +166,7 @@ class NamespaceConsumerProcess:
                 prefix=self.namespace_info.get('redis_prefix', 'jettask'),
                 namespace_id=self.namespace_info.get('id'),
                 namespace_name=self.namespace_info.get('name'),
-                consumer_strategy=ConsumerStrategy.HEARTBEAT
+                # consumer_strategy 参数已移除，现在只使用 HEARTBEAT 策略
             )
             
             logger.info(f"命名空间 {namespace_name} 的消费者实例创建成功，准备启动")
@@ -147,25 +174,45 @@ class NamespaceConsumerProcess:
             # 启动消费者
             await consumer.start()
             logger.info(f"命名空间 {namespace_name} 的消费者已启动，进入运行状态")
-            
+
             # 保持运行直到收到停止信号
             try:
-                while True:
-                    await asyncio.sleep(10)  # 每10秒检查一次
-                    # 可以在这里添加健康检查逻辑
+                while not self.shutdown_event.is_set():
+                    await asyncio.sleep(1)  # 每秒检查一次退出信号
             except asyncio.CancelledError:
                 logger.info(f"命名空间 {namespace_name} 的消费者收到取消信号")
-                await consumer.stop()
-                raise
-                
+            finally:
+                # 确保清理（带超时）
+                logger.info(f"命名空间 {namespace_name} 开始执行优雅退出")
+                try:
+                    # 设置 5 秒超时，避免因网络问题导致清理卡住
+                    await asyncio.wait_for(consumer.stop(), timeout=5.0)
+                    logger.info(f"命名空间 {namespace_name} 清理完成")
+                except asyncio.TimeoutError:
+                    logger.warning(f"命名空间 {namespace_name} 清理超时（5秒），强制退出")
+                except Exception as e:
+                    logger.error(f"命名空间 {namespace_name} 清理失败: {e}")
+
         except Exception as e:
             logger.error(f"命名空间 {namespace_name} 的消费者启动失败: {e}", exc_info=True)
+            # 确保清理（带超时）
+            if consumer:
+                try:
+                    await asyncio.wait_for(consumer.stop(), timeout=5.0)
+                except asyncio.TimeoutError:
+                    logger.warning(f"清理失败的消费者超时，跳过")
+                except Exception as cleanup_error:
+                    logger.error(f"清理失败的消费者时出错: {cleanup_error}")
             raise
+        finally:
+            # 返回 consumer 实例以便在外层 finally 中使用
+            return consumer
             
     def _signal_handler(self, signum, frame):
-        """信号处理器"""
+        """信号处理器 - 触发优雅退出"""
         logger.info(f"命名空间 {self.namespace_info['name']} 的消费进程收到信号 {signum}")
-        sys.exit(0)
+        # 设置退出事件，让主循环优雅退出
+        self.shutdown_event.set()
 
 
 class MultiNamespaceConsumerManager:

jettask/{unified_consumer_manager.py → webui/namespace_manager/unified.py}

@@ -7,7 +7,7 @@ import logging
 import multiprocessing
 from typing import Dict, Optional, Set
 from jettask.core.unified_manager_base import UnifiedManagerBase
-from .multi_namespace_consumer import NamespaceConsumerProcess
+from .multi import NamespaceConsumerProcess
 
 logger = logging.getLogger(__name__)
 

jettask/{run.py → webui/run.py}

@@ -11,7 +11,7 @@ import sys
 import os
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 
-from jettask.api import app, monitor
+from jettask.webui.app import app, monitor
 import uvicorn
 
 def main():
@@ -35,7 +35,7 @@ def main():
     
     # 运行服务器
     uvicorn.run(
-        "jettask.webui.api:app",
+        "jettask.webui.app:app",
         host=args.host,
         port=args.port,
         log_level="info",

jettask/{services → webui/services}/__init__.py

@@ -10,15 +10,13 @@ from .alert_service import AlertService
 from .analytics_service import AnalyticsService
 from .settings_service import SettingsService
 from .task_service import TaskService
-from .redis_monitor_service import RedisMonitorService
 
 __all__ = [
     'OverviewService',
-    'QueueService', 
+    'QueueService',
     'ScheduledTaskService',
     'AlertService',
     'AnalyticsService',
     'SettingsService',
     'TaskService',
-    'RedisMonitorService'
 ]

jettask/{services → webui/services}/overview_service.py

@@ -11,7 +11,7 @@ from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from jettask.schemas import TimeRangeQuery
-from jettask.core.db_manager import get_db_manager
+from jettask.persistence.db_manager import get_db_manager
 
 logger = logging.getLogger(__name__)
 
@@ -84,26 +84,44 @@ class OverviewService:
                 'workers': 0
             }
             
+            # 使用注册管理器进行统计，避免 scan
+            from jettask.messaging.registry import QueueRegistry
+            from jettask.task.task_registry import TaskRegistry
+            from jettask.worker.manager import WorkerState as WorkerRegistry
+
+            queue_registry = QueueRegistry(
+                redis_client=None,
+                async_redis_client=redis_client,
+                redis_prefix=namespace
+            )
+
+            task_registry = TaskRegistry(
+                redis_client=None,
+                async_redis_client=redis_client,
+                redis_prefix=namespace
+            )
+
+            worker_registry = WorkerRegistry(
+                redis_client=None,
+                async_redis_client=redis_client,
+                redis_prefix=namespace
+            )
+
             # 统计队列数量
-            queue_pattern = f"{namespace}:QUEUE:*"
-            async for _ in redis_client.scan_iter(match=queue_pattern):
-                stats['queues'] += 1
-            
+            stats['queues'] = await queue_registry.get_queue_count()
+
             # 统计任务数量
-            task_pattern = f"{namespace}:TASK:*"
-            async for _ in redis_client.scan_iter(match=task_pattern):
-                stats['tasks'] += 1
-            
+            stats['tasks'] = await task_registry.get_task_count_from_redis()
+
             # 统计延迟任务数量
-            delayed_pattern = f"{namespace}:DELAYED_QUEUE:*"
-            async for key in redis_client.scan_iter(match=delayed_pattern):
-                count = await redis_client.zcard(key)
+            delayed_queues = await queue_registry.get_all_delayed_queues()
+            for queue in delayed_queues:
+                delayed_key = f"{namespace}:DELAYED_QUEUE:{queue}"
+                count = await redis_client.zcard(delayed_key)
                 stats['delayed_tasks'] += count
-            
+
             # 统计工作进程数量
-            worker_pattern = f"{namespace}:WORKER:*"
-            async for _ in redis_client.scan_iter(match=worker_pattern):
-                stats['workers'] += 1
+            stats['workers'] = await worker_registry.get_worker_count()
             
             return stats
             

jettask/{services → webui/services}/queue_service.py

@@ -226,7 +226,7 @@ class QueueService:
         
         try:
             # 导入 QueueStatsV2
-            from jettask.backend.queue_stats_v2 import QueueStatsV2
+            from jettask.webui.services.queue_stats_v2 import QueueStatsV2
             
             # 创建统计服务实例
             stats_service = QueueStatsV2(

jettask/{backend → webui/services}/queue_stats_v2.py

@@ -9,7 +9,7 @@ from collections import defaultdict
 import redis.asyncio as redis
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
-from jettask.constants import is_internal_consumer
+from jettask.config.constants import is_internal_consumer
 
 logger = logging.getLogger(__name__)
 

jettask/{services → webui/services}/settings_service.py

@@ -9,7 +9,7 @@ from datetime import datetime
 from sqlalchemy import text
 from urllib.parse import urlparse
 
-from jettask.core.db_manager import get_db_manager
+from jettask.persistence.db_manager import get_db_manager
 from jettask.schemas import (
     ConfigMode,
     NamespaceCreate,

jettask/worker/__init__.py

@@ -0,0 +1,53 @@
+"""
+Worker 管理模块
+
+提供 Worker 的创建、生命周期管理、状态监控等功能。
+
+推荐使用新接口:
+    from jettask.worker import WorkerManager
+
+    manager = WorkerManager(redis, async_redis, 'jettask')
+    worker_id = await manager.start_worker('MyApp', ['queue1'])
+"""
+
+# 主要接口
+from .manager import (
+    WorkerManager,
+    WorkerState,       # Worker 状态管理（之前叫 WorkerRegistry）
+    ConsumerManager,
+    WorkerNaming,
+)
+
+# 生命周期和状态管理
+from .lifecycle import (
+    WorkerLifecycle,
+    WorkerStateManager,
+    HeartbeatThreadManager,
+    WorkerScanner,
+    HeartbeatConsumerStrategy,  # 兼容性
+)
+
+# 恢复
+from .recovery import OfflineWorkerRecovery
+
+# 兼容性别名：WorkerRegistry -> WorkerState
+WorkerRegistry = WorkerState
+
+__all__ = [
+    # 主要接口
+    'WorkerManager',
+    'WorkerState',        # 新名称（推荐使用）
+    'WorkerRegistry',     # 兼容性别名
+    'ConsumerManager',
+    'WorkerNaming',
+
+    # 生命周期和状态管理
+    'WorkerLifecycle',
+    'WorkerStateManager',
+    'HeartbeatThreadManager',
+    'WorkerScanner',
+    'HeartbeatConsumerStrategy',  # 兼容性
+
+    # 恢复
+    'OfflineWorkerRecovery',
+]