jettask 0.2.14__py3-none-any.whl → 0.2.16__py3-none-any.whl

jettask/config/nacos_config.py

@@ -0,0 +1,386 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+配置管理模块
+使用 Nacos 作为配置中心
+支持定时刷新和配置变更监听
+"""
+import os
+import json
+import yaml
+import threading
+from datetime import datetime
+from typing import Dict, Any, Callable, List
+from dotenv import load_dotenv
+from nacos import NacosClient
+import logging
+
+
+
+logger = logging.getLogger(__name__)
+
+
+class Config:
+    """配置管理类，支持定时刷新和配置监听"""
+    
+    def __init__(self, refresh_interval: int = 30):
+        """初始化配置管理器
+        
+        Args:
+            refresh_interval: 配置刷新间隔（秒），默认30秒
+        """
+        self.nacos_config = None 
+        
+        self.nacos_group = None
+        self.nacos_data_id = None
+        self.local_dev_mode = None
+        # 服务注册信息
+        self.service_info = None
+        
+        # 刷新相关
+        self.refresh_interval = refresh_interval
+        self.refresh_thread = None
+        self.stop_refresh = threading.Event()
+        self.last_refresh_time = None
+        self.config_version = 0
+        
+        # 配置变更监听器
+        self.change_listeners: List[Callable[[Dict[str, Any]], None]] = []
+        
+        # Nacos客户端（保持长连接）
+        self.nacos_client = None
+        
+        # 配置锁，保证线程安全
+        self._config_lock = threading.RLock()
+        
+        # 初始化配置
+        self._init_config()
+        
+        self._config = self._load_config()
+        
+        
+        # 启动定时刷新
+        if not self.local_dev_mode and refresh_interval > 0:
+            self.start_refresh_thread()
+    
+    @property
+    def config(self) -> dict:
+        """获取当前配置（线程安全）"""
+        with self._config_lock:
+            if not hasattr(self, '_config'):
+                self._config = self._load_config()
+            return self._config
+    
+    def _init_config(self):
+        """初始化配置"""
+        try:
+            # 加载 .env 文件
+            load_dotenv()
+            self.nacos_config = {
+                'server_addresses': os.getenv('NACOS_SERVER', '127.0.0.1:8848'),
+                'namespace': os.getenv('NACOS_NAMESPACE', 'tamar_console_dev'),
+                'username': os.getenv('NACOS_USERNAME', 'nacos'),
+                'password': os.getenv('NACOS_PASSWORD', 'nacos')
+            }
+            
+            self.nacos_group = os.getenv('NACOS_GROUP', 'DEFAULT_GROUP')
+            self.nacos_data_id = os.getenv('NACOS_DATA_ID', 'tamar-console-dev')
+            self.local_dev_mode = os.getenv('LOCAL_DEV_MODE', 'true').lower() == 'true'
+            
+            # 服务注册信息
+            self.service_info = {
+                'name': os.getenv('SERVICE_NAME', 'tamar-console'),
+                'domain': os.getenv('SERVICE_DOMAIN', '127.0.0.1'),
+                'port': int(os.getenv('SERVICE_PORT', '8002'))
+            }
+            
+            # 创建 Nacos 客户端（保持连接）
+            if not self.nacos_client:
+                self.nacos_client = NacosClient(
+                    self.nacos_config['server_addresses'],
+                    namespace=self.nacos_config['namespace'],
+                    username=self.nacos_config['username'],
+                    password=self.nacos_config['password']
+                )
+                logger.info(f"Nacos客户端初始化成功: {self.nacos_config['server_addresses']}")
+        except Exception as e:
+            logger.error(f"初始化配置失败: {e}")
+            raise
+    
+    def _load_config(self):
+        """加载配置"""
+        try:
+            # 尝试从 Nacos 加载配置
+            config = self._load_from_nacos()
+            self.last_refresh_time = datetime.now()
+            self.config_version += 1
+            logger.info(f"配置加载成功，版本: {self.config_version}")
+            return config
+        except Exception as e:
+            logger.error(f"从 Nacos 加载配置失败: {e}")
+            # 如果是首次加载失败，抛出异常
+            if not hasattr(self, '_config'):
+                raise
+            # 如果是刷新失败，保持现有配置
+            logger.warning("配置刷新失败，保持现有配置")
+            return self._config
+    
+    def _load_from_nacos(self):
+        """从 Nacos 加载配置"""
+        # if self.local_dev_mode:
+        #     # 本地开发模式，使用默认配置
+        #     logger.info("本地开发模式，使用默认配置")
+        #     return self._get_default_config()
+        
+        if not self.nacos_client:
+            raise ValueError("Nacos客户端未初始化")
+        
+        # 获取配置
+        config_str = self.nacos_client.get_config(self.nacos_data_id, self.nacos_group)
+        
+        if not config_str:
+            logger.warning(f"Nacos配置为空: {self.nacos_data_id}/{self.nacos_group}")
+            return self._get_default_config()
+        
+        # 解析配置（支持 Properties、YAML 和 JSON）
+        if self._is_properties_format(config_str):
+            config = self._parse_properties(config_str)
+        else:
+            # 尝试 YAML 或 JSON
+            try:
+                config = yaml.safe_load(config_str)
+            except:
+                try:
+                    config = json.loads(config_str)
+                except:
+                    logger.error(f"无法解析配置格式: {config_str[:100]}...")
+                    raise
+        logger.debug(f"成功加载配置，包含 {len(config)} 个配置项")
+        return config
+    
+    def _get_default_config(self) -> Dict[str, Any]:
+        """获取默认配置"""
+        return {
+            'PG_DB_HOST': 'localhost',
+            'PG_DB_PORT': 5432,
+            'PG_DB_USERNAME': 'jettask',
+            'PG_DB_PASSWORD': '123456',
+            'PG_DB_DATABASE': 'jettask',
+            'REDIS_HOST': 'localhost',
+            'REDIS_PORT': 6379,
+            'REDIS_DB': 0,
+            'REDIS_PASSWORD': None
+        }
+    
+    def _is_properties_format(self, config_str):
+        """判断是否为 Properties 格式"""
+        lines = config_str.strip().split('\n')
+        for line in lines:
+            line = line.strip()
+            if line and not line.startswith('#'):
+                return '=' in line
+        return False
+    
+    def _parse_properties(self, config_str):
+        """解析 Properties 格式配置"""
+        config = {}
+        lines = config_str.strip().split('\n')
+        
+        for line in lines:
+            line = line.strip()
+            # 跳过空行和注释
+            if not line or line.startswith('#'):
+                continue
+            
+            # 解析键值对
+            if '=' in line:
+                key, value = line.split('=', 1)
+                key = key.strip()
+                value = value.strip()
+                
+                # 转换为嵌套字典
+                parts = key.split('.')
+                current = config
+                for i, part in enumerate(parts[:-1]):
+                    if part not in current:
+                        current[part] = {}
+                    current = current[part]
+                
+                # 尝试转换数据类型
+                final_value = value
+                if value.lower() == 'true':
+                    final_value = True
+                elif value.lower() == 'false':
+                    final_value = False
+                elif value.isdigit():
+                    final_value = int(value)
+                elif self._is_float(value):
+                    final_value = float(value)
+                
+                current[parts[-1]] = final_value
+        
+        return config
+    
+    def _is_float(self, value):
+        """判断字符串是否为浮点数"""
+        try:
+            float(value)
+            return '.' in value
+        except:
+            return False
+    
+    def get_database_url(self):
+        """构建数据库连接字符串（FastAPI 使用）"""
+        return self._get_database_uri()
+    
+    def _get_database_uri(self):
+        """构建数据库连接字符串"""
+        config = self.config
+        host = config.get("PG_DB_HOST")
+        port = config.get("PG_DB_PORT")
+        username = config.get("PG_DB_USERNAME")
+        password = config.get("PG_DB_PASSWORD")
+        database = config.get("PG_DB_DATABASE")
+        return f'postgresql+asyncpg://{username}:{password}@{host}:{port}/{database}'
+    
+    def get_redis_connection(self):
+        """获取 Redis 异步连接实例"""
+        config = self.config
+
+        redis_host = config.get("REDIS_HOST", "localhost")
+        redis_port = int(config.get("REDIS_PORT", "6379"))
+        redis_db = int(config.get("REDIS_DB", "0"))
+        redis_password = config.get("REDIS_PASSWORD") or None
+        redis_user = config.get("REDIS_USER") or None
+
+        # 构建 redis:// 或 rediss:// 的 URL（注意 password 可能为空）
+        if redis_user:
+            redis_url = f"redis://{redis_user}:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+        elif redis_password:
+            redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
+        else:
+            redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
+        print(f'{redis_url=}')
+        return redis_url
+    
+    def get(self, key, default=None):
+        """获取配置项（线程安全）"""
+        with self._config_lock:
+            return self.config.get(key, default)
+    
+    def refresh(self) -> bool:
+        """手动刷新配置
+        
+        Returns:
+            True if config was refreshed, False otherwise
+        """
+        try:
+            logger.info("手动触发配置刷新")
+            old_config = self._config.copy() if hasattr(self, '_config') else {}
+            
+            with self._config_lock:
+                new_config = self._load_config()
+                self._config = new_config
+            
+            # 检查配置是否有变化
+            if old_config != new_config:
+                logger.info("配置已更新，通知监听器")
+                self._notify_listeners(new_config)
+                return True
+            else:
+                logger.debug("配置无变化")
+                return False
+        except Exception as e:
+            logger.error(f"刷新配置失败: {e}")
+            return False
+    
+    def start_refresh_thread(self):
+        """启动定时刷新线程"""
+        if self.refresh_thread and self.refresh_thread.is_alive():
+            logger.warning("刷新线程已在运行")
+            return
+        
+        self.stop_refresh.clear()
+        self.refresh_thread = threading.Thread(
+            target=self._refresh_loop,
+            name="NacosConfigRefresh",
+            daemon=True
+        )
+        self.refresh_thread.start()
+        logger.info(f"配置刷新线程已启动，刷新间隔: {self.refresh_interval}秒")
+    
+    def stop_refresh_thread(self):
+        """停止定时刷新线程"""
+        if self.refresh_thread and self.refresh_thread.is_alive():
+            logger.info("正在停止配置刷新线程...")
+            self.stop_refresh.set()
+            self.refresh_thread.join(timeout=5)
+            logger.info("配置刷新线程已停止")
+    
+    def _refresh_loop(self):
+        """刷新循环"""
+        logger.info("配置刷新循环已开始")
+        
+        while not self.stop_refresh.is_set():
+            try:
+                # 等待指定的刷新间隔
+                if self.stop_refresh.wait(self.refresh_interval):
+                    break
+                
+                # 执行刷新
+                self.refresh()
+                
+            except Exception as e:
+                logger.error(f"配置刷新循环异常: {e}")
+                # 发生异常后等待一段时间再重试
+                if self.stop_refresh.wait(10):
+                    break
+        
+        logger.info("配置刷新循环已结束")
+    
+    def add_change_listener(self, listener: Callable[[Dict[str, Any]], None]):
+        """添加配置变更监听器
+        
+        Args:
+            listener: 配置变更时的回调函数，接收新配置作为参数
+        """
+        if listener not in self.change_listeners:
+            self.change_listeners.append(listener)
+            logger.info(f"添加配置变更监听器: {listener.__name__}")
+    
+    def remove_change_listener(self, listener: Callable[[Dict[str, Any]], None]):
+        """移除配置变更监听器"""
+        if listener in self.change_listeners:
+            self.change_listeners.remove(listener)
+            logger.info(f"移除配置变更监听器: {listener.__name__}")
+    
+    def _notify_listeners(self, new_config: Dict[str, Any]):
+        """通知所有监听器配置已变更"""
+        for listener in self.change_listeners:
+            try:
+                listener(new_config)
+            except Exception as e:
+                logger.error(f"配置变更监听器执行失败 {listener.__name__}: {e}")
+    
+    def get_config_info(self) -> Dict[str, Any]:
+        """获取配置信息"""
+        return {
+            'version': self.config_version,
+            'last_refresh_time': self.last_refresh_time.isoformat() if self.last_refresh_time else None,
+            'refresh_interval': self.refresh_interval,
+            'refresh_thread_active': self.refresh_thread.is_alive() if self.refresh_thread else False,
+            'nacos_server': self.nacos_config['server_addresses'] if self.nacos_config else None,
+            'nacos_namespace': self.nacos_config['namespace'] if self.nacos_config else None,
+            'nacos_group': self.nacos_group,
+            'nacos_data_id': self.nacos_data_id,
+            'local_dev_mode': self.local_dev_mode,
+            'config_items_count': len(self.config) if hasattr(self, '_config') else 0
+        }
+    
+    def __del__(self):
+        """析构函数，确保线程正确关闭"""
+        self.stop_refresh_thread()
+
+
+# 创建全局配置实例（默认30秒刷新一次）
+config = Config(refresh_interval=30)

jettask/core/app.py

@@ -298,76 +298,7 @@ class Jettask(object):
         self._should_exit = False
         self._worker_started = False
         self._handlers_registered = False
-    
-    async def _load_config_from_task_center_async(self):
-        """异步加载任务中心配置"""
-        try:
-            if not self.task_center:
-                return False
-            
-            # 连接并获取配置
-            connected = await self.task_center.connect()
-            if not connected:
-                return False
-            
-            config = {
-                'redis_config': self.task_center.redis_config,
-                'pg_config': self.task_center.pg_config,
-                'namespace_name': self.task_center.namespace_name,
-                'version': self.task_center.version
-            }
-            
-            if config['redis_config']:
-                # 任务中心配置优先级高于手动配置
-                redis_config = config.get('redis_config', {})
-                pg_config = config.get('pg_config', {})
-                # 构建Redis URL
-                if redis_config:
-                    redis_host = redis_config.get('host', 'localhost')
-                    redis_port = redis_config.get('port', 6379)
-                    redis_password = redis_config.get('password')
-                    redis_db = redis_config.get('db', 0)
-                    
-                    if redis_password:
-                        self.redis_url = f"redis://:{redis_password}@{redis_host}:{redis_port}/{redis_db}"
-                    else:
-                        self.redis_url = f"redis://{redis_host}:{redis_port}/{redis_db}"
-                    
-                    logger.info(f"从任务中心加载Redis配置: {redis_host}:{redis_port}/{redis_db}")
-                
-                # 构建PostgreSQL URL
-                if pg_config:
-                    pg_host = pg_config.get('host', 'localhost')
-                    pg_port = pg_config.get('port', 5432)
-                    pg_user = pg_config.get('user', 'postgres')
-                    pg_password = pg_config.get('password', '')
-                    pg_database = pg_config.get('database', 'jettask')
-                    
-                    self.pg_url = f"postgresql://{pg_user}:{pg_password}@{pg_host}:{pg_port}/{pg_database}"
-                    logger.info(f"从任务中心加载PostgreSQL配置: {pg_host}:{pg_port}/{pg_database}")
-                
-                # 保存配置供后续使用
-                self._task_center_config = config
-                
-                # 更新Redis前缀为命名空间名称
-                if self.task_center and self.task_center.redis_prefix != "jettask":
-                    self.redis_prefix = self.task_center.redis_prefix
-                    # 更新相关前缀
-                    self.STATUS_PREFIX = f"{self.redis_prefix}:STATUS:"
-                    self.RESULT_PREFIX = f"{self.redis_prefix}:RESULT:"
-                
-                # 清理已缓存的Redis连接，强制重新创建
-                if hasattr(self, '_redis'):
-                    delattr(self, '_redis')
-                if hasattr(self, '_async_redis'):
-                    delattr(self, '_async_redis')
-                if hasattr(self, '_ep'):
-                    delattr(self, '_ep')
-                    
-                return True
-        except Exception as e:
-            logger.warning(f"从任务中心加载配置失败: {e}")
-            return False
+   
     
     def _load_config_from_task_center(self):
         """从任务中心加载配置"""
@@ -461,7 +392,7 @@ class Jettask(object):
             task_center: TaskCenter实例
             
         使用示例：
-            from jettask.webui.task_center import TaskCenter
+            from jettask.task_center import TaskCenter
             
             # 创建任务中心客户端（可复用）
             task_center = TaskCenter("http://localhost:8001/api/namespaces/demo")
@@ -510,24 +441,6 @@ class Jettask(object):
                 'version': task_center.version
             }
     
-    async def init_from_task_center(self):
-        """
-        从任务中心初始化配置（异步方法）
-        
-        在异步环境中发送任务前调用此方法，确保配置已加载
-        
-        使用示例：
-            from jettask.webui.task_center import TaskCenter
-            
-            task_center = TaskCenter("http://localhost:8001/api/namespaces/demo")
-            app = Jettask()
-            app.mount_task_center(task_center)
-            await app.init_from_task_center()  # 会自动连接task_center
-            await task.apply_async(...)
-        """
-        if self.task_center and self.task_center.is_enabled and not self._task_center_config:
-            return await self._load_config_from_task_center_async()
-        return True
     
     def _setup_cleanup_handlers(self):
         """设置清理处理器"""
@@ -619,9 +532,9 @@ class Jettask(object):
             return getattr(self, name)
         
         # 如果配置了任务中心且还未加载配置，先加载配置
-        if self.task_center and self.task_center.is_enabled and not self._task_center_config:
-            self._load_config_from_task_center()
-        
+        # if self.task_center and self.task_center.is_enabled and not self._task_center_config:
+        #     self._load_config_from_task_center()
+        print(f'{self.redis_url=}')
         pool = get_redis_pool(self.redis_url, self.max_connections)
         redis_cli = redis.StrictRedis(connection_pool=pool)
         setattr(self, name, redis_cli)
@@ -1059,12 +972,7 @@ class Jettask(object):
         
         event_queue = deque()
         
-        # 创建消费者组
-        try:
-            self.ep.create_group()
-        except Exception as e:
-            logger.warning(f"创建消费者组时出错: {e}")
-            # 继续执行，listening_event会自动处理
+        # 消费者组会在listening_event方法内部自动创建
         
         # 根据执行器类型创建对应的执行器
         if execute_type == "asyncio":
@@ -1138,8 +1046,8 @@ class Jettask(object):
         # 标记worker已启动
         self._worker_started = True
         
-        # 如果配置了任务中心，从任务中心获取配置
-        if self.task_center and self.task_center.is_enabled:
+        # 如果配置了任务中心且配置尚未加载，从任务中心获取配置
+        if self.task_center and self.task_center.is_enabled and not self._task_center_config:
             self._load_config_from_task_center()
         
         # 注册清理处理器（只在启动worker时注册）