mdbq 4.0.110__tar.gz → 4.0.112__tar.gz

{mdbq-4.0.110 → mdbq-4.0.112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.110
+Version: 4.0.112
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.112/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.112'

{mdbq-4.0.110 → mdbq-4.0.112}/mdbq/redis/redis_cache.py

@@ -18,9 +18,17 @@ import socket
 from datetime import datetime, date
 from decimal import Decimal
 from uuid import UUID
-from typing import Optional, Dict, Any, List, Union
+from typing import Optional, Dict, Any, List, Union, Callable
 import redis
 from mdbq.log import mylogger
+
+import enum
+from typing import Optional, Dict, Any, List, Union, Callable
+from concurrent.futures import ThreadPoolExecutor
+import asyncio
+from threading import Event
+import functools
+
 logger = mylogger.MyLogger(
     logging_mode='file',
     log_level='info',
@@ -34,6 +42,14 @@ logger = mylogger.MyLogger(
 )
 
 
+class CacheSystemState(enum.Enum):
+    """缓存系统状态枚举"""
+    INITIALIZING = "initializing"
+    READY = "ready" 
+    MYSQL_READY = "mysql_ready"
+    ERROR = "error"
+
+
 class MySQLDataEncoder(json.JSONEncoder):
     """自定义JSON编码器，支持MySQL常见数据类型"""
     def default(self, obj):
@@ -73,6 +89,64 @@ class MySQLDataEncoder(json.JSONEncoder):
         return super().default(obj)
 
 
+class SmartTTLConfig:
+    """智能TTL配置策略"""
+    
+    # TTL策略映射表（秒）
+    TTL_STRATEGIES = {
+        # 数据库相关缓存
+        'sycm_database': 1800,      # 数据库列表：30分钟
+        'sycm_tables': 1200,        # 表列表：20分钟  
+        'sycm_table_data': 300,     # 表数据：5分钟
+        
+        # 用户会话相关
+        'user_session': 3600,       # 用户会话：1小时
+        'user_info': 1800,          # 用户信息：30分钟
+        
+        # 静态配置
+        'static_config': 86400,     # 静态配置：24小时
+        'system_config': 43200,     # 系统配置：12小时
+        
+        # API相关
+        'api_response': 600,        # API响应：10分钟
+        'api_cache': 900,           # API缓存：15分钟
+        
+        # 临时数据
+        'temp_data': 300,           # 临时数据：5分钟
+        'short_cache': 60,          # 短期缓存：1分钟
+    }
+    
+    @classmethod
+    def get_ttl(cls, namespace: str, default: int = 3600) -> int:
+        """
+        根据命名空间获取对应的TTL值
+        
+        Args:
+            namespace: 缓存命名空间
+            default: 默认TTL值（秒）
+            
+        Returns:
+            int: TTL值（秒）
+        """
+        return cls.TTL_STRATEGIES.get(namespace, default)
+    
+    @classmethod
+    def add_strategy(cls, namespace: str, ttl: int):
+        """
+        添加新的TTL策略
+        
+        Args:
+            namespace: 命名空间
+            ttl: TTL值（秒）
+        """
+        cls.TTL_STRATEGIES[namespace] = ttl
+    
+    @classmethod
+    def get_all_strategies(cls) -> Dict[str, int]:
+        """获取所有TTL策略"""
+        return cls.TTL_STRATEGIES.copy()
+
+
 class CacheConfig:
     """缓存系统配置类"""
     
@@ -98,6 +172,10 @@ class CacheConfig:
         self.max_value_size = 1024 * 1024  # 1MB
         self.batch_size = 100
         
+        # 热点键配置
+        self.max_hot_keys = 1000    # 最大热点键数量
+        self.hot_keys_cleanup_threshold = 800  # 热点键清理阈值
+        
         # MySQL数据库配置
         self.db_name = db_name
         self.table_name = table_name
@@ -129,6 +207,11 @@ class SmartCacheSystem:
             
         self.logger = logger
         
+        # 状态管理
+        self._state = CacheSystemState.INITIALIZING
+        self._ready_event = Event()
+        self._mysql_ready_event = Event()
+        
         # 统计数据
         self.stats = {
             'hits': 0,
@@ -150,16 +233,69 @@ class SmartCacheSystem:
         self._stats_thread = None
         self._stats_lock = threading.RLock()
         
-        # 初始化
-        self._init_mysql_db()
-        self._start_stats_worker()
+        # 使用线程池异步初始化
+        self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=f"cache_{instance_name}")
+        
+        # 立即设置为基础就绪状态（Redis缓存可用）
+        if self._test_redis_connection():
+            self._state = CacheSystemState.READY
+            self._ready_event.set()
+        
+        # 异步初始化MySQL相关功能
+        self._executor.submit(self._initialize_mysql_features)
         
         self.logger.info("智能缓存系统初始化完成", {
             'instance_name': self.instance_name,
-            'mysql_enabled': self.mysql_pool is not None,
-            'redis_connected': self._test_redis_connection()
+            'state': self._state.value,
+            'redis_ready': self._ready_event.is_set(),
+            'mysql_enabled': self.mysql_pool is not None
         })
     
+    def _initialize_mysql_features(self):
+        """在后台初始化MySQL相关功能"""
+        try:
+            if self.mysql_pool:
+                # 延迟初始化MySQL（给系统一些启动时间）
+                time.sleep(5)  # 等待5秒让主系统完全启动
+                
+                if self._init_mysql_db():
+                    self._state = CacheSystemState.MYSQL_READY
+                    self._mysql_ready_event.set()
+                    
+                    # MySQL就绪后再启动统计线程
+                    self._start_stats_worker()
+                    
+                    self.logger.info("MySQL功能初始化完成", {
+                        'instance_name': self.instance_name,
+                        'state': self._state.value
+                    })
+                else:
+                    self.logger.warning("MySQL初始化失败，缓存功能仍可正常使用")
+            else:
+                self.logger.info("未配置MySQL，跳过统计功能")
+                
+        except Exception as e:
+            self._state = CacheSystemState.ERROR
+            self.logger.error(f"MySQL功能初始化异常: {e}")
+    
+    @property
+    def is_ready(self) -> bool:
+        """检查缓存系统是否就绪（Redis可用即为就绪）"""
+        return self._ready_event.is_set()
+    
+    @property 
+    def is_mysql_ready(self) -> bool:
+        """检查MySQL功能是否就绪"""
+        return self._mysql_ready_event.is_set()
+    
+    def wait_for_ready(self, timeout: float = 1.0) -> bool:
+        """等待缓存系统就绪（非阻塞，有超时）"""
+        return self._ready_event.wait(timeout)
+    
+    def wait_for_mysql_ready(self, timeout: float = 5.0) -> bool:
+        """等待MySQL功能就绪（非阻塞，有超时）"""
+        return self._mysql_ready_event.wait(timeout)
+    
     def _test_redis_connection(self) -> bool:
         """测试Redis连接"""
         try:
@@ -176,9 +312,23 @@ class SmartCacheSystem:
             return False
         
         try:
+            # 设置连接超时，避免长时间阻塞
             connection = self.mysql_pool.connection()
+            # 设置查询超时（如果支持）
+            try:
+                connection.autocommit(False)  # 确保事务控制
+            except:
+                pass  # 忽略不支持的操作
+                
             try:
                 with connection.cursor() as cursor:
+                    # 设置会话超时
+                    try:
+                        cursor.execute("SET SESSION wait_timeout = 30")
+                        cursor.execute("SET SESSION interactive_timeout = 30")
+                    except:
+                        pass  # 忽略不支持的操作
+                    
                     # 创建数据库
                     cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
                     cursor.execute(f"USE `{self.config.db_name}`")
@@ -230,6 +380,7 @@ class SmartCacheSystem:
                 
         except Exception as e:
             self.logger.error(f"MySQL数据库初始化失败: {e}")
+            # 不抛出异常，让系统继续运行
             return False
     
     def _generate_cache_key(self, key: str, namespace: str = "") -> str:
@@ -254,75 +405,106 @@ class SmartCacheSystem:
         """记录热点键"""
         cache_key = self._generate_cache_key(key, namespace)
         with self.hot_keys_lock:
+            # 检查是否需要清理热点键（防止内存泄漏）
+            if len(self.hot_keys) >= self.config.hot_keys_cleanup_threshold:
+                # 保留访问次数最高的键，移除访问次数最少的键
+                sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
+                # 保留前600个热点键，为新键留出空间
+                self.hot_keys = dict(sorted_keys[:600])
+                
+                self.logger.info("热点键内存清理", {
+                    '清理前数量': len(sorted_keys),
+                    '清理后数量': len(self.hot_keys),
+                    '清理阈值': self.config.hot_keys_cleanup_threshold
+                })
+            
+            # 记录或更新热点键访问次数
             self.hot_keys[cache_key] = self.hot_keys.get(cache_key, 0) + 1
     
+    def _cache_operation(operation_name: str):
+        """缓存操作装饰器工厂"""
+        def decorator(func):
+            @functools.wraps(func)
+            def wrapper(self, *args, **kwargs):
+                start_time = time.time()
+                
+                # 检查系统是否就绪
+                if not self.is_ready:
+                    self._record_operation('errors')
+                    self.logger.warning(f"缓存系统未就绪，{operation_name}操作失败", {
+                        'state': self._state.value
+                    })
+                    return None if operation_name == 'get' else False
+                
+                try:
+                    result = func(self, *args, **kwargs)
+                    response_time = (time.time() - start_time) * 1000
+                    self._record_operation(operation_name, response_time)
+                    return result
+                    
+                except Exception as e:
+                    self._record_operation('errors')
+                    self.logger.error(f"缓存{operation_name}操作失败: {e}", {
+                        'args': args[:2] if len(args) > 2 else args,  # 避免记录过长的参数
+                        'operation': operation_name
+                    })
+                    return None if operation_name == 'get' else False
+                    
+            return wrapper
+        return decorator
+    
+    @_cache_operation('get')
     def get(self, key: str, namespace: str = "", default=None) -> Any:
         """获取缓存值"""
-        start_time = time.time()
+        cache_key = self._generate_cache_key(key, namespace)
         
-        try:
-            cache_key = self._generate_cache_key(key, namespace)
-            
-            # 获取缓存值
-            value = self.redis_client.get(cache_key)
-            response_time = (time.time() - start_time) * 1000
+        # 获取缓存值
+        value = self.redis_client.get(cache_key)
+        
+        if value is not None:
+            # 缓存命中
+            self._record_hot_key(key, namespace)
             
-            if value is not None:
-                # 缓存命中
-                self._record_operation('hits', response_time)
-                self._record_hot_key(key, namespace)
-                
-                try:
-                    return json.loads(value.decode('utf-8'))
-                except (json.JSONDecodeError, UnicodeDecodeError):
-                    return value.decode('utf-8')
-            else:
-                # 缓存未命中
-                self._record_operation('misses', response_time)
-                return default
-                
-        except Exception as e:
-            self._record_operation('errors')
-            self.logger.error(f"缓存获取失败: {e}", {
-                'key': key,
-                'namespace': namespace
-            })
+            try:
+                return json.loads(value.decode('utf-8'))
+            except (json.JSONDecodeError, UnicodeDecodeError):
+                return value.decode('utf-8')
+        else:
+            # 缓存未命中
             return default
     
+    @_cache_operation('set')
     def set(self, key: str, value: Any, ttl: int = None, namespace: str = "") -> bool:
         """设置缓存值"""
-        start_time = time.time()
+        cache_key = self._generate_cache_key(key, namespace)
         
-        try:
-            cache_key = self._generate_cache_key(key, namespace)
-            ttl = ttl or self.config.default_ttl
-            
-            # 序列化值
-            if isinstance(value, (dict, list, tuple)):
-                serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
-            else:
-                serialized_value = str(value)
-            
-            # 检查值大小
-            if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
-                self.logger.warning(f"缓存值过大，跳过设置: {len(serialized_value)} bytes")
-                return False
-            
-            # 设置缓存
-            result = self.redis_client.setex(cache_key, ttl, serialized_value)
-            response_time = (time.time() - start_time) * 1000
-            
-            self._record_operation('sets', response_time)
-            return bool(result)
-            
-        except Exception as e:
-            self._record_operation('errors')
-            self.logger.error(f"缓存设置失败: {e}", {
-                'key': key,
+        # 使用智能TTL策略
+        if ttl is None:
+            ttl = SmartTTLConfig.get_ttl(namespace, self.config.default_ttl)
+        
+        # 序列化值
+        if isinstance(value, (dict, list, tuple)):
+            serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
+        else:
+            serialized_value = str(value)
+        
+        # 检查值大小
+        if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
+            self.logger.warning(f"缓存值过大，跳过设置: {len(serialized_value)} bytes")
+            return False
+        
+        # 设置缓存
+        result = self.redis_client.setex(cache_key, ttl, serialized_value)
+        
+        # 记录TTL策略使用情况
+        if namespace and ttl != self.config.default_ttl:
+            self.logger.debug("使用智能TTL策略", {
                 'namespace': namespace,
-                'ttl': ttl
+                'ttl': ttl,
+                'key': key[:50] + "..." if len(key) > 50 else key
             })
-            return False
+        
+        return bool(result)
     
     def delete(self, key: str, namespace: str = "") -> bool:
         """删除缓存值"""
@@ -437,45 +619,87 @@ class SmartCacheSystem:
     
     def _start_stats_worker(self):
         """启动统计工作线程"""
-        if not self._stats_running:
+        if not self._stats_running and self.mysql_pool:
             self._stats_running = True
-            self._stats_thread = threading.Thread(target=self._stats_worker, daemon=True)
+            self._stats_thread = threading.Thread(
+                target=self._stats_worker, 
+                daemon=True,
+                name=f"stats_worker_{self.instance_name}"
+            )
             self._stats_thread.start()
-            self.logger.info("统计工作线程已启动")
+            self.logger.info("统计工作线程已启动", {
+                'instance_name': self.instance_name,
+                'delay_first_run': True
+            })
     
     def _stats_worker(self):
         """后台统计工作线程"""
-        cleanup_counter = 0  # 清理计数器
+        cleanup_counter = 0
+        
+        # 延迟启动：等待一个完整的统计间隔
+        self.logger.info("统计线程启动，等待首个统计间隔", {
+            'instance_name': self.instance_name,
+            'interval': self.config.stats_interval,
+            'delay_reason': '避免初始化时的阻塞操作'
+        })
+        
+        # 使用可中断的等待
+        if not self._interruptible_sleep(self.config.stats_interval):
+            return  # 如果被中断则退出
+        
         while self._stats_running:
             try:
                 # 收集统计数据
                 stats_data = self.get_stats()
                 
-                # 提交到MySQL
+                # 提交到MySQL（带状态检查）
                 self._submit_stats_to_mysql(stats_data)
                 
                 # 清理过期的热点键统计
                 self._cleanup_hot_keys()
                 
-                # 每24次统计周期（约2小时）执行一次过期数据清理
+                # 定期清理过期数据
                 cleanup_counter += 1
-                if cleanup_counter >= 24:  # 24 * 300秒 = 2小时
+                if cleanup_counter >= 24:  # 24 * 1800秒 = 12小时
                     self._cleanup_expired_mysql_data()
                     cleanup_counter = 0
                 
             except Exception as e:
                 self.logger.error(f"统计工作线程异常: {e}")
             
-            # 等待下一个统计间隔
-            time.sleep(self.config.stats_interval)
+            # 可中断的等待下一个统计间隔
+            if not self._interruptible_sleep(self.config.stats_interval):
+                break
+    
+    def _interruptible_sleep(self, duration: float) -> bool:
+        """可中断的睡眠"""
+        sleep_interval = 1.0  # 每秒检查一次
+        elapsed = 0.0
+        
+        while elapsed < duration and self._stats_running:
+            time.sleep(min(sleep_interval, duration - elapsed))
+            elapsed += sleep_interval
+            
+        return self._stats_running
     
     def _submit_stats_to_mysql(self, stats_data: Dict[str, Any]):
         """提交统计数据到MySQL"""
         if not self.mysql_pool:
             self.logger.debug("MySQL连接池不可用，跳过统计数据提交")
             return
-        
+            
+        # 使用状态检查而不是属性检查
+        if not self.is_mysql_ready:
+            self.logger.debug("MySQL尚未就绪，跳过统计数据提交", {
+                'state': self._state.value,
+                'instance_name': self.instance_name
+            })
+            return
+            
+        logger.info("统计数据", {'stats_data': stats_data})
+
         try:
+            # 使用上下文管理器确保连接正确关闭
             connection = self.mysql_pool.connection()
             try:
                 with connection.cursor() as cursor:
@@ -483,7 +707,7 @@ class SmartCacheSystem:
                     
                     # 准备数据
                     now = datetime.now()
-                    date_str = now.strftime("%Y-%m-%d")  # 格式: 2024-10-01
+                    date_str = now.strftime("%Y-%m-%d")
                     time_period = now.strftime("%Y%m%d_%H%M")
                     
                     insert_sql = f"""
@@ -519,25 +743,40 @@ class SmartCacheSystem:
                     
                     connection.commit()
                     
-                    self.logger.debug("统计数据已提交到MySQL", {
+                    self.logger.info("统计数据已提交到MySQL", {
                         'time_period': time_period,
                         'total_operations': stats_data['total_operations'],
-                        'hit_rate': stats_data['hit_rate']
+                        'hit_rate': stats_data['hit_rate'],
+                        'instance_name': self.instance_name
                     })
                     
             finally:
                 connection.close()
                 
         except Exception as e:
-            self.logger.error(f"提交统计数据到MySQL失败: {e}")
+            self.logger.error(f"提交统计数据到MySQL失败: {e}", {
+                'instance_name': self.instance_name,
+                'error_type': type(e).__name__
+            })
     
     def _cleanup_hot_keys(self):
-        """清理热点键统计（保留访问次数最高的1000个）"""
+        """清理热点键统计 """
         with self.hot_keys_lock:
-            if len(self.hot_keys) > 1000:
-                # 保留访问次数最高的1000个键
+            current_count = len(self.hot_keys)
+            
+            # 如果热点键数量超过最大限制，进行清理
+            if current_count > self.config.max_hot_keys:
+                # 保留访问次数最高的键
                 sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
-                self.hot_keys = dict(sorted_keys[:1000])
+                # 保留80%的热点键
+                keep_count = int(self.config.max_hot_keys * 0.8)
+                self.hot_keys = dict(sorted_keys[:keep_count])
+                
+                self.logger.info("定期热点键清理", {
+                    '清理前数量': current_count,
+                    '清理后数量': len(self.hot_keys),
+                    '保留比例': '80%'
+                })
     
     def _cleanup_expired_mysql_data(self):
         """清理过期的MySQL统计数据"""
@@ -579,21 +818,46 @@ class SmartCacheSystem:
     
     def shutdown(self):
         """关闭缓存系统"""
-        self.logger.info("正在关闭缓存系统...")
+        self.logger.info("正在关闭缓存系统...", {
+            'instance_name': self.instance_name,
+            'state': self._state.value
+        })
         
         # 停止统计线程
         self._stats_running = False
         if self._stats_thread and self._stats_thread.is_alive():
-            self._stats_thread.join(timeout=5)
+            self.logger.info("等待统计线程结束...")
+            self._stats_thread.join(timeout=10)  # 最多等待10秒
+            
+            if self._stats_thread.is_alive():
+                self.logger.warning("统计线程未能在超时时间内结束")
         
-        # 最后一次提交统计数据
-        try:
-            stats_data = self.get_stats()
-            self._submit_stats_to_mysql(stats_data)
-        except Exception as e:
-            self.logger.error(f"关闭时提交统计数据失败: {e}")
+        # 关闭线程池
+        if hasattr(self, '_executor'):
+            self.logger.info("关闭线程池...")
+            self._executor.shutdown(wait=True, timeout=5)
         
-        self.logger.info("缓存系统已关闭")
+        # 最后一次提交统计数据（如果MySQL可用）
+        if self.is_mysql_ready:
+            try:
+                self.logger.info("提交最终统计数据...")
+                stats_data = self.get_stats()
+                self._submit_stats_to_mysql(stats_data)
+            except Exception as e:
+                self.logger.error(f"关闭时提交统计数据失败: {e}")
+        
+        self.logger.info("缓存系统已关闭", {
+            'instance_name': self.instance_name
+        })
+    
+    def __enter__(self):
+        """上下文管理器支持"""
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """上下文管理器退出时自动关闭"""
+        self.shutdown()
+        return False
 
 
 class CacheManager:
@@ -621,8 +885,9 @@ class CacheManager:
     
     def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
                    config: CacheConfig = None, db_name: str = None, table_name: str = None):
-        """初始化缓存系统"""
+        """初始化缓存系统（非阻塞）"""
         try:
+            # 立即创建缓存实例，内部会异步初始化MySQL和统计线程
             self.cache_instance = SmartCacheSystem(
                 redis_client=redis_client,
                 mysql_pool=mysql_pool,
@@ -632,11 +897,12 @@ class CacheManager:
                 table_name=table_name
             )
             self.initialization_error = None
-            self.logger.info("缓存管理器初始化成功", {
+            self.logger.info("缓存管理器初始化成功（异步模式）", {
                 'instance_name': instance_name,
                 'mysql_enabled': mysql_pool is not None,
                 'db_name': self.cache_instance.config.db_name,
-                'table_name': self.cache_instance.config.table_name
+                'table_name': self.cache_instance.config.table_name,
+                'note': 'MySQL初始化将在后台异步完成'
             })
             return self  # 支持链式调用
             

{mdbq-4.0.110 → mdbq-4.0.112}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.110
+Version: 4.0.112
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.110/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.110'