mdbq 4.0.106__tar.gz → 4.0.107__tar.gz

{mdbq-4.0.106 → mdbq-4.0.107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.106
+Version: 4.0.107
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.107/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.107'

{mdbq-4.0.106 → mdbq-4.0.107}/mdbq/redis/redis_cache.py

@@ -35,44 +35,57 @@ logger = mylogger.MyLogger(
 class CacheConfig:
     """缓存系统配置类"""
     
-    # TTL配置（秒）
-    DEFAULT_TTL = 3600  # 1小时
-    SHORT_TTL = 300     # 5分钟
-    MEDIUM_TTL = 1800   # 30分钟
-    LONG_TTL = 7200     # 2小时
-    VERY_LONG_TTL = 86400  # 24小时
-    
-    # 缓存键前缀
-    CACHE_PREFIX = "smart_cache:"
-    STATS_PREFIX = "cache_stats:"
-    LOCK_PREFIX = "cache_lock:"
-    
-    # 统计配置
-    STATS_INTERVAL = 300  # 统计间隔（秒）
-    STATS_RETENTION = 7   # 统计数据保留天数
-    
-    # 性能配置
-    MAX_KEY_LENGTH = 250
-    MAX_VALUE_SIZE = 1024 * 1024  # 1MB
-    BATCH_SIZE = 100
-    
-    # 数据库配置
-    DB_NAME = "redis统计"
-    TABLE_NAME = "dpflask路由分析"
-    
-    # 锁配置
-    LOCK_TIMEOUT = 30     # 分布式锁超时时间
-    LOCK_RETRY_DELAY = 0.1  # 锁重试延迟
+    def __init__(self, db_name: str = "redis统计", table_name: str = "dpflask路由分析"):
+        # TTL配置（秒）
+        self.default_ttl = 3600  # 1小时
+        self.short_ttl = 300     # 5分钟
+        self.medium_ttl = 1800   # 30分钟
+        self.long_ttl = 7200     # 2小时
+        self.very_long_ttl = 86400  # 24小时
+        
+        # 缓存键前缀
+        self.cache_prefix = "smart_cache:"
+        self.stats_prefix = "cache_stats:"
+        self.lock_prefix = "cache_lock:"
+        
+        # 统计配置
+        self.stats_interval = 900  # 统计间隔（秒）, 自动提交统计信息到MySQL的间隔
+        self.stats_retention = 30   # MySQL统计数据保留天数，超过此天数的数据将被自动删除
+        
+        # 性能配置
+        self.max_key_length = 250
+        self.max_value_size = 1024 * 1024  # 1MB
+        self.batch_size = 100
+        
+        # MySQL数据库配置
+        self.db_name = db_name
+        self.table_name = table_name
+        
+        # 锁配置
+        self.lock_timeout = 30     # 分布式锁超时时间
+        self.lock_retry_delay = 0.1  # 锁重试延迟
 
 
 class SmartCacheSystem:
     """智能缓存系统核心类"""
     
-    def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default"):
+    def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", 
+                 config: CacheConfig = None, db_name: str = None, table_name: str = None):
         self.redis_client = redis_client
         self.mysql_pool = mysql_pool
         self.instance_name = instance_name
-        self.config = CacheConfig()
+        
+        # 配置优先级：传入的config > 自定义db_name/table_name > 默认配置
+        if config:
+            self.config = config
+        elif db_name or table_name:
+            self.config = CacheConfig(
+                db_name=db_name or "redis统计",
+                table_name=table_name or "dpflask路由分析"
+            )
+        else:
+            self.config = CacheConfig()
+            
         self.logger = logger
         
         # 统计数据
@@ -126,13 +139,14 @@ class SmartCacheSystem:
             try:
                 with connection.cursor() as cursor:
                     # 创建数据库
-                    cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.DB_NAME}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
-                    cursor.execute(f"USE `{self.config.DB_NAME}`")
+                    cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
+                    cursor.execute(f"USE `{self.config.db_name}`")
                     
                     # 创建表（MySQL 8.4+兼容语法）
                     create_table_sql = f"""
-                    CREATE TABLE IF NOT EXISTS `{self.config.TABLE_NAME}` (
+                    CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
                         `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
+                        `日期` date NOT NULL COMMENT '统计日期',
                         `统计时间` datetime NOT NULL COMMENT '统计时间',
                         `时间段` varchar(20) NOT NULL COMMENT '时间段标识',
                         `缓存命中数` bigint DEFAULT 0 COMMENT '缓存命中次数',
@@ -152,6 +166,7 @@ class SmartCacheSystem:
                         `创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
                         `更新时间` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
                         PRIMARY KEY (`id`),
+                        KEY `idx_stats_date` (`日期`),
                         KEY `idx_stats_time` (`统计时间`),
                         KEY `idx_time_period` (`时间段`),
                         KEY `idx_hit_rate` (`命中率`),
@@ -164,8 +179,8 @@ class SmartCacheSystem:
                     connection.commit()
                     
                     self.logger.info("MySQL数据库表初始化成功", {
-                        'database': self.config.DB_NAME,
-                        'table': self.config.TABLE_NAME
+                        'database': self.config.db_name,
+                        'table': self.config.table_name
                     })
                     return True
                     
@@ -179,8 +194,8 @@ class SmartCacheSystem:
     def _generate_cache_key(self, key: str, namespace: str = "") -> str:
         """生成缓存键"""
         if namespace:
-            return f"{self.config.CACHE_PREFIX}{namespace}:{key}"
-        return f"{self.config.CACHE_PREFIX}{key}"
+            return f"{self.config.cache_prefix}{namespace}:{key}"
+        return f"{self.config.cache_prefix}{key}"
     
     def _record_operation(self, operation: str, response_time: float = 0):
         """记录操作统计"""
@@ -239,7 +254,7 @@ class SmartCacheSystem:
         
         try:
             cache_key = self._generate_cache_key(key, namespace)
-            ttl = ttl or self.config.DEFAULT_TTL
+            ttl = ttl or self.config.default_ttl
             
             # 序列化值
             if isinstance(value, (dict, list, tuple)):
@@ -248,7 +263,7 @@ class SmartCacheSystem:
                 serialized_value = str(value)
             
             # 检查值大小
-            if len(serialized_value.encode('utf-8')) > self.config.MAX_VALUE_SIZE:
+            if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
                 self.logger.warning(f"缓存值过大，跳过设置: {len(serialized_value)} bytes")
                 return False
             
@@ -300,7 +315,7 @@ class SmartCacheSystem:
     def clear_namespace(self, namespace: str) -> int:
         """清除指定命名空间的所有缓存"""
         try:
-            pattern = f"{self.config.CACHE_PREFIX}{namespace}:*"
+            pattern = f"{self.config.cache_prefix}{namespace}:*"
             keys = self.redis_client.keys(pattern)
             
             if keys:
@@ -389,6 +404,7 @@ class SmartCacheSystem:
     
     def _stats_worker(self):
         """后台统计工作线程"""
+        cleanup_counter = 0  # 清理计数器
         while self._stats_running:
             try:
                 # 收集统计数据
@@ -400,11 +416,17 @@ class SmartCacheSystem:
                 # 清理过期的热点键统计
                 self._cleanup_hot_keys()
                 
+                # 每24次统计周期（约2小时）执行一次过期数据清理
+                cleanup_counter += 1
+                if cleanup_counter >= 24:  # 24 * 300秒 = 2小时
+                    self._cleanup_expired_mysql_data()
+                    cleanup_counter = 0
+                
             except Exception as e:
                 self.logger.error(f"统计工作线程异常: {e}")
             
             # 等待下一个统计间隔
-            time.sleep(self.config.STATS_INTERVAL)
+            time.sleep(self.config.stats_interval)
     
     def _submit_stats_to_mysql(self, stats_data: Dict[str, Any]):
         """提交统计数据到MySQL"""
@@ -416,24 +438,26 @@ class SmartCacheSystem:
             connection = self.mysql_pool.connection()
             try:
                 with connection.cursor() as cursor:
-                    cursor.execute(f"USE `{self.config.DB_NAME}`")
+                    cursor.execute(f"USE `{self.config.db_name}`")
                     
                     # 准备数据
                     now = datetime.now()
+                    date_str = now.strftime("%Y-%m-%d")  # 格式: 2024-10-01
                     time_period = now.strftime("%Y%m%d_%H%M")
                     
                     insert_sql = f"""
-                    INSERT INTO `{self.config.TABLE_NAME}` (
-                        `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
+                    INSERT INTO `{self.config.table_name}` (
+                        `日期`, `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
                         `缓存删除数`, `缓存错误数`, `命中率`, `总操作数`, `平均响应时间`,
                         `每秒操作数`, `唯一键数量`, `系统运行时间`, `热点键统计`,
                         `服务器主机`, `实例名称`
                     ) VALUES (
-                        %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
+                        %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
                     )
                     """
                     
                     cursor.execute(insert_sql, (
+                        date_str,
                         now,
                         time_period,
                         stats_data['hits'],
@@ -474,6 +498,44 @@ class SmartCacheSystem:
                 sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
                 self.hot_keys = dict(sorted_keys[:1000])
     
+    def _cleanup_expired_mysql_data(self):
+        """清理过期的MySQL统计数据"""
+        if not self.mysql_pool:
+            return
+        
+        try:
+            connection = self.mysql_pool.connection()
+            try:
+                with connection.cursor() as cursor:
+                    cursor.execute(f"USE `{self.config.db_name}`")
+                    
+                    # 计算过期时间点
+                    from datetime import timedelta
+                    expire_date = datetime.now() - timedelta(days=self.config.stats_retention)
+                    
+                    # 删除过期数据
+                    delete_sql = f"""
+                    DELETE FROM `{self.config.table_name}` 
+                    WHERE `统计时间` < %s
+                    """
+                    
+                    cursor.execute(delete_sql, (expire_date,))
+                    deleted_rows = cursor.rowcount
+                    connection.commit()
+                    
+                    if deleted_rows > 0:
+                        self.logger.info("清理过期MySQL统计数据", {
+                            'deleted_rows': deleted_rows,
+                            'expire_date': expire_date.strftime('%Y-%m-%d %H:%M:%S'),
+                            'retention_days': self.config.stats_retention
+                        })
+                    
+            finally:
+                connection.close()
+                
+        except Exception as e:
+            self.logger.error(f"清理过期MySQL统计数据失败: {e}")
+    
     def shutdown(self):
         """关闭缓存系统"""
         self.logger.info("正在关闭缓存系统...")
@@ -516,18 +578,24 @@ class CacheManager:
         self._initialized = True
         self.logger = logger
     
-    def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default"):
+    def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
+                   config: CacheConfig = None, db_name: str = None, table_name: str = None):
         """初始化缓存系统"""
         try:
             self.cache_instance = SmartCacheSystem(
                 redis_client=redis_client,
                 mysql_pool=mysql_pool,
-                instance_name=instance_name
+                instance_name=instance_name,
+                config=config,
+                db_name=db_name,
+                table_name=table_name
             )
             self.initialization_error = None
             self.logger.info("缓存管理器初始化成功", {
                 'instance_name': instance_name,
-                'mysql_enabled': mysql_pool is not None
+                'mysql_enabled': mysql_pool is not None,
+                'db_name': self.cache_instance.config.db_name,
+                'table_name': self.cache_instance.config.table_name
             })
             return self  # 支持链式调用
             

{mdbq-4.0.106 → mdbq-4.0.107}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.106
+Version: 4.0.107
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.106/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.106'