mdbq 4.0.114__tar.gz → 4.0.116__tar.gz

{mdbq-4.0.114 → mdbq-4.0.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.114
+Version: 4.0.116
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.116/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.116'

{mdbq-4.0.114 → mdbq-4.0.116}/mdbq/redis/redis_cache.py

@@ -44,7 +44,7 @@ logger = mylogger.MyLogger(
 
 
 class CacheStatsCollector:
-    """缓存统计收集器 - 单线程同步版本"""
+    """缓存统计收集器 """
     
     def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
         self.enabled = enabled
@@ -67,10 +67,20 @@ class CacheStatsCollector:
         self.response_times = deque(maxlen=1000)
         self.namespace_stats = defaultdict(int)
         
-        # 同步提交控制
-        self.submit_interval = self.config.get('submit_interval', 300)  # 每N次操作提交一次统计数据
+        # 定时提交控制
+        self.submit_interval = self.config.get('submit_interval', 600)  # 每隔N秒定时提交一次（期间有新操作时）
         self.last_submit_time = time.time()
-        self.min_submit_interval = self.config.get('min_submit_interval', 300)  # 最少N秒提交一次
+        self.last_operation_count = 0  # 上次提交时的操作总数
+        
+        # 后台定时器
+        self._timer = None
+        self._shutdown_event = threading.Event()
+        self._error_count = 0  # 连续错误计数
+        self._max_errors = 5   # 最大连续错误次数
+        
+        # 启动后台定时提交
+        if self.enabled and self.mysql_pool:
+            self._start_background_timer()
 
     def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
         """记录操作统计"""
@@ -101,15 +111,6 @@ class CacheStatsCollector:
                         'operation': operation,
                         'submit_error': str(submit_error)
                     })
-                
-                logger.debug("缓存操作统计已记录", {
-                    'operation': operation,
-                    'response_time_ms': round(response_time, 2),
-                    'namespace': namespace,
-                    'process_id': self.process_id,
-                    'old_total_operations': old_total,
-                    'new_total_operations': self.stats['total_operations']
-                })
         except Exception as e:
             # 统计记录失败不应影响缓存操作
             logger.error("统计记录失败，但缓存操作继续", {
@@ -119,69 +120,104 @@ class CacheStatsCollector:
                 'error': str(e)
             })
     
-    def _check_and_submit(self):
-        """检查并提交统计数据（同步）"""
+    def _start_background_timer(self):
+        """启动后台定时提交线程"""
+        if self._timer is not None:
+            return  # 已经启动
+        
+        logger.debug("启动后台定时提交", {
+            'instance_name': self.instance_name,
+            'submit_interval': self.submit_interval
+        })
+        
+        self._timer = threading.Timer(self.submit_interval, self._background_submit)
+        self._timer.daemon = True  # 设置为守护线程
+        self._timer.start()
+    
+    def _background_submit(self):
+        """后台定时提交方法"""
+        if self._shutdown_event.is_set():
+            return  # 已关闭，不再提交
+        
+        try:
+            # 执行提交检查（强制检查，不受时间间隔限制）
+            self._check_and_submit(force_check=True)
+            # 成功执行，重置错误计数
+            self._error_count = 0
+            
+        except Exception as e:
+            self._error_count += 1
+            logger.error("后台定时提交失败", {
+                'instance_name': self.instance_name,
+                'process_id': self.process_id,
+                'error': str(e),
+                'error_type': type(e).__name__,
+                'error_count': self._error_count,
+                'max_errors': self._max_errors
+            })
+            
+            # 如果连续错误次数过多，停止定时器
+            if self._error_count >= self._max_errors:
+                logger.error("后台定时器连续错误过多，停止定时提交", {
+                    'instance_name': self.instance_name,
+                    'process_id': self.process_id,
+                    'error_count': self._error_count
+                })
+                return  # 不再安排下一次定时器
+                
+        finally:
+            # 安排下一次定时提交（仅在未达到最大错误次数时）
+            if not self._shutdown_event.is_set() and self._error_count < self._max_errors:
+                self._timer = threading.Timer(self.submit_interval, self._background_submit)
+                self._timer.daemon = True
+                self._timer.start()
+    
+    def _check_and_submit(self, force_check=False):
+        """检查并提交统计数据
+        
+        Args:
+            force_check: 是否强制检查（用于后台定时器）
+        """
         if not self.mysql_pool:
             return
             
         current_time = time.time()
         time_since_last_submit = current_time - self.last_submit_time
-        should_submit = False
-        reason = ""
-        
-        # 重构后的提交逻辑：
-        # 1. 每N次操作提交一次，但每X秒内只能提交一次（防止频繁提交）
-        # 2. 每隔X秒提交一次，但必须满足N次操作
-        
-        if self.stats['total_operations'] % self.submit_interval == 0:
-            # 达到操作次数阈值
-            if time_since_last_submit >= self.min_submit_interval:
-                # 满足时间间隔要求，可以提交
-                should_submit = True
-                reason = f"达到操作次数阈值({self.submit_interval}次)且满足时间间隔({self.min_submit_interval}秒)"
-            else:
-                # 时间间隔不够，延迟提交
-                remaining_time = self.min_submit_interval - time_since_last_submit
-                logger.debug("操作次数达标但时间间隔不足，延迟提交", {
-                    'instance_name': self.instance_name,
-                    'process_id': self.process_id,
-                    'total_operations': self.stats['total_operations'],
-                    'time_since_last_submit': round(time_since_last_submit, 2),
-                    'remaining_time': round(remaining_time, 2)
-                })
-        elif time_since_last_submit >= self.min_submit_interval:
-            # 达到时间间隔阈值
-            if self.stats['total_operations'] >= self.submit_interval:
-                # 满足操作次数要求，可以提交
-                should_submit = True
-                reason = f"达到时间间隔阈值({self.min_submit_interval}秒)且满足操作次数({self.submit_interval}次)"
-            else:
-                # 操作次数不够，不提交
-                logger.debug("时间间隔达标但操作次数不足，暂不提交", {
-                    'instance_name': self.instance_name,
-                    'process_id': self.process_id,
-                    'total_operations': self.stats['total_operations'],
-                    'required_operations': self.submit_interval,
-                    'time_since_last_submit': round(time_since_last_submit, 2)
-                })
         
-        if should_submit and self.stats['total_operations'] > 0:
-            try:
-                self._submit_to_mysql()
+        # 提交逻辑：每隔固定秒数且期间有新操作则提交
+        should_check_time = force_check or time_since_last_submit >= self.submit_interval
+        
+        if should_check_time:
+            # 检查是否有新的操作（与上次提交时相比）
+            new_operations = self.stats['total_operations'] - self.last_operation_count
+            
+            if new_operations > 0:
+                # 有新操作，提交统计数据
+                try:
+                    self._submit_to_mysql()
+                    self.last_submit_time = current_time
+                    self.last_operation_count = self.stats['total_operations']
+                    
+                    logger.info("统计数据提交成功", {
+                        'instance_name': self.instance_name,
+                        'total_operations': self.stats['total_operations'],
+                        'new_operations': new_operations,
+                        'trigger_type': 'background_timer' if force_check else 'operation_triggered'
+                    })
+                except Exception as e:
+                    logger.error("统计数据提交失败", {
+                        'instance_name': self.instance_name,
+                        'error': str(e),
+                        'trigger_type': 'background_timer' if force_check else 'operation_triggered'
+                    })
+            else:
+                # 无新操作，跳过提交但更新时间
                 self.last_submit_time = current_time
-                logger.info("统计数据已同步提交", {
-                    'instance_name': self.instance_name,
-                    'process_id': self.process_id,
-                    'total_operations': self.stats['total_operations'],
-                    'time_since_last_submit': round(time_since_last_submit, 2),
-                    'reason': reason
-                })
-            except Exception as e:
-                logger.error("统计数据提交失败", {
-                    'instance_name': self.instance_name,
-                    'process_id': self.process_id,
-                    'error': str(e)
-                })
+                if force_check:  # 仅在后台定时器触发时记录
+                    logger.debug("后台检查：无新操作，跳过提交", {
+                        'instance_name': self.instance_name,
+                        'total_operations': self.stats['total_operations']
+                    })
     
     def _submit_to_mysql(self):
         """同步提交统计数据到MySQL"""
@@ -191,15 +227,17 @@ class CacheStatsCollector:
         stats_data = self.get_stats()
         if not stats_data.get('enabled'):
             return
+        
+        db_name = self.config.get('db_name', 'redis_stats')
+        table_name = self.config.get('table_name', 'cache_performance')
             
         try:
             connection = self.mysql_pool.connection()
             with connection.cursor() as cursor:
                 # 选择数据库
-                cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
+                cursor.execute(f"USE `{db_name}`")
                 
                 # 插入统计数据
-                table_name = self.config.get('table_name', 'cache_performance')
                 insert_sql = f"""
                 INSERT INTO `{table_name}` (
                     `日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
@@ -232,9 +270,10 @@ class CacheStatsCollector:
             connection.close()
             
         except Exception as e:
-            logger.error("MySQL同步提交失败", {
+            logger.error("MySQL提交失败", {
                 'instance_name': self.instance_name,
-                'process_id': self.process_id,
+                'database': db_name,
+                'table': table_name,
                 'error': str(e)
             })
             raise
@@ -266,6 +305,20 @@ class CacheStatsCollector:
                 'process_id': self.process_id
             }
     
+    def shutdown(self):
+        """关闭统计收集器，停止后台定时器"""
+        logger.info("关闭统计收集器", {
+            'instance_name': self.instance_name
+        })
+        
+        # 设置关闭标志
+        self._shutdown_event.set()
+        
+        # 取消定时器
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+    
     def reset_stats(self):
         """重置统计数据"""
         if not self.enabled:
@@ -284,6 +337,7 @@ class CacheStatsCollector:
             self.response_times.clear()
             self.namespace_stats.clear()
             self.last_submit_time = time.time()
+            self.last_operation_count = 0
 
 
 class CacheSystemState(enum.Enum):
@@ -300,8 +354,7 @@ class CacheConfig:
     def __init__(self, **kwargs):
         # 基础配置
         self.default_ttl = kwargs.get('default_ttl', 3600)  # 默认过期时间(秒)
-        self.stats_interval = kwargs.get('stats_interval', 300)  # 每N次操作提交一次统计数据
-        self.stats_initial_delay = kwargs.get('stats_initial_delay', 300)  # 最少N秒提交一次
+        self.stats_submit_interval = kwargs.get('stats_submit_interval', 600)  # 每隔N秒定时提交一次（期间有新操作时）
         self.enable_stats = kwargs.get('enable_stats', True)  # 是否启用统计功能
         self.max_value_size = kwargs.get('max_value_size', 10 * 1024 * 1024)  # 最大值大小(字节)
         self.cache_prefix = kwargs.get('cache_prefix', 'cache')  # 缓存键前缀
@@ -343,8 +396,7 @@ class SmartCacheSystem:
                 mysql_pool=self.mysql_pool,
                 config={
                     'instance_name': self.instance_name,
-                    'submit_interval': self.config.stats_interval,  # 每N次操作提交一次
-                    'min_submit_interval': 300,  # 最少N秒提交一次
+                    'submit_interval': self.config.stats_submit_interval,  # 每隔N秒定时提交一次（期间有新操作时）
                     'table_name': self.config.table_name,
                     'db_name': self.config.db_name
                 }
@@ -781,8 +833,8 @@ class SmartCacheSystem:
         self._state = CacheSystemState.SHUTDOWN
         
         if self.stats_collector:
-            # The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
-            self.stats_collector.reset_stats()
+            # 关闭统计收集器（包括后台定时器）
+            self.stats_collector.shutdown()
         
         logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
 

{mdbq-4.0.114 → mdbq-4.0.116}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.114
+Version: 4.0.116
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.114/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.114'