mdbq 4.0.115__py3-none-any.whl → 4.0.117__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.115'
+VERSION = '4.0.117'

mdbq/redis/redis_cache.py

@@ -44,7 +44,7 @@ logger = mylogger.MyLogger(
 
 
 class CacheStatsCollector:
-    """缓存统计收集器 - 单线程同步版本"""
+    """缓存统计收集器 """
     
     def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
         self.enabled = enabled
@@ -67,10 +67,20 @@ class CacheStatsCollector:
         self.response_times = deque(maxlen=1000)
         self.namespace_stats = defaultdict(int)
         
-        # 同步提交控制
+        # 定时提交控制
         self.submit_interval = self.config.get('submit_interval', 600)  # 每隔N秒定时提交一次（期间有新操作时）
         self.last_submit_time = time.time()
         self.last_operation_count = 0  # 上次提交时的操作总数
+        
+        # 后台定时器
+        self._timer = None
+        self._shutdown_event = threading.Event()
+        self._error_count = 0  # 连续错误计数
+        self._max_errors = 5   # 最大连续错误次数
+        
+        # 启动后台定时提交
+        if self.enabled and self.mysql_pool:
+            self._start_background_timer()
 
     def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
         """记录操作统计"""
@@ -101,15 +111,6 @@ class CacheStatsCollector:
                         'operation': operation,
                         'submit_error': str(submit_error)
                     })
-                
-                logger.debug("缓存操作统计已记录", {
-                    'operation': operation,
-                    'response_time_ms': round(response_time, 2),
-                    'namespace': namespace,
-                    'process_id': self.process_id,
-                    'old_total_operations': old_total,
-                    'new_total_operations': self.stats['total_operations']
-                })
         except Exception as e:
             # 统计记录失败不应影响缓存操作
             logger.error("统计记录失败，但缓存操作继续", {
@@ -119,8 +120,64 @@ class CacheStatsCollector:
                 'error': str(e)
             })
     
-    def _check_and_submit(self):
-        """检查并提交统计数据 """
+    def _start_background_timer(self):
+        """启动后台定时提交线程"""
+        if self._timer is not None:
+            return  # 已经启动
+        
+        logger.debug("启动后台定时提交", {
+            'instance_name': self.instance_name,
+            'submit_interval': self.submit_interval
+        })
+        
+        self._timer = threading.Timer(self.submit_interval, self._background_submit)
+        self._timer.daemon = True  # 设置为守护线程
+        self._timer.start()
+    
+    def _background_submit(self):
+        """后台定时提交方法"""
+        if self._shutdown_event.is_set():
+            return  # 已关闭，不再提交
+        
+        try:
+            # 执行提交检查（强制检查，不受时间间隔限制）
+            self._check_and_submit(force_check=True)
+            # 成功执行，重置错误计数
+            self._error_count = 0
+            
+        except Exception as e:
+            self._error_count += 1
+            logger.error("后台定时提交失败", {
+                'instance_name': self.instance_name,
+                'process_id': self.process_id,
+                'error': str(e),
+                'error_type': type(e).__name__,
+                'error_count': self._error_count,
+                'max_errors': self._max_errors
+            })
+            
+            # 如果连续错误次数过多，停止定时器
+            if self._error_count >= self._max_errors:
+                logger.error("后台定时器连续错误过多，停止定时提交", {
+                    'instance_name': self.instance_name,
+                    'process_id': self.process_id,
+                    'error_count': self._error_count
+                })
+                return  # 不再安排下一次定时器
+                
+        finally:
+            # 安排下一次定时提交（仅在未达到最大错误次数时）
+            if not self._shutdown_event.is_set() and self._error_count < self._max_errors:
+                self._timer = threading.Timer(self.submit_interval, self._background_submit)
+                self._timer.daemon = True
+                self._timer.start()
+    
+    def _check_and_submit(self, force_check=False):
+        """检查并提交统计数据
+        
+        Args:
+            force_check: 是否强制检查（用于后台定时器）
+        """
         if not self.mysql_pool:
             return
             
@@ -128,7 +185,9 @@ class CacheStatsCollector:
         time_since_last_submit = current_time - self.last_submit_time
         
         # 提交逻辑：每隔固定秒数且期间有新操作则提交
-        if time_since_last_submit >= self.submit_interval:
+        should_check_time = force_check or time_since_last_submit >= self.submit_interval
+        
+        if should_check_time:
             # 检查是否有新的操作（与上次提交时相比）
             new_operations = self.stats['total_operations'] - self.last_operation_count
             
@@ -139,29 +198,26 @@ class CacheStatsCollector:
                     self.last_submit_time = current_time
                     self.last_operation_count = self.stats['total_operations']
                     
-                    logger.info("统计数据定时提交", {
+                    logger.info("统计数据提交成功", {
                         'instance_name': self.instance_name,
-                        'process_id': self.process_id,
                         'total_operations': self.stats['total_operations'],
                         'new_operations': new_operations,
-                        'time_since_last_submit': round(time_since_last_submit, 2),
-                        'submit_interval': self.submit_interval
+                        'trigger_type': 'background_timer' if force_check else 'operation_triggered'
                     })
                 except Exception as e:
                     logger.error("统计数据提交失败", {
                         'instance_name': self.instance_name,
-                        'process_id': self.process_id,
-                        'error': str(e)
+                        'error': str(e),
+                        'trigger_type': 'background_timer' if force_check else 'operation_triggered'
                     })
             else:
                 # 无新操作，跳过提交但更新时间
                 self.last_submit_time = current_time
-                logger.debug("定时检查：期间无新操作，跳过提交", {
-                    'instance_name': self.instance_name,
-                    'process_id': self.process_id,
-                    'total_operations': self.stats['total_operations'],
-                    'time_since_last_submit': round(time_since_last_submit, 2)
-                })
+                if force_check:  # 仅在后台定时器触发时记录
+                    logger.debug("后台检查：无新操作，跳过提交", {
+                        'instance_name': self.instance_name,
+                        'total_operations': self.stats['total_operations']
+                    })
     
     def _submit_to_mysql(self):
         """同步提交统计数据到MySQL"""
@@ -171,15 +227,17 @@ class CacheStatsCollector:
         stats_data = self.get_stats()
         if not stats_data.get('enabled'):
             return
+        
+        db_name = self.config.get('db_name', 'redis_stats')
+        table_name = self.config.get('table_name', 'cache_performance')
             
         try:
             connection = self.mysql_pool.connection()
             with connection.cursor() as cursor:
                 # 选择数据库
-                cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
+                cursor.execute(f"USE `{db_name}`")
                 
                 # 插入统计数据
-                table_name = self.config.get('table_name', 'cache_performance')
                 insert_sql = f"""
                 INSERT INTO `{table_name}` (
                     `日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
@@ -212,9 +270,10 @@ class CacheStatsCollector:
             connection.close()
             
         except Exception as e:
-            logger.error("MySQL同步提交失败", {
+            logger.error("MySQL提交失败", {
                 'instance_name': self.instance_name,
-                'process_id': self.process_id,
+                'database': db_name,
+                'table': table_name,
                 'error': str(e)
             })
             raise
@@ -246,6 +305,20 @@ class CacheStatsCollector:
                 'process_id': self.process_id
             }
     
+    def shutdown(self):
+        """关闭统计收集器，停止后台定时器"""
+        logger.info("关闭统计收集器", {
+            'instance_name': self.instance_name
+        })
+        
+        # 设置关闭标志
+        self._shutdown_event.set()
+        
+        # 取消定时器
+        if self._timer is not None:
+            self._timer.cancel()
+            self._timer = None
+    
     def reset_stats(self):
         """重置统计数据"""
         if not self.enabled:
@@ -605,7 +678,7 @@ class SmartCacheSystem:
             r'.*table.*|.*column.*|.*field.*': 1200,  # 20分钟
             
             # 数据查询相关 - 变化频率中等，TTL中等
-            r'.*data.*|.*query.*|.*result.*': 300,    # 5分钟
+            r'.*data.*|.*query.*|.*result.*': 600,    # 10分钟
             
             # 用户会话相关 - 变化频率高，TTL较短
             r'.*session.*|.*user.*|.*auth.*': 900,    # 15分钟
@@ -614,7 +687,7 @@ class SmartCacheSystem:
             r'.*config.*|.*setting.*|.*param.*': 3600, # 1小时
             
             # 统计相关 - 变化频率中等，TTL中等
-            r'.*stats.*|.*metric.*|.*count.*': 600,   # 10分钟
+            r'.*stats.*|.*metric.*|.*count.*': 720,   # 12分钟
             
             # 缓存相关 - 变化频率高，TTL较短
             r'.*cache.*|.*temp.*|.*tmp.*': 180,       # 3分钟
@@ -650,12 +723,20 @@ class SmartCacheSystem:
         # 3. 基于数据大小的调整
         size_factor = 1.0
         if data_size > 0:
-            if data_size > 1024 * 1024:  # > 1MB，延长TTL减少重建开销
-                size_factor = 1.5
+            if data_size > 100 * 1024 * 1024:  # > 100MB，延长TTL减少重建开销
+                size_factor = 10.0
+            elif data_size > 20 * 1024 * 1024:  # > 20MB，延长TTL减少重建开销
+                size_factor = 8.0
+            elif data_size > 10 * 1024 * 1024:  # > 10MB，延长TTL减少重建开销
+                size_factor = 6.0
+            elif data_size > 5 * 1024 * 1024:  # > 5MB，延长TTL减少重建开销
+                size_factor = 4.0
+            elif data_size > 1024 * 1024:  # > 1MB，延长TTL减少重建开销
+                size_factor = 3.0
             elif data_size > 100 * 1024:  # > 100KB
-                size_factor = 1.2
+                size_factor = 2.0
             elif data_size < 1024:  # < 1KB，缩短TTL减少内存占用
-                size_factor = 0.8
+                size_factor = 1.0
         
         # 4. 计算基础TTL
         base_ttl = self.config.default_ttl
@@ -760,8 +841,8 @@ class SmartCacheSystem:
         self._state = CacheSystemState.SHUTDOWN
         
         if self.stats_collector:
-            # The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
-            self.stats_collector.reset_stats()
+            # 关闭统计收集器（包括后台定时器）
+            self.stats_collector.shutdown()
         
         logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
 

mdbq/route/monitor.py

@@ -17,10 +17,10 @@ import pymysql
 import hashlib
 import functools
 from datetime import datetime, timedelta
-from typing import Dict, Any, Optional, List
-from urllib.parse import urlparse, parse_qs
-from dbutils.pooled_db import PooledDB
-from mdbq.myconf import myconf
+from typing import Dict, Any
+from urllib.parse import urlparse
+from dbutils.pooled_db import PooledDB # type: ignore
+from mdbq.myconf import myconf # type: ignore
 from mdbq.log import mylogger
 from flask import request, g
 import re
@@ -754,7 +754,20 @@ class RouteMonitor:
                 process_time = round((end_time - start_time) * 1000, 3)
                 
                 response_status = getattr(response, 'status_code', 200) if hasattr(response, 'status_code') else 200
-                response_size = len(str(response.get_data() if hasattr(response, 'get_data') else ''))
+                
+                # 🚀 兼容流式响应：检查是否为直通模式
+                response_size = 0
+                try:
+                    if hasattr(response, 'direct_passthrough') and response.direct_passthrough:
+                        # 流式响应模式，无法获取准确大小，使用估算值
+                        response_size = -1  # 标记为流式响应
+                    elif hasattr(response, 'get_data'):
+                        response_size = len(str(response.get_data()))
+                    else:
+                        response_size = 0
+                except (RuntimeError, Exception):
+                    # 如果获取数据失败（如直通模式），使用默认值
+                    response_size = -1  # 标记为无法获取大小
                 
                 response_data = {
                     'response_status': response_status,

{mdbq-4.0.115.dist-info → mdbq-4.0.117.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: mdbq
-Version: 4.0.115
+Version: 4.0.117
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.115.dist-info → mdbq-4.0.117.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=Rfgqx61BUy_2mTyN4lAP0ll3WI8bxTQ_3sH7MZ4M7bw,19
+mdbq/__version__.py,sha256=PoAOdx_PmyCqgx7t2JxhQUunBadMGsH3LujMSNP6cyM,19
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=-6PyJNwGnZd8BwFGnOn3396tj2fxy1uP5K1f0gC0zow,86562
 mdbq/auth/rate_limiter.py,sha256=1m_Paxp8pDNpmyoFGRpFMVOJpbmeIvfVcfiQ2oH72qM,32850
@@ -26,15 +26,15 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
-mdbq/redis/redis_cache.py,sha256=qecMsJ6mCHVzWFIl_GjeFHYRPVnlNG_vZASd_sHxjDo,32650
+mdbq/redis/redis_cache.py,sha256=2uhAmG9FYHjzByZEg7o8fFFxuF6AnOASVvjdAM4Gx5E,35719
 mdbq/route/__init__.py,sha256=BT_dAY7V-U2o72bevq1B9Mq9QA7GodwtkxyLNdGaoE8,22
 mdbq/route/analytics.py,sha256=dngj5hVwKddEUy59nSYbOoJ9C7OVrtCmCkvW6Uj9RYM,28097
-mdbq/route/monitor.py,sha256=7gLyeq7TqnbhPwhUw0dg-hw9-0OWeKoMdMhcANSDGVs,42255
+mdbq/route/monitor.py,sha256=8uscuoJF4eMr5o8cAqywQv868m1yIGCuK1l4ZWN6KPE,42954
 mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.0.115.dist-info/METADATA,sha256=XaIuaBJCUpKQmxOUa4YU0dXWZRoOdgnd_6CJ5CrTSpg,365
-mdbq-4.0.115.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.115.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.115.dist-info/RECORD,,
+mdbq-4.0.117.dist-info/METADATA,sha256=wTTsk7ZB4zxhXnlAG8PvZ69aXGa63gs7eaNs27sKY8s,365
+mdbq-4.0.117.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mdbq-4.0.117.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.117.dist-info/RECORD,,

{mdbq-4.0.115.dist-info → mdbq-4.0.117.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any