mdbq 4.0.113__py3-none-any.whl → 4.0.115__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mdbq might be problematic. Click here for more details.

mdbq/redis/redis_cache.py CHANGED
@@ -5,30 +5,31 @@ Redis智能缓存系统
5
5
  主要功能:
6
6
  1. Redis缓存的CRUD操作
7
7
  2. 命名空间隔离
8
- 3. 分布式锁防止缓存击穿
9
- 4. 自动统计分析并提交到MySQL
8
+ 3. 智能TTL策略
9
+ 4. 专业统计分析并提交到MySQL
10
10
  5. 缓存健康检查和监控
11
-
11
+ 6. 热点键分析
12
+ 7. 响应时间分析
13
+ 8. 业务指标计算
12
14
  """
13
15
 
14
16
  import json
15
17
  import time
16
18
  import threading
17
19
  import socket
18
- from datetime import datetime, date
19
- from decimal import Decimal
20
- from uuid import UUID
21
- from typing import Optional, Dict, Any, List, Union, Callable
20
+ import os
21
+ import statistics
22
+ import enum
23
+ import re
24
+ from datetime import datetime, timedelta
25
+ from typing import Optional, Dict, Any, List, Callable
26
+ from threading import Event
27
+ from collections import defaultdict, deque
22
28
  import redis
23
29
  from mdbq.log import mylogger
24
30
 
25
- import enum
26
- from typing import Optional, Dict, Any, List, Union, Callable
27
- from concurrent.futures import ThreadPoolExecutor
28
- import asyncio
29
- from threading import Event
30
- import functools
31
31
 
32
+ # 全局日志器
32
33
  logger = mylogger.MyLogger(
33
34
  logging_mode='file',
34
35
  log_level='info',
@@ -42,259 +43,319 @@ logger = mylogger.MyLogger(
42
43
  )
43
44
 
44
45
 
45
- class CacheSystemState(enum.Enum):
46
- """缓存系统状态枚举"""
47
- INITIALIZING = "initializing"
48
- READY = "ready"
49
- MYSQL_READY = "mysql_ready"
50
- ERROR = "error"
51
-
46
+ class CacheStatsCollector:
47
+ """缓存统计收集器 - 单线程同步版本"""
48
+
49
+ def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
50
+ self.enabled = enabled
51
+ self.mysql_pool = mysql_pool
52
+ self.config = config or {}
53
+ self.process_id = os.getpid()
54
+ self.instance_name = self.config.get('instance_name', 'default')
55
+
56
+ # 统计数据
57
+ self.stats = {
58
+ 'hits': 0,
59
+ 'misses': 0,
60
+ 'sets': 0,
61
+ 'deletes': 0,
62
+ 'errors': 0,
63
+ 'total_operations': 0,
64
+ 'start_time': time.time()
65
+ }
66
+ self._lock = threading.RLock()
67
+ self.response_times = deque(maxlen=1000)
68
+ self.namespace_stats = defaultdict(int)
69
+
70
+ # 同步提交控制
71
+ self.submit_interval = self.config.get('submit_interval', 600) # 每隔N秒定时提交一次(期间有新操作时)
72
+ self.last_submit_time = time.time()
73
+ self.last_operation_count = 0 # 上次提交时的操作总数
52
74
 
53
- class MySQLDataEncoder(json.JSONEncoder):
54
- """自定义JSON编码器,支持MySQL常见数据类型"""
55
- def default(self, obj):
75
+ def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
76
+ """记录操作统计"""
77
+ if not self.enabled:
78
+ return
79
+
56
80
  try:
57
- if isinstance(obj, datetime):
58
- return obj.isoformat()
59
- elif isinstance(obj, date):
60
- return obj.isoformat()
61
- elif isinstance(obj, Decimal):
62
- # Decimal转换为字符串保持精度
63
- return str(obj)
64
- elif isinstance(obj, UUID):
65
- return str(obj)
66
- elif isinstance(obj, bytes):
67
- # 二进制数据转换为base64
68
- import base64
69
- return base64.b64encode(obj).decode('utf-8')
70
- elif isinstance(obj, (set, frozenset)):
71
- # 集合类型转换为列表
72
- return list(obj)
73
- elif hasattr(obj, 'isoformat'):
74
- # 其他日期时间类型
75
- return obj.isoformat()
76
- elif hasattr(obj, '__float__'):
77
- # 数值类型
78
- return float(obj)
79
- elif hasattr(obj, '__str__'):
80
- # 其他有字符串表示的对象
81
- return str(obj)
82
- elif hasattr(obj, '__dict__'):
83
- # 处理其他对象,转换为字典
84
- return obj.__dict__
81
+ with self._lock:
82
+ old_total = self.stats['total_operations']
83
+
84
+ self.stats['total_operations'] += 1
85
+ self.stats[operation] = self.stats.get(operation, 0) + 1
86
+
87
+ if response_time > 0:
88
+ self.response_times.append(response_time)
89
+
90
+ if namespace:
91
+ self.namespace_stats[namespace] += 1
92
+
93
+ # 检查是否需要提交统计数据(容错处理)
94
+ try:
95
+ self._check_and_submit()
96
+ except Exception as submit_error:
97
+ # 统计提交失败不应影响统计记录
98
+ logger.error("统计数据提交检查失败,但统计记录继续", {
99
+ 'instance_name': self.instance_name,
100
+ 'process_id': self.process_id,
101
+ 'operation': operation,
102
+ 'submit_error': str(submit_error)
103
+ })
104
+
105
+ logger.debug("缓存操作统计已记录", {
106
+ 'operation': operation,
107
+ 'response_time_ms': round(response_time, 2),
108
+ 'namespace': namespace,
109
+ 'process_id': self.process_id,
110
+ 'old_total_operations': old_total,
111
+ 'new_total_operations': self.stats['total_operations']
112
+ })
85
113
  except Exception as e:
86
- # 如果所有方法都失败,返回对象的字符串表示
87
- return f"<{type(obj).__name__}: {str(obj)[:100]}>"
88
-
89
- return super().default(obj)
90
-
91
-
92
- class SmartTTLConfig:
93
- """智能TTL配置策略"""
94
-
95
- # TTL策略映射表(秒)
96
- TTL_STRATEGIES = {
97
- # 数据库相关缓存
98
- 'sycm_database': 1800, # 数据库列表:30分钟
99
- 'sycm_tables': 1200, # 表列表:20分钟
100
- 'sycm_table_data': 300, # 表数据:5分钟
101
-
102
- # 用户会话相关
103
- 'user_session': 3600, # 用户会话:1小时
104
- 'user_info': 1800, # 用户信息:30分钟
105
-
106
- # 静态配置
107
- 'static_config': 86400, # 静态配置:24小时
108
- 'system_config': 43200, # 系统配置:12小时
109
-
110
- # API相关
111
- 'api_response': 600, # API响应:10分钟
112
- 'api_cache': 900, # API缓存:15分钟
113
-
114
- # 临时数据
115
- 'temp_data': 300, # 临时数据:5分钟
116
- 'short_cache': 60, # 短期缓存:1分钟
117
- }
114
+ # 统计记录失败不应影响缓存操作
115
+ logger.error("统计记录失败,但缓存操作继续", {
116
+ 'instance_name': self.instance_name,
117
+ 'process_id': self.process_id,
118
+ 'operation': operation,
119
+ 'error': str(e)
120
+ })
118
121
 
119
- @classmethod
120
- def get_ttl(cls, namespace: str, default: int = 3600) -> int:
121
- """
122
- 根据命名空间获取对应的TTL值
122
+ def _check_and_submit(self):
123
+ """检查并提交统计数据 """
124
+ if not self.mysql_pool:
125
+ return
126
+
127
+ current_time = time.time()
128
+ time_since_last_submit = current_time - self.last_submit_time
123
129
 
124
- Args:
125
- namespace: 缓存命名空间
126
- default: 默认TTL值(秒)
130
+ # 提交逻辑:每隔固定秒数且期间有新操作则提交
131
+ if time_since_last_submit >= self.submit_interval:
132
+ # 检查是否有新的操作(与上次提交时相比)
133
+ new_operations = self.stats['total_operations'] - self.last_operation_count
127
134
 
128
- Returns:
129
- int: TTL值(秒)
130
- """
131
- return cls.TTL_STRATEGIES.get(namespace, default)
135
+ if new_operations > 0:
136
+ # 有新操作,提交统计数据
137
+ try:
138
+ self._submit_to_mysql()
139
+ self.last_submit_time = current_time
140
+ self.last_operation_count = self.stats['total_operations']
141
+
142
+ logger.info("统计数据定时提交", {
143
+ 'instance_name': self.instance_name,
144
+ 'process_id': self.process_id,
145
+ 'total_operations': self.stats['total_operations'],
146
+ 'new_operations': new_operations,
147
+ 'time_since_last_submit': round(time_since_last_submit, 2),
148
+ 'submit_interval': self.submit_interval
149
+ })
150
+ except Exception as e:
151
+ logger.error("统计数据提交失败", {
152
+ 'instance_name': self.instance_name,
153
+ 'process_id': self.process_id,
154
+ 'error': str(e)
155
+ })
156
+ else:
157
+ # 无新操作,跳过提交但更新时间
158
+ self.last_submit_time = current_time
159
+ logger.debug("定时检查:期间无新操作,跳过提交", {
160
+ 'instance_name': self.instance_name,
161
+ 'process_id': self.process_id,
162
+ 'total_operations': self.stats['total_operations'],
163
+ 'time_since_last_submit': round(time_since_last_submit, 2)
164
+ })
132
165
 
133
- @classmethod
134
- def add_strategy(cls, namespace: str, ttl: int):
135
- """
136
- 添加新的TTL策略
166
+ def _submit_to_mysql(self):
167
+ """同步提交统计数据到MySQL"""
168
+ if not self.mysql_pool:
169
+ return
170
+
171
+ stats_data = self.get_stats()
172
+ if not stats_data.get('enabled'):
173
+ return
174
+
175
+ try:
176
+ connection = self.mysql_pool.connection()
177
+ with connection.cursor() as cursor:
178
+ # 选择数据库
179
+ cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
180
+
181
+ # 插入统计数据
182
+ table_name = self.config.get('table_name', 'cache_performance')
183
+ insert_sql = f"""
184
+ INSERT INTO `{table_name}` (
185
+ `日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
186
+ `缓存命中`, `缓存未命中`, `缓存设置`, `缓存删除`, `缓存错误`, `总操作数`,
187
+ `命中率`, `平均响应时间`, `运行时间`, `命名空间统计`
188
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
189
+ """
190
+
191
+ insert_data = (
192
+ datetime.now().strftime('%Y-%m-%d'),
193
+ f"{self.instance_name}_pid_{self.process_id}",
194
+ socket.gethostname(),
195
+ self.process_id,
196
+ datetime.now(),
197
+ stats_data['hits'],
198
+ stats_data['misses'],
199
+ stats_data['sets'],
200
+ stats_data['deletes'],
201
+ stats_data['errors'],
202
+ stats_data['total_operations'],
203
+ stats_data['hit_rate_percent'],
204
+ stats_data['avg_response_time_ms'],
205
+ stats_data['uptime_seconds'],
206
+ json.dumps(stats_data['namespace_stats'], ensure_ascii=False)
207
+ )
208
+
209
+ cursor.execute(insert_sql, insert_data)
210
+ connection.commit()
211
+
212
+ connection.close()
213
+
214
+ except Exception as e:
215
+ logger.error("MySQL同步提交失败", {
216
+ 'instance_name': self.instance_name,
217
+ 'process_id': self.process_id,
218
+ 'error': str(e)
219
+ })
220
+ raise
221
+
222
+ def get_stats(self) -> Dict[str, Any]:
223
+ """获取统计数据"""
224
+ if not self.enabled:
225
+ return {'enabled': False, 'message': '统计功能已禁用'}
137
226
 
138
- Args:
139
- namespace: 命名空间
140
- ttl: TTL值(秒)
141
- """
142
- cls.TTL_STRATEGIES[namespace] = ttl
227
+ with self._lock:
228
+ uptime = time.time() - self.stats['start_time']
229
+ total_cache_ops = self.stats['hits'] + self.stats['misses']
230
+ hit_rate = (self.stats['hits'] / total_cache_ops * 100) if total_cache_ops > 0 else 0
231
+ avg_response_time = statistics.mean(self.response_times) if self.response_times else 0
232
+
233
+ return {
234
+ 'enabled': True,
235
+ 'hits': self.stats['hits'],
236
+ 'misses': self.stats['misses'],
237
+ 'sets': self.stats['sets'],
238
+ 'deletes': self.stats['deletes'],
239
+ 'errors': self.stats['errors'],
240
+ 'total_operations': self.stats['total_operations'],
241
+ 'hit_rate_percent': round(hit_rate, 2),
242
+ 'avg_response_time_ms': round(avg_response_time, 2),
243
+ 'uptime_seconds': round(uptime, 2),
244
+ 'namespace_stats': dict(self.namespace_stats),
245
+ 'last_updated': datetime.now().isoformat(),
246
+ 'process_id': self.process_id
247
+ }
143
248
 
144
- @classmethod
145
- def get_all_strategies(cls) -> Dict[str, int]:
146
- """获取所有TTL策略"""
147
- return cls.TTL_STRATEGIES.copy()
249
+ def reset_stats(self):
250
+ """重置统计数据"""
251
+ if not self.enabled:
252
+ return
253
+
254
+ with self._lock:
255
+ self.stats = {
256
+ 'hits': 0,
257
+ 'misses': 0,
258
+ 'sets': 0,
259
+ 'deletes': 0,
260
+ 'errors': 0,
261
+ 'total_operations': 0,
262
+ 'start_time': time.time()
263
+ }
264
+ self.response_times.clear()
265
+ self.namespace_stats.clear()
266
+ self.last_submit_time = time.time()
267
+ self.last_operation_count = 0
268
+
269
+
270
+ class CacheSystemState(enum.Enum):
271
+ """缓存系统状态枚举"""
272
+ INITIALIZING = "initializing"
273
+ READY = "ready"
274
+ MYSQL_READY = "mysql_ready"
275
+ ERROR = "error"
276
+ SHUTDOWN = "shutdown"
148
277
 
149
278
 
150
279
  class CacheConfig:
151
- """缓存系统配置类"""
152
-
153
- def __init__(self, db_name: str = "redis统计", table_name: str = "dpflask路由分析"):
154
- # TTL配置(秒)
155
- self.default_ttl = 3600 # 1小时
156
- self.short_ttl = 300 # 5分钟
157
- self.medium_ttl = 1800 # 30分钟
158
- self.long_ttl = 7200 # 2小时
159
- self.very_long_ttl = 86400 # 24小时
160
-
161
- # 缓存键前缀
162
- self.cache_prefix = "smart_cache:"
163
- self.stats_prefix = "cache_stats:"
164
- self.lock_prefix = "cache_lock:"
165
-
166
- # 统计配置
167
- self.stats_interval = 1800 # 统计间隔(秒), 自动提交统计信息到MySQL的间隔
168
- self.stats_retention = 7 # MySQL统计数据保留天数,超过此天数的数据将被自动删除
169
-
170
- # 性能配置
171
- self.max_key_length = 250
172
- self.max_value_size = 1024 * 1024 # 1MB
173
- self.batch_size = 100
174
-
175
- # 热点键配置
176
- self.max_hot_keys = 1000 # 最大热点键数量
177
- self.hot_keys_cleanup_threshold = 800 # 热点键清理阈值
178
-
179
- # MySQL数据库配置
180
- self.db_name = db_name
181
- self.table_name = table_name
182
-
183
- # 锁配置
184
- self.lock_timeout = 30 # 分布式锁超时时间
185
- self.lock_retry_delay = 0.1 # 锁重试延迟
280
+ """缓存配置类"""
281
+ def __init__(self, **kwargs):
282
+ # 基础配置
283
+ self.default_ttl = kwargs.get('default_ttl', 3600) # 默认过期时间(秒)
284
+ self.stats_submit_interval = kwargs.get('stats_submit_interval', 600) # 每隔N秒定时提交一次(期间有新操作时)
285
+ self.enable_stats = kwargs.get('enable_stats', True) # 是否启用统计功能
286
+ self.max_value_size = kwargs.get('max_value_size', 10 * 1024 * 1024) # 最大值大小(字节)
287
+ self.cache_prefix = kwargs.get('cache_prefix', 'cache') # 缓存键前缀
288
+ self.enable_compression = kwargs.get('enable_compression', True)
289
+
290
+ # 数据库配置
291
+ self.db_name = kwargs.get('db_name', 'redis_stats')
292
+ self.table_name = kwargs.get('table_name', 'cache_performance')
293
+
294
+ # 智能TTL策略配置
295
+ self.enable_smart_ttl = kwargs.get('enable_smart_ttl', True) # 启用智能TTL
296
+ self.ttl_min = kwargs.get('ttl_min', 60) # 最小TTL(秒)
297
+ self.ttl_max = kwargs.get('ttl_max', 7200) # 最大TTL(秒)
298
+ self.debug_ttl = kwargs.get('debug_ttl', False) # TTL调试模式
299
+
300
+ # 自定义TTL模式
301
+ self.custom_namespace_patterns = kwargs.get('custom_namespace_patterns', {})
302
+ self.custom_key_patterns = kwargs.get('custom_key_patterns', {})
186
303
 
187
304
 
188
305
  class SmartCacheSystem:
189
- """智能缓存系统核心类"""
306
+ """智能缓存系统 - 单线程"""
190
307
 
191
- def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
192
- config: CacheConfig = None, db_name: str = None, table_name: str = None):
308
+ def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
193
309
  self.redis_client = redis_client
194
310
  self.mysql_pool = mysql_pool
195
311
  self.instance_name = instance_name
312
+ self.config = CacheConfig(**config)
196
313
 
197
- # 配置优先级:传入的config > 自定义db_name/table_name > 默认配置
198
- if config:
199
- self.config = config
200
- elif db_name or table_name:
201
- self.config = CacheConfig(
202
- db_name=db_name or "redis统计",
203
- table_name=table_name or "dpflask路由分析"
204
- )
205
- else:
206
- self.config = CacheConfig()
207
-
208
- self.logger = logger
209
-
210
- # 状态管理
314
+ # 系统状态管理
211
315
  self._state = CacheSystemState.INITIALIZING
212
316
  self._ready_event = Event()
213
- self._mysql_ready_event = Event()
214
-
215
- # 统计数据
216
- self.stats = {
217
- 'hits': 0,
218
- 'misses': 0,
219
- 'sets': 0,
220
- 'deletes': 0,
221
- 'errors': 0,
222
- 'total_operations': 0,
223
- 'start_time': time.time(),
224
- 'response_times': []
225
- }
226
-
227
- # 热点键统计
228
- self.hot_keys = {}
229
- self.hot_keys_lock = threading.RLock()
230
317
 
231
- # 统计线程控制
232
- self._stats_running = False
233
- self._stats_thread = None
234
- self._stats_lock = threading.RLock()
235
-
236
- # 使用线程池异步初始化
237
- self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=f"cache_{instance_name}")
318
+ # 直接初始化统计系统
319
+ self.stats_collector = None
320
+ if self.config.enable_stats:
321
+ self.stats_collector = CacheStatsCollector(
322
+ enabled=True,
323
+ mysql_pool=self.mysql_pool,
324
+ config={
325
+ 'instance_name': self.instance_name,
326
+ 'submit_interval': self.config.stats_submit_interval, # 每隔N秒定时提交一次(期间有新操作时)
327
+ 'table_name': self.config.table_name,
328
+ 'db_name': self.config.db_name
329
+ }
330
+ )
238
331
 
239
- # 立即设置为基础就绪状态(Redis缓存可用)
332
+ # 初始化系统
333
+ self._initialize()
334
+
335
+ def _initialize(self):
336
+ """初始化缓存系统"""
337
+ # 测试Redis连接
240
338
  if self._test_redis_connection():
241
339
  self._state = CacheSystemState.READY
242
340
  self._ready_event.set()
243
-
244
- # 异步初始化MySQL相关功能
245
- self._executor.submit(self._initialize_mysql_features)
246
-
247
- self.logger.info("智能缓存系统初始化完成", {
248
- 'instance_name': self.instance_name,
249
- 'state': self._state.value,
250
- 'redis_ready': self._ready_event.is_set(),
251
- 'mysql_enabled': self.mysql_pool is not None
252
- })
253
-
254
- def _initialize_mysql_features(self):
255
- """在后台初始化MySQL相关功能"""
256
- try:
257
- if self.mysql_pool:
258
- # 延迟初始化MySQL(给系统一些启动时间)
259
- time.sleep(5) # 等待5秒让主系统完全启动
260
-
261
- if self._init_mysql_db():
341
+
342
+ # 如果启用统计且有MySQL连接,创建统计表
343
+ if self.config.enable_stats and self.mysql_pool:
344
+ try:
345
+ self._create_simple_stats_table()
262
346
  self._state = CacheSystemState.MYSQL_READY
263
- self._mysql_ready_event.set()
264
-
265
- # MySQL就绪后再启动统计线程
266
- self._start_stats_worker()
267
-
268
- self.logger.info("MySQL功能初始化完成", {
347
+ logger.info("统计功能已启用", {
269
348
  'instance_name': self.instance_name,
270
- 'state': self._state.value
349
+ 'process_id': os.getpid()
271
350
  })
272
- else:
273
- self.logger.warning("MySQL初始化失败,缓存功能仍可正常使用")
274
- else:
275
- self.logger.info("未配置MySQL,跳过统计功能")
276
-
277
- except Exception as e:
351
+ except Exception as e:
352
+ logger.error("统计表创建失败", {
353
+ 'instance_name': self.instance_name,
354
+ 'error': str(e)
355
+ })
356
+ else:
278
357
  self._state = CacheSystemState.ERROR
279
- self.logger.error(f"MySQL功能初始化异常: {e}")
280
-
281
- @property
282
- def is_ready(self) -> bool:
283
- """检查缓存系统是否就绪(Redis可用即为就绪)"""
284
- return self._ready_event.is_set()
285
-
286
- @property
287
- def is_mysql_ready(self) -> bool:
288
- """检查MySQL功能是否就绪"""
289
- return self._mysql_ready_event.is_set()
290
-
291
- def wait_for_ready(self, timeout: float = 1.0) -> bool:
292
- """等待缓存系统就绪(非阻塞,有超时)"""
293
- return self._ready_event.wait(timeout)
294
-
295
- def wait_for_mysql_ready(self, timeout: float = 5.0) -> bool:
296
- """等待MySQL功能就绪(非阻塞,有超时)"""
297
- return self._mysql_ready_event.wait(timeout)
358
+ logger.error("Redis连接失败", {'instance_name': self.instance_name})
298
359
 
299
360
  def _test_redis_connection(self) -> bool:
300
361
  """测试Redis连接"""
@@ -302,652 +363,470 @@ class SmartCacheSystem:
302
363
  self.redis_client.ping()
303
364
  return True
304
365
  except Exception as e:
305
- self.logger.error(f"Redis连接测试失败: {e}")
366
+ logger.error("Redis连接测试失败", {'error': str(e)})
306
367
  return False
307
368
 
308
- def _init_mysql_db(self) -> bool:
309
- """初始化MySQL数据库和表"""
369
+ def _create_simple_stats_table(self):
370
+ """创建统计表"""
310
371
  if not self.mysql_pool:
311
- self.logger.warning("MySQL连接池未提供,统计功能将被禁用")
312
- return False
372
+ return
313
373
 
314
374
  try:
315
- # 设置连接超时,避免长时间阻塞
316
375
  connection = self.mysql_pool.connection()
317
- # 设置查询超时(如果支持)
318
- try:
319
- connection.autocommit(False) # 确保事务控制
320
- except:
321
- pass # 忽略不支持的操作
376
+ with connection.cursor() as cursor:
377
+ # 创建数据库
378
+ cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci")
379
+ cursor.execute(f"USE `{self.config.db_name}`")
322
380
 
323
- try:
324
- with connection.cursor() as cursor:
325
- # 设置会话超时
326
- try:
327
- cursor.execute("SET SESSION wait_timeout = 30")
328
- cursor.execute("SET SESSION interactive_timeout = 30")
329
- except:
330
- pass # 忽略不支持的操作
381
+ # 创建统计表
382
+ create_table_sql = f"""
383
+ CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
384
+ `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
385
+ `日期` date NOT NULL COMMENT '统计日期',
386
+ `实例标识` varchar(64) NOT NULL COMMENT '缓存实例标识',
387
+ `主机名` varchar(100) NOT NULL COMMENT '服务器主机名',
388
+ `进程ID` int NOT NULL COMMENT '进程ID',
389
+ `统计时间` timestamp NOT NULL COMMENT '统计时间',
331
390
 
332
- # 创建数据库
333
- cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
334
- cursor.execute(f"USE `{self.config.db_name}`")
391
+ -- 基础操作统计
392
+ `缓存命中` bigint DEFAULT 0 COMMENT '缓存命中次数',
393
+ `缓存未命中` bigint DEFAULT 0 COMMENT '缓存未命中次数',
394
+ `缓存设置` bigint DEFAULT 0 COMMENT '缓存设置次数',
395
+ `缓存删除` bigint DEFAULT 0 COMMENT '缓存删除次数',
396
+ `缓存错误` bigint DEFAULT 0 COMMENT '缓存错误次数',
397
+ `总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
335
398
 
336
- # 创建表(MySQL 8.4+兼容语法)
337
- create_table_sql = f"""
338
- CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
339
- `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
340
- `日期` date NOT NULL COMMENT '统计日期',
341
- `统计时间` datetime NOT NULL COMMENT '统计时间',
342
- `时间段` varchar(20) NOT NULL COMMENT '时间段标识',
343
- `缓存命中数` bigint DEFAULT 0 COMMENT '缓存命中次数',
344
- `缓存未命中数` bigint DEFAULT 0 COMMENT '缓存未命中次数',
345
- `缓存设置数` bigint DEFAULT 0 COMMENT '缓存设置次数',
346
- `缓存删除数` bigint DEFAULT 0 COMMENT '缓存删除次数',
347
- `缓存错误数` bigint DEFAULT 0 COMMENT '缓存错误次数',
348
- `命中率` decimal(5,2) DEFAULT 0.00 COMMENT '缓存命中率(%)',
349
- `总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
350
- `平均响应时间` decimal(10,4) DEFAULT 0.0000 COMMENT '平均响应时间(ms)',
351
- `每秒操作数` decimal(10,2) DEFAULT 0.00 COMMENT '每秒操作数',
352
- `唯一键数量` int DEFAULT 0 COMMENT '唯一键数量',
353
- `系统运行时间` bigint DEFAULT 0 COMMENT '系统运行时间(秒)',
354
- `热点键统计` json DEFAULT NULL COMMENT '热点键统计信息',
355
- `服务器主机` varchar(100) DEFAULT NULL COMMENT '服务器主机名',
356
- `实例名称` varchar(100) DEFAULT NULL COMMENT '缓存实例名称',
357
- `创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
358
- `更新时间` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
359
- PRIMARY KEY (`id`),
360
- KEY `idx_stats_date` (`日期`),
361
- KEY `idx_stats_time` (`统计时间`),
362
- KEY `idx_time_period` (`时间段`),
363
- KEY `idx_hit_rate` (`命中率`),
364
- KEY `idx_instance` (`实例名称`),
365
- KEY `idx_create_time` (`创建时间`)
366
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存系统统计分析表'
367
- """
399
+ -- 性能指标
400
+ `命中率` decimal(5,2) DEFAULT 0.00 COMMENT '命中率百分比',
401
+ `平均响应时间` decimal(10,2) DEFAULT 0.00 COMMENT '平均响应时间(毫秒)',
402
+ `运行时间` bigint DEFAULT 0 COMMENT '运行时间(秒)',
368
403
 
369
- cursor.execute(create_table_sql)
370
- connection.commit()
404
+ -- 命名空间统计
405
+ `命名空间统计` json COMMENT '命名空间统计详情',
371
406
 
372
- self.logger.info("MySQL数据库表初始化成功", {
373
- 'database': self.config.db_name,
374
- 'table': self.config.table_name
375
- })
376
- return True
407
+ -- 时间戳
408
+ `创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
377
409
 
378
- finally:
379
- connection.close()
410
+ PRIMARY KEY (`id`),
411
+ KEY `idx_日期` (`日期`),
412
+ KEY `idx_实例时间` (`实例标识`, `统计时间`),
413
+ KEY `idx_创建时间` (`创建时间`)
414
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存统计表'
415
+ """
380
416
 
381
- except Exception as e:
382
- self.logger.error(f"MySQL数据库初始化失败: {e}")
383
- # 不抛出异常,让系统继续运行
384
- return False
385
-
386
- def _generate_cache_key(self, key: str, namespace: str = "") -> str:
387
- """生成缓存键"""
388
- if namespace:
389
- return f"{self.config.cache_prefix}{namespace}:{key}"
390
- return f"{self.config.cache_prefix}{key}"
391
-
392
- def _record_operation(self, operation: str, response_time: float = 0):
393
- """记录操作统计"""
394
- with self._stats_lock:
395
- self.stats['total_operations'] += 1
396
- if operation in self.stats:
397
- self.stats[operation] += 1
398
- if response_time > 0:
399
- self.stats['response_times'].append(response_time)
400
- # 只保留最近1000次操作的响应时间
401
- if len(self.stats['response_times']) > 1000:
402
- self.stats['response_times'] = self.stats['response_times'][-1000:]
403
-
404
- def _record_hot_key(self, key: str, namespace: str = ""):
405
- """记录热点键"""
406
- cache_key = self._generate_cache_key(key, namespace)
407
- with self.hot_keys_lock:
408
- # 检查是否需要清理热点键(防止内存泄漏)
409
- if len(self.hot_keys) >= self.config.hot_keys_cleanup_threshold:
410
- # 保留访问次数最高的键,移除访问次数最少的键
411
- sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
412
- # 保留前600个热点键,为新键留出空间
413
- self.hot_keys = dict(sorted_keys[:600])
417
+ cursor.execute(create_table_sql)
418
+ connection.commit()
414
419
 
415
- self.logger.info("热点键内存清理", {
416
- '清理前数量': len(sorted_keys),
417
- '清理后数量': len(self.hot_keys),
418
- '清理阈值': self.config.hot_keys_cleanup_threshold
419
- })
420
+ connection.close()
420
421
 
421
- # 记录或更新热点键访问次数
422
- self.hot_keys[cache_key] = self.hot_keys.get(cache_key, 0) + 1
422
+ except Exception as e:
423
+ logger.error("统计表初始化失败", {'error': str(e)})
424
+ raise
423
425
 
424
- def _cache_operation(operation_name: str):
425
- """缓存操作装饰器工厂"""
426
- def decorator(func):
427
- @functools.wraps(func)
428
- def wrapper(self, *args, **kwargs):
429
- start_time = time.time()
430
-
431
- # 检查系统是否就绪
432
- if not self.is_ready:
433
- self._record_operation('errors')
434
- self.logger.warning(f"缓存系统未就绪,{operation_name}操作失败", {
435
- 'state': self._state.value
436
- })
437
- return None if operation_name == 'get' else False
438
-
439
- try:
440
- result = func(self, *args, **kwargs)
441
- response_time = (time.time() - start_time) * 1000
442
- self._record_operation(operation_name, response_time)
443
- return result
444
-
445
- except Exception as e:
446
- self._record_operation('errors')
447
- self.logger.error(f"缓存{operation_name}操作失败: {e}", {
448
- 'args': args[:2] if len(args) > 2 else args, # 避免记录过长的参数
449
- 'operation': operation_name
450
- })
451
- return None if operation_name == 'get' else False
452
-
453
- return wrapper
454
- return decorator
426
+ @property
427
+ def is_ready(self) -> bool:
428
+ """检查系统是否就绪"""
429
+ return self._ready_event.is_set()
430
+
431
+ @property
432
+ def is_mysql_ready(self) -> bool:
433
+ """检查MySQL是否就绪"""
434
+ return self._state == CacheSystemState.MYSQL_READY
455
435
 
456
- @_cache_operation('get')
457
436
  def get(self, key: str, namespace: str = "", default=None) -> Any:
458
437
  """获取缓存值"""
459
- cache_key = self._generate_cache_key(key, namespace)
460
-
461
- # 获取缓存值
462
- value = self.redis_client.get(cache_key)
438
+ return self._get_with_stats(key, namespace, default)
439
+
440
+ def _get_with_stats(self, key: str, namespace: str = "", default=None) -> Any:
441
+ """带统计的获取缓存值"""
442
+ if not self.is_ready:
443
+ return default
463
444
 
464
- if value is not None:
465
- # 缓存命中
466
- self._record_hot_key(key, namespace)
445
+ start_time = time.time()
446
+ try:
447
+ cache_key = self._generate_cache_key(key, namespace)
448
+ value = self.redis_client.get(cache_key)
449
+ response_time = (time.time() - start_time) * 1000
467
450
 
468
- try:
469
- return json.loads(value.decode('utf-8'))
470
- except (json.JSONDecodeError, UnicodeDecodeError):
471
- return value.decode('utf-8')
472
- else:
473
- # 缓存未命中
451
+ if value is not None:
452
+ # 缓存命中
453
+ if self.stats_collector:
454
+ self.stats_collector.record_operation('hits', response_time, namespace)
455
+ try:
456
+ return json.loads(value.decode('utf-8'))
457
+ except (json.JSONDecodeError, UnicodeDecodeError):
458
+ return value.decode('utf-8')
459
+ else:
460
+ # 缓存未命中
461
+ if self.stats_collector:
462
+ self.stats_collector.record_operation('misses', response_time, namespace)
463
+ return default
464
+
465
+ except Exception as e:
466
+ response_time = (time.time() - start_time) * 1000
467
+ if self.stats_collector:
468
+ self.stats_collector.record_operation('errors', response_time, namespace)
469
+ logger.error("缓存获取失败", {
470
+ 'key': key,
471
+ 'namespace': namespace,
472
+ 'error': str(e)
473
+ })
474
474
  return default
475
475
 
476
- @_cache_operation('set')
477
- def set(self, key: str, value: Any, ttl: int = None, namespace: str = "") -> bool:
476
+ def set(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
478
477
  """设置缓存值"""
479
- cache_key = self._generate_cache_key(key, namespace)
480
-
481
- # 使用智能TTL策略
482
- if ttl is None:
483
- ttl = SmartTTLConfig.get_ttl(namespace, self.config.default_ttl)
484
-
485
- # 序列化值
486
- if isinstance(value, (dict, list, tuple)):
487
- serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
488
- else:
489
- serialized_value = str(value)
490
-
491
- # 检查值大小
492
- if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
493
- self.logger.warning(f"缓存值过大,跳过设置: {len(serialized_value)} bytes")
478
+ return self._set_with_stats(key, value, ttl, namespace)
479
+
480
+ def _set_with_stats(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
481
+ """带统计的设置缓存值"""
482
+ if not self.is_ready:
494
483
  return False
495
484
 
496
- # 设置缓存
497
- result = self.redis_client.setex(cache_key, ttl, serialized_value)
498
-
499
- # 记录TTL策略使用情况
500
- if namespace and ttl != self.config.default_ttl:
501
- self.logger.debug("使用智能TTL策略", {
502
- 'namespace': namespace,
503
- 'ttl': ttl,
504
- 'key': key[:50] + "..." if len(key) > 50 else key
505
- })
506
-
507
- return bool(result)
508
-
509
- def delete(self, key: str, namespace: str = "") -> bool:
510
- """删除缓存值"""
511
485
  start_time = time.time()
512
-
513
486
  try:
514
487
  cache_key = self._generate_cache_key(key, namespace)
515
- result = self.redis_client.delete(cache_key)
488
+
489
+ # 智能TTL策略
490
+ if ttl is None:
491
+ ttl = self._get_smart_ttl(namespace, key, len(json.dumps(value, ensure_ascii=False, default=str)))
492
+
493
+ # 序列化值
494
+ if isinstance(value, (dict, list)):
495
+ serialized_value = json.dumps(value, ensure_ascii=False, default=str)
496
+ else:
497
+ serialized_value = str(value)
498
+
499
+ # 检查值大小
500
+ value_size = len(serialized_value.encode('utf-8'))
501
+ if value_size > self.config.max_value_size:
502
+ if self.stats_collector:
503
+ self.stats_collector.record_operation('errors', 0, namespace)
504
+ logger.warning("缓存值过大,跳过设置", {
505
+ 'key': key,
506
+ 'size': len(serialized_value),
507
+ 'max_size': self.config.max_value_size
508
+ })
509
+ return False
510
+
511
+ result = self.redis_client.setex(cache_key, ttl, serialized_value)
516
512
  response_time = (time.time() - start_time) * 1000
517
513
 
518
- self._record_operation('deletes', response_time)
514
+ if self.stats_collector:
515
+ self.stats_collector.record_operation('sets', response_time, namespace)
519
516
  return bool(result)
520
517
 
521
518
  except Exception as e:
522
- self._record_operation('errors')
523
- self.logger.error(f"缓存删除失败: {e}", {
519
+ response_time = (time.time() - start_time) * 1000
520
+ if self.stats_collector:
521
+ self.stats_collector.record_operation('errors', response_time, namespace)
522
+ logger.error("缓存设置失败", {
524
523
  'key': key,
525
- 'namespace': namespace
524
+ 'namespace': namespace,
525
+ 'error': str(e)
526
526
  })
527
527
  return False
528
528
 
529
- def exists(self, key: str, namespace: str = "") -> bool:
530
- """检查缓存键是否存在"""
529
+ def delete(self, key: str, namespace: str = "") -> bool:
530
+ """删除缓存值"""
531
+ return self._delete_with_stats(key, namespace)
532
+
533
+ def _delete_with_stats(self, key: str, namespace: str = "") -> bool:
534
+ """带统计的删除缓存值"""
535
+ if not self.is_ready:
536
+ return False
537
+
538
+ start_time = time.time()
531
539
  try:
532
540
  cache_key = self._generate_cache_key(key, namespace)
533
- return bool(self.redis_client.exists(cache_key))
541
+ result = self.redis_client.delete(cache_key)
542
+ response_time = (time.time() - start_time) * 1000
543
+ if self.stats_collector:
544
+ self.stats_collector.record_operation('deletes', response_time, namespace)
545
+ return bool(result)
546
+
534
547
  except Exception as e:
535
- self.logger.error(f"缓存存在性检查失败: {e}")
548
+ response_time = (time.time() - start_time) * 1000
549
+ if self.stats_collector:
550
+ self.stats_collector.record_operation('errors', response_time, namespace)
551
+ logger.error("缓存删除失败", {
552
+ 'key': key,
553
+ 'namespace': namespace,
554
+ 'error': str(e)
555
+ })
536
556
  return False
537
557
 
538
558
  def clear_namespace(self, namespace: str) -> int:
539
559
  """清除指定命名空间的所有缓存"""
560
+ if not self.is_ready:
561
+ return 0
562
+
540
563
  try:
541
- pattern = f"{self.config.cache_prefix}{namespace}:*"
564
+ pattern = f"{self.config.cache_prefix}:{namespace}:*"
542
565
  keys = self.redis_client.keys(pattern)
543
566
 
544
567
  if keys:
545
568
  deleted = self.redis_client.delete(*keys)
546
- self.logger.info(f"清除命名空间缓存: {namespace}, 删除键数: {deleted}")
547
569
  return deleted
548
570
  return 0
549
571
 
550
572
  except Exception as e:
551
- self.logger.error(f"清除命名空间缓存失败: {e}")
573
+ logger.error("清除命名空间失败", {
574
+ 'namespace': namespace,
575
+ 'error': str(e)
576
+ })
552
577
  return 0
553
578
 
554
- def get_stats(self) -> Dict[str, Any]:
555
- """获取缓存统计信息"""
556
- with self._stats_lock:
557
- total_ops = self.stats['total_operations']
558
- hits = self.stats['hits']
579
+ def _generate_cache_key(self, key: str, namespace: str = "") -> str:
580
+ """生成缓存键"""
581
+ if namespace:
582
+ return f"{self.config.cache_prefix}:{namespace}:{key}"
583
+ return f"{self.config.cache_prefix}:{key}"
584
+
585
+ def _get_smart_ttl(self, namespace: str, key: str = "", data_size: int = 0) -> int:
586
+ """智能TTL策略 - 通用版本
587
+
588
+ 基于多种因素智能计算TTL:
589
+ 1. 命名空间模式匹配
590
+ 2. 数据类型推断
591
+ 3. 数据大小考虑
592
+ 4. 访问频率预测
593
+ """
594
+
595
+ # 如果禁用智能TTL,直接返回默认值
596
+ if not self.config.enable_smart_ttl:
597
+ return self.config.default_ttl
598
+
599
+ # 1. 基于命名空间模式的智能匹配
600
+ namespace_patterns = {
601
+ # 数据库相关 - 变化频率低,TTL较长
602
+ r'.*database.*|.*db.*|.*schema.*': 1800, # 30分钟
559
603
 
560
- # 计算命中率
561
- hit_rate = (hits / total_ops * 100) if total_ops > 0 else 0
604
+ # 表结构相关 - 变化频率低,TTL较长
605
+ r'.*table.*|.*column.*|.*field.*': 1200, # 20分钟
562
606
 
563
- # 计算平均响应时间
564
- response_times = self.stats['response_times']
565
- avg_response_time = sum(response_times) / len(response_times) if response_times else 0
607
+ # 数据查询相关 - 变化频率中等,TTL中等
608
+ r'.*data.*|.*query.*|.*result.*': 300, # 5分钟
566
609
 
567
- # 计算运行时间
568
- uptime = time.time() - self.stats['start_time']
610
+ # 用户会话相关 - 变化频率高,TTL较短
611
+ r'.*session.*|.*user.*|.*auth.*': 900, # 15分钟
569
612
 
570
- # 计算每秒操作数
571
- ops_per_second = total_ops / uptime if uptime > 0 else 0
613
+ # 配置相关 - 变化频率很低,TTL很长
614
+ r'.*config.*|.*setting.*|.*param.*': 3600, # 1小时
572
615
 
573
- # 获取热点键(前10个)
574
- with self.hot_keys_lock:
575
- top_hot_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)[:10]
616
+ # 统计相关 - 变化频率中等,TTL中等
617
+ r'.*stats.*|.*metric.*|.*count.*': 600, # 10分钟
576
618
 
577
- return {
578
- 'hits': hits,
579
- 'misses': self.stats['misses'],
580
- 'sets': self.stats['sets'],
581
- 'deletes': self.stats['deletes'],
582
- 'errors': self.stats['errors'],
583
- 'total_operations': total_ops,
584
- 'hit_rate': round(hit_rate, 2),
585
- 'avg_response_time': round(avg_response_time, 4),
586
- 'ops_per_second': round(ops_per_second, 2),
587
- 'uptime_seconds': int(uptime),
588
- 'hot_keys': dict(top_hot_keys),
589
- 'unique_keys_count': len(self.hot_keys)
590
- }
591
-
592
- def health_check(self) -> Dict[str, Any]:
593
- """健康检查"""
594
- health_info = {
595
- 'redis_connected': False,
596
- 'mysql_available': False,
597
- 'stats_worker_running': self._stats_running,
598
- 'instance_name': self.instance_name,
599
- 'timestamp': datetime.now().isoformat()
619
+ # 缓存相关 - 变化频率高,TTL较短
620
+ r'.*cache.*|.*temp.*|.*tmp.*': 180, # 3分钟
621
+
622
+ # 列表相关 - 变化频率中等,TTL中等
623
+ r'.*list.*|.*index.*|.*catalog.*': 450, # 7.5分钟
600
624
  }
601
625
 
602
- # 检查Redis连接
603
- try:
604
- self.redis_client.ping()
605
- health_info['redis_connected'] = True
606
- except Exception as e:
607
- health_info['redis_error'] = str(e)
608
-
609
- # 检查MySQL连接
610
- if self.mysql_pool:
611
- try:
612
- connection = self.mysql_pool.connection()
613
- connection.close()
614
- health_info['mysql_available'] = True
615
- except Exception as e:
616
- health_info['mysql_error'] = str(e)
617
-
618
- return health_info
619
-
620
- def _start_stats_worker(self):
621
- """启动统计工作线程"""
622
- if not self._stats_running and self.mysql_pool:
623
- self._stats_running = True
624
- self._stats_thread = threading.Thread(
625
- target=self._stats_worker,
626
- daemon=True,
627
- name=f"stats_worker_{self.instance_name}"
628
- )
629
- self._stats_thread.start()
630
- self.logger.info("统计工作线程已启动", {
631
- 'instance_name': self.instance_name,
632
- 'delay_first_run': True
633
- })
634
-
635
- def _stats_worker(self):
636
- """后台统计工作线程"""
637
- cleanup_counter = 0
626
+ # 合并用户自定义模式(优先级更高)
627
+ namespace_patterns.update(self.config.custom_namespace_patterns)
638
628
 
639
- # 延迟启动:等待一个完整的统计间隔
640
- self.logger.info("统计线程启动,等待首个统计间隔", {
641
- 'instance_name': self.instance_name,
642
- 'interval': self.config.stats_interval,
643
- 'delay_reason': '避免初始化时的阻塞操作'
644
- })
629
+ # 2. 基于键名模式的智能匹配
630
+ key_patterns = {
631
+ # ID相关 - 相对稳定
632
+ r'.*_id$|^id_.*': 1.2, # TTL倍数
633
+
634
+ # 详情相关 - 相对稳定
635
+ r'.*detail.*|.*info.*': 1.1,
636
+
637
+ # 列表相关 - 变化较频繁
638
+ r'.*list.*|.*items.*': 0.8,
639
+
640
+ # 计数相关 - 变化频繁
641
+ r'.*count.*|.*num.*|.*total.*': 0.6,
642
+
643
+ # 状态相关 - 变化很频繁
644
+ r'.*status.*|.*state.*': 0.5,
645
+ }
645
646
 
646
- # 使用可中断的等待
647
- if not self._interruptible_sleep(self.config.stats_interval):
648
- return # 如果被中断则退出
647
+ # 合并用户自定义键模式
648
+ key_patterns.update(self.config.custom_key_patterns)
649
+
650
+ # 3. 基于数据大小的调整
651
+ size_factor = 1.0
652
+ if data_size > 0:
653
+ if data_size > 1024 * 1024: # > 1MB,延长TTL减少重建开销
654
+ size_factor = 1.5
655
+ elif data_size > 100 * 1024: # > 100KB
656
+ size_factor = 1.2
657
+ elif data_size < 1024: # < 1KB,缩短TTL减少内存占用
658
+ size_factor = 0.8
659
+
660
+ # 4. 计算基础TTL
661
+ base_ttl = self.config.default_ttl
662
+
663
+ # 匹配命名空间模式
664
+ for pattern, ttl in namespace_patterns.items():
665
+ if re.match(pattern, namespace.lower()):
666
+ base_ttl = ttl
667
+ break
649
668
 
650
- while self._stats_running:
651
- try:
652
- # 收集统计数据
653
- stats_data = self.get_stats()
654
-
655
- # 提交到MySQL(带状态检查)
656
- self._submit_stats_to_mysql(stats_data)
657
-
658
- # 清理过期的热点键统计
659
- self._cleanup_hot_keys()
660
-
661
- # 定期清理过期数据
662
- cleanup_counter += 1
663
- if cleanup_counter >= 24: # 24 * 1800秒 = 12小时
664
- self._cleanup_expired_mysql_data()
665
- cleanup_counter = 0
666
-
667
- except Exception as e:
668
- self.logger.error(f"统计工作线程异常: {e}")
669
-
670
- # 可中断的等待下一个统计间隔
671
- if not self._interruptible_sleep(self.config.stats_interval):
669
+ # 5. 应用键名模式调整
670
+ key_factor = 1.0
671
+ for pattern, factor in key_patterns.items():
672
+ if re.match(pattern, key.lower()):
673
+ key_factor = factor
672
674
  break
675
+
676
+ # 6. 计算最终TTL
677
+ final_ttl = int(base_ttl * key_factor * size_factor)
678
+
679
+ # 7. TTL边界限制(使用配置值)
680
+ final_ttl = max(self.config.ttl_min, min(self.config.ttl_max, final_ttl))
681
+
682
+ # 8. 记录智能TTL决策(使用配置开关)
683
+ if self.config.debug_ttl:
684
+ logger.debug("智能TTL计算", {
685
+ 'namespace': namespace,
686
+ 'key': key[:50] + "..." if len(key) > 50 else key,
687
+ 'data_size': data_size,
688
+ 'base_ttl': base_ttl,
689
+ 'key_factor': key_factor,
690
+ 'size_factor': size_factor,
691
+ 'final_ttl': final_ttl
692
+ })
693
+
694
+ return final_ttl
673
695
 
674
- def _interruptible_sleep(self, duration: float) -> bool:
675
- """可中断的睡眠"""
676
- sleep_interval = 1.0 # 每秒检查一次
677
- elapsed = 0.0
696
+ def get_stats(self) -> Dict[str, Any]:
697
+ """获取统计信息"""
698
+ # 确保统计系统已初始化
699
+ if self.stats_collector:
700
+ return self.stats_collector.get_stats()
678
701
 
679
- while elapsed < duration and self._stats_running:
680
- time.sleep(min(sleep_interval, duration - elapsed))
681
- elapsed += sleep_interval
682
-
683
- return self._stats_running
702
+ logger.debug("统计系统未初始化,返回空统计信息", {
703
+ 'instance_name': self.instance_name,
704
+ 'process_id': os.getpid()
705
+ })
706
+ return {
707
+ 'enabled': False,
708
+ 'message': '统计系统未初始化',
709
+ 'process_id': os.getpid()
710
+ }
684
711
 
685
- def _submit_stats_to_mysql(self, stats_data: Dict[str, Any]):
686
- """提交统计数据到MySQL"""
687
- if not self.mysql_pool:
688
- self.logger.debug("MySQL连接池不可用,跳过统计数据提交")
689
- return
712
+ def health_check(self) -> Dict[str, Any]:
713
+ """健康检查"""
714
+ try:
715
+ # Redis连接检查
716
+ redis_latency = time.time()
717
+ self.redis_client.ping()
718
+ redis_latency = (time.time() - redis_latency) * 1000
690
719
 
691
- # 使用状态检查而不是属性检查
692
- if not self.is_mysql_ready:
693
- self.logger.debug("MySQL尚未就绪,跳过统计数据提交", {
694
- 'state': self._state.value,
695
- 'instance_name': self.instance_name
696
- })
697
- return
720
+ # 获取Redis信息
721
+ redis_info = self.redis_client.info()
722
+
723
+ return {
724
+ 'status': 'healthy',
725
+ 'redis_connected': True,
726
+ 'redis_latency_ms': round(redis_latency, 2),
727
+ 'mysql_enabled': self.mysql_pool is not None,
728
+ 'mysql_ready': self.is_mysql_ready,
729
+ 'system_state': self._state.value,
730
+ 'uptime_seconds': time.time() - (self.stats_collector.stats['start_time'] if self.stats_collector else time.time()),
731
+ 'redis_memory_used': redis_info.get('used_memory', 0),
732
+ 'redis_connected_clients': redis_info.get('connected_clients', 0),
733
+ 'process_id': os.getpid()
734
+ }
698
735
 
699
- logger.info("统计数据", {'stats_data': stats_data})
700
-
701
- try:
702
- # 使用上下文管理器确保连接正确关闭
703
- connection = self.mysql_pool.connection()
704
- try:
705
- with connection.cursor() as cursor:
706
- cursor.execute(f"USE `{self.config.db_name}`")
707
-
708
- # 准备数据
709
- now = datetime.now()
710
- date_str = now.strftime("%Y-%m-%d")
711
- time_period = now.strftime("%Y%m%d_%H%M")
712
-
713
- insert_sql = f"""
714
- INSERT INTO `{self.config.table_name}` (
715
- `日期`, `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
716
- `缓存删除数`, `缓存错误数`, `命中率`, `总操作数`, `平均响应时间`,
717
- `每秒操作数`, `唯一键数量`, `系统运行时间`, `热点键统计`,
718
- `服务器主机`, `实例名称`
719
- ) VALUES (
720
- %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
721
- )
722
- """
723
-
724
- cursor.execute(insert_sql, (
725
- date_str,
726
- now,
727
- time_period,
728
- stats_data['hits'],
729
- stats_data['misses'],
730
- stats_data['sets'],
731
- stats_data['deletes'],
732
- stats_data['errors'],
733
- stats_data['hit_rate'],
734
- stats_data['total_operations'],
735
- stats_data['avg_response_time'],
736
- stats_data['ops_per_second'],
737
- stats_data['unique_keys_count'],
738
- stats_data['uptime_seconds'],
739
- json.dumps(stats_data['hot_keys'], ensure_ascii=False, cls=MySQLDataEncoder),
740
- socket.gethostname(),
741
- self.instance_name
742
- ))
743
-
744
- connection.commit()
745
-
746
- self.logger.info("统计数据已提交到MySQL", {
747
- 'time_period': time_period,
748
- 'total_operations': stats_data['total_operations'],
749
- 'hit_rate': stats_data['hit_rate'],
750
- 'instance_name': self.instance_name
751
- })
752
-
753
- finally:
754
- connection.close()
755
-
756
736
  except Exception as e:
757
- self.logger.error(f"提交统计数据到MySQL失败: {e}", {
758
- 'instance_name': self.instance_name,
759
- 'error_type': type(e).__name__
760
- })
761
-
762
- def _cleanup_hot_keys(self):
763
- """清理热点键统计 """
764
- with self.hot_keys_lock:
765
- current_count = len(self.hot_keys)
766
-
767
- # 如果热点键数量超过最大限制,进行清理
768
- if current_count > self.config.max_hot_keys:
769
- # 保留访问次数最高的键
770
- sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
771
- # 保留80%的热点键
772
- keep_count = int(self.config.max_hot_keys * 0.8)
773
- self.hot_keys = dict(sorted_keys[:keep_count])
774
-
775
- self.logger.info("定期热点键清理", {
776
- '清理前数量': current_count,
777
- '清理后数量': len(self.hot_keys),
778
- '保留比例': '80%'
779
- })
737
+ return {
738
+ 'status': 'unhealthy',
739
+ 'error': str(e),
740
+ 'redis_connected': False,
741
+ 'system_state': self._state.value
742
+ }
780
743
 
781
- def _cleanup_expired_mysql_data(self):
782
- """清理过期的MySQL统计数据"""
783
- if not self.mysql_pool:
744
+ def _record_operation(self, operation: str, response_time: float = 0, namespace: str = "", key: str = ""):
745
+ """记录操作统计 """
746
+ # 如果禁用统计功能,直接返回
747
+ if not self.config.enable_stats:
784
748
  return
749
+
750
+ logger.debug("调用操作记录", {
751
+ 'operation': operation,
752
+ 'response_time_ms': round(response_time, 2),
753
+ 'namespace': namespace,
754
+ 'key': key[:50] + "..." if len(key) > 50 else key,
755
+ 'instance_name': self.instance_name
756
+ })
785
757
 
786
- try:
787
- connection = self.mysql_pool.connection()
788
- try:
789
- with connection.cursor() as cursor:
790
- cursor.execute(f"USE `{self.config.db_name}`")
791
-
792
- # 计算过期时间点
793
- from datetime import timedelta
794
- expire_date = datetime.now() - timedelta(days=self.config.stats_retention)
795
-
796
- # 删除过期数据
797
- delete_sql = f"""
798
- DELETE FROM `{self.config.table_name}`
799
- WHERE `统计时间` < %s
800
- """
801
-
802
- cursor.execute(delete_sql, (expire_date,))
803
- deleted_rows = cursor.rowcount
804
- connection.commit()
805
-
806
- if deleted_rows > 0:
807
- self.logger.info("清理过期MySQL统计数据", {
808
- 'deleted_rows': deleted_rows,
809
- 'expire_date': expire_date.strftime('%Y-%m-%d %H:%M:%S'),
810
- 'retention_days': self.config.stats_retention
811
- })
812
-
813
- finally:
814
- connection.close()
815
-
816
- except Exception as e:
817
- self.logger.error(f"清理过期MySQL统计数据失败: {e}")
818
-
819
758
  def shutdown(self):
820
759
  """关闭缓存系统"""
821
- self.logger.info("正在关闭缓存系统...", {
822
- 'instance_name': self.instance_name,
823
- 'state': self._state.value
824
- })
760
+ self._state = CacheSystemState.SHUTDOWN
825
761
 
826
- # 停止统计线程
827
- self._stats_running = False
828
- if self._stats_thread and self._stats_thread.is_alive():
829
- self.logger.info("等待统计线程结束...")
830
- self._stats_thread.join(timeout=10) # 最多等待10秒
831
-
832
- if self._stats_thread.is_alive():
833
- self.logger.warning("统计线程未能在超时时间内结束")
834
-
835
- # 关闭线程池
836
- if hasattr(self, '_executor'):
837
- self.logger.info("关闭线程池...")
838
- self._executor.shutdown(wait=True, timeout=5)
839
-
840
- # 最后一次提交统计数据(如果MySQL可用)
841
- if self.is_mysql_ready:
842
- try:
843
- self.logger.info("提交最终统计数据...")
844
- stats_data = self.get_stats()
845
- self._submit_stats_to_mysql(stats_data)
846
- except Exception as e:
847
- self.logger.error(f"关闭时提交统计数据失败: {e}")
848
-
849
- self.logger.info("缓存系统已关闭", {
850
- 'instance_name': self.instance_name
851
- })
852
-
853
- def __enter__(self):
854
- """上下文管理器支持"""
855
- return self
856
-
857
- def __exit__(self, exc_type, exc_val, exc_tb):
858
- """上下文管理器退出时自动关闭"""
859
- self.shutdown()
860
- return False
762
+ if self.stats_collector:
763
+ # The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
764
+ self.stats_collector.reset_stats()
765
+
766
+ logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
861
767
 
862
768
 
863
769
  class CacheManager:
864
- """缓存管理器 - 单例模式"""
770
+ """缓存管理器"""
865
771
 
866
772
  _instance = None
867
- _lock = threading.RLock()
773
+ _lock = threading.Lock()
868
774
 
869
775
  def __new__(cls):
870
- with cls._lock:
871
- if cls._instance is None:
872
- cls._instance = super().__new__(cls)
873
- cls._instance._initialized = False
874
- return cls._instance
776
+ if cls._instance is None:
777
+ with cls._lock:
778
+ if cls._instance is None:
779
+ cls._instance = super().__new__(cls)
780
+ return cls._instance
875
781
 
876
782
  def __init__(self):
877
- if self._initialized:
783
+ if not hasattr(self, '_initialized'):
784
+ self.cache_instance = None
785
+ self._initialized = True
786
+
787
+ def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
788
+ """初始化缓存系统"""
789
+ if self.cache_instance is not None:
790
+ logger.warning("缓存系统已初始化,跳过重复初始化", {
791
+ 'existing_instance': self.cache_instance.instance_name,
792
+ 'new_instance': instance_name
793
+ })
878
794
  return
879
795
 
880
- self.cache_instance = None
881
- self.enabled = True
882
- self.initialization_error = None
883
- self._initialized = True
884
- self.logger = logger
885
-
886
- def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
887
- config: CacheConfig = None, db_name: str = None, table_name: str = None):
888
- """初始化缓存系统(非阻塞)"""
889
- try:
890
- # 立即创建缓存实例,内部会异步初始化MySQL和统计线程
891
- self.cache_instance = SmartCacheSystem(
892
- redis_client=redis_client,
893
- mysql_pool=mysql_pool,
894
- instance_name=instance_name,
895
- config=config,
896
- db_name=db_name,
897
- table_name=table_name
898
- )
899
- self.initialization_error = None
900
- self.logger.info("缓存管理器初始化成功(异步模式)", {
901
- 'instance_name': instance_name,
902
- 'mysql_enabled': mysql_pool is not None,
903
- 'db_name': self.cache_instance.config.db_name,
904
- 'table_name': self.cache_instance.config.table_name,
905
- 'note': 'MySQL初始化将在后台异步完成'
906
- })
907
- return self # 支持链式调用
908
-
909
- except Exception as e:
910
- self.initialization_error = str(e)
911
- self.cache_instance = None
912
- self.logger.error(f"缓存管理器初始化失败: {e}")
913
- return self
796
+ self.cache_instance = SmartCacheSystem(
797
+ redis_client=redis_client,
798
+ mysql_pool=mysql_pool,
799
+ instance_name=instance_name,
800
+ **config
801
+ )
914
802
 
915
803
  def get_cache(self) -> Optional[SmartCacheSystem]:
916
804
  """获取缓存实例"""
917
- return self.cache_instance if self.enabled else None
805
+ return self.cache_instance
918
806
 
919
807
  def is_available(self) -> bool:
920
808
  """检查缓存是否可用"""
921
- return self.cache_instance is not None and self.enabled
922
-
923
- def enable(self):
924
- """启用缓存"""
925
- self.enabled = True
926
- self.logger.info("缓存系统已启用")
927
- return self # 支持链式调用
928
-
929
- def disable(self):
930
- """禁用缓存"""
931
- self.enabled = False
932
- self.logger.info("缓存系统已禁用")
933
- return self # 支持链式调用
809
+ return self.cache_instance is not None and self.cache_instance.is_ready
934
810
 
935
811
  def get_status(self) -> Dict[str, Any]:
936
812
  """获取缓存状态"""
813
+ if self.cache_instance is None:
814
+ return {
815
+ 'enabled': False,
816
+ 'available': False,
817
+ 'stats_enabled': False,
818
+ 'initialization_error': 'Cache not initialized'
819
+ }
820
+
937
821
  return {
938
- 'enabled': self.enabled,
939
- 'available': self.cache_instance is not None,
940
- 'initialization_error': self.initialization_error,
941
- 'instance_name': getattr(self.cache_instance, 'instance_name', None) if self.cache_instance else None
822
+ 'enabled': True,
823
+ 'available': self.cache_instance.is_ready,
824
+ 'mysql_ready': self.cache_instance.is_mysql_ready,
825
+ 'stats_enabled': self.cache_instance.config.enable_stats,
826
+ 'state': self.cache_instance._state.value,
827
+ 'instance_name': self.cache_instance.instance_name
942
828
  }
943
-
944
- def shutdown(self):
945
- """关闭缓存系统"""
946
- if self.cache_instance:
947
- self.cache_instance.shutdown()
948
- self.cache_instance = None
949
- self.logger.info("缓存管理器已关闭")
950
829
 
951
830
 
952
- # 导出单例实例
831
+ # 全局缓存管理器实例
953
832
  cache_manager = CacheManager()