mdbq 4.0.112__py3-none-any.whl → 4.0.114__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mdbq might be problematic. Click here for more details.

mdbq/redis/redis_cache.py CHANGED
@@ -5,30 +5,31 @@ Redis智能缓存系统
5
5
  主要功能:
6
6
  1. Redis缓存的CRUD操作
7
7
  2. 命名空间隔离
8
- 3. 分布式锁防止缓存击穿
9
- 4. 自动统计分析并提交到MySQL
8
+ 3. 智能TTL策略
9
+ 4. 专业统计分析并提交到MySQL
10
10
  5. 缓存健康检查和监控
11
-
11
+ 6. 热点键分析
12
+ 7. 响应时间分析
13
+ 8. 业务指标计算
12
14
  """
13
15
 
14
16
  import json
15
17
  import time
16
18
  import threading
17
19
  import socket
18
- from datetime import datetime, date
19
- from decimal import Decimal
20
- from uuid import UUID
21
- from typing import Optional, Dict, Any, List, Union, Callable
20
+ import os
21
+ import statistics
22
+ import enum
23
+ import re
24
+ from datetime import datetime, timedelta
25
+ from typing import Optional, Dict, Any, List, Callable
26
+ from threading import Event
27
+ from collections import defaultdict, deque
22
28
  import redis
23
29
  from mdbq.log import mylogger
24
30
 
25
- import enum
26
- from typing import Optional, Dict, Any, List, Union, Callable
27
- from concurrent.futures import ThreadPoolExecutor
28
- import asyncio
29
- from threading import Event
30
- import functools
31
31
 
32
+ # 全局日志器
32
33
  logger = mylogger.MyLogger(
33
34
  logging_mode='file',
34
35
  log_level='info',
@@ -42,259 +43,340 @@ logger = mylogger.MyLogger(
42
43
  )
43
44
 
44
45
 
45
- class CacheSystemState(enum.Enum):
46
- """缓存系统状态枚举"""
47
- INITIALIZING = "initializing"
48
- READY = "ready"
49
- MYSQL_READY = "mysql_ready"
50
- ERROR = "error"
51
-
46
+ class CacheStatsCollector:
47
+ """缓存统计收集器 - 单线程同步版本"""
48
+
49
+ def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
50
+ self.enabled = enabled
51
+ self.mysql_pool = mysql_pool
52
+ self.config = config or {}
53
+ self.process_id = os.getpid()
54
+ self.instance_name = self.config.get('instance_name', 'default')
55
+
56
+ # 统计数据
57
+ self.stats = {
58
+ 'hits': 0,
59
+ 'misses': 0,
60
+ 'sets': 0,
61
+ 'deletes': 0,
62
+ 'errors': 0,
63
+ 'total_operations': 0,
64
+ 'start_time': time.time()
65
+ }
66
+ self._lock = threading.RLock()
67
+ self.response_times = deque(maxlen=1000)
68
+ self.namespace_stats = defaultdict(int)
69
+
70
+ # 同步提交控制
71
+ self.submit_interval = self.config.get('submit_interval', 300) # 每N次操作提交一次统计数据
72
+ self.last_submit_time = time.time()
73
+ self.min_submit_interval = self.config.get('min_submit_interval', 300) # 最少N秒提交一次
52
74
 
53
- class MySQLDataEncoder(json.JSONEncoder):
54
- """自定义JSON编码器,支持MySQL常见数据类型"""
55
- def default(self, obj):
75
+ def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
76
+ """记录操作统计"""
77
+ if not self.enabled:
78
+ return
79
+
56
80
  try:
57
- if isinstance(obj, datetime):
58
- return obj.isoformat()
59
- elif isinstance(obj, date):
60
- return obj.isoformat()
61
- elif isinstance(obj, Decimal):
62
- # Decimal转换为字符串保持精度
63
- return str(obj)
64
- elif isinstance(obj, UUID):
65
- return str(obj)
66
- elif isinstance(obj, bytes):
67
- # 二进制数据转换为base64
68
- import base64
69
- return base64.b64encode(obj).decode('utf-8')
70
- elif isinstance(obj, (set, frozenset)):
71
- # 集合类型转换为列表
72
- return list(obj)
73
- elif hasattr(obj, 'isoformat'):
74
- # 其他日期时间类型
75
- return obj.isoformat()
76
- elif hasattr(obj, '__float__'):
77
- # 数值类型
78
- return float(obj)
79
- elif hasattr(obj, '__str__'):
80
- # 其他有字符串表示的对象
81
- return str(obj)
82
- elif hasattr(obj, '__dict__'):
83
- # 处理其他对象,转换为字典
84
- return obj.__dict__
81
+ with self._lock:
82
+ old_total = self.stats['total_operations']
83
+
84
+ self.stats['total_operations'] += 1
85
+ self.stats[operation] = self.stats.get(operation, 0) + 1
86
+
87
+ if response_time > 0:
88
+ self.response_times.append(response_time)
89
+
90
+ if namespace:
91
+ self.namespace_stats[namespace] += 1
92
+
93
+ # 检查是否需要提交统计数据(容错处理)
94
+ try:
95
+ self._check_and_submit()
96
+ except Exception as submit_error:
97
+ # 统计提交失败不应影响统计记录
98
+ logger.error("统计数据提交检查失败,但统计记录继续", {
99
+ 'instance_name': self.instance_name,
100
+ 'process_id': self.process_id,
101
+ 'operation': operation,
102
+ 'submit_error': str(submit_error)
103
+ })
104
+
105
+ logger.debug("缓存操作统计已记录", {
106
+ 'operation': operation,
107
+ 'response_time_ms': round(response_time, 2),
108
+ 'namespace': namespace,
109
+ 'process_id': self.process_id,
110
+ 'old_total_operations': old_total,
111
+ 'new_total_operations': self.stats['total_operations']
112
+ })
85
113
  except Exception as e:
86
- # 如果所有方法都失败,返回对象的字符串表示
87
- return f"<{type(obj).__name__}: {str(obj)[:100]}>"
88
-
89
- return super().default(obj)
90
-
91
-
92
- class SmartTTLConfig:
93
- """智能TTL配置策略"""
94
-
95
- # TTL策略映射表(秒)
96
- TTL_STRATEGIES = {
97
- # 数据库相关缓存
98
- 'sycm_database': 1800, # 数据库列表:30分钟
99
- 'sycm_tables': 1200, # 表列表:20分钟
100
- 'sycm_table_data': 300, # 表数据:5分钟
101
-
102
- # 用户会话相关
103
- 'user_session': 3600, # 用户会话:1小时
104
- 'user_info': 1800, # 用户信息:30分钟
105
-
106
- # 静态配置
107
- 'static_config': 86400, # 静态配置:24小时
108
- 'system_config': 43200, # 系统配置:12小时
109
-
110
- # API相关
111
- 'api_response': 600, # API响应:10分钟
112
- 'api_cache': 900, # API缓存:15分钟
113
-
114
- # 临时数据
115
- 'temp_data': 300, # 临时数据:5分钟
116
- 'short_cache': 60, # 短期缓存:1分钟
117
- }
114
+ # 统计记录失败不应影响缓存操作
115
+ logger.error("统计记录失败,但缓存操作继续", {
116
+ 'instance_name': self.instance_name,
117
+ 'process_id': self.process_id,
118
+ 'operation': operation,
119
+ 'error': str(e)
120
+ })
118
121
 
119
- @classmethod
120
- def get_ttl(cls, namespace: str, default: int = 3600) -> int:
121
- """
122
- 根据命名空间获取对应的TTL值
122
+ def _check_and_submit(self):
123
+ """检查并提交统计数据(同步)"""
124
+ if not self.mysql_pool:
125
+ return
126
+
127
+ current_time = time.time()
128
+ time_since_last_submit = current_time - self.last_submit_time
129
+ should_submit = False
130
+ reason = ""
131
+
132
+ # 重构后的提交逻辑:
133
+ # 1. 每N次操作提交一次,但每X秒内只能提交一次(防止频繁提交)
134
+ # 2. 每隔X秒提交一次,但必须满足N次操作
135
+
136
+ if self.stats['total_operations'] % self.submit_interval == 0:
137
+ # 达到操作次数阈值
138
+ if time_since_last_submit >= self.min_submit_interval:
139
+ # 满足时间间隔要求,可以提交
140
+ should_submit = True
141
+ reason = f"达到操作次数阈值({self.submit_interval}次)且满足时间间隔({self.min_submit_interval}秒)"
142
+ else:
143
+ # 时间间隔不够,延迟提交
144
+ remaining_time = self.min_submit_interval - time_since_last_submit
145
+ logger.debug("操作次数达标但时间间隔不足,延迟提交", {
146
+ 'instance_name': self.instance_name,
147
+ 'process_id': self.process_id,
148
+ 'total_operations': self.stats['total_operations'],
149
+ 'time_since_last_submit': round(time_since_last_submit, 2),
150
+ 'remaining_time': round(remaining_time, 2)
151
+ })
152
+ elif time_since_last_submit >= self.min_submit_interval:
153
+ # 达到时间间隔阈值
154
+ if self.stats['total_operations'] >= self.submit_interval:
155
+ # 满足操作次数要求,可以提交
156
+ should_submit = True
157
+ reason = f"达到时间间隔阈值({self.min_submit_interval}秒)且满足操作次数({self.submit_interval}次)"
158
+ else:
159
+ # 操作次数不够,不提交
160
+ logger.debug("时间间隔达标但操作次数不足,暂不提交", {
161
+ 'instance_name': self.instance_name,
162
+ 'process_id': self.process_id,
163
+ 'total_operations': self.stats['total_operations'],
164
+ 'required_operations': self.submit_interval,
165
+ 'time_since_last_submit': round(time_since_last_submit, 2)
166
+ })
123
167
 
124
- Args:
125
- namespace: 缓存命名空间
126
- default: 默认TTL值(秒)
168
+ if should_submit and self.stats['total_operations'] > 0:
169
+ try:
170
+ self._submit_to_mysql()
171
+ self.last_submit_time = current_time
172
+ logger.info("统计数据已同步提交", {
173
+ 'instance_name': self.instance_name,
174
+ 'process_id': self.process_id,
175
+ 'total_operations': self.stats['total_operations'],
176
+ 'time_since_last_submit': round(time_since_last_submit, 2),
177
+ 'reason': reason
178
+ })
179
+ except Exception as e:
180
+ logger.error("统计数据提交失败", {
181
+ 'instance_name': self.instance_name,
182
+ 'process_id': self.process_id,
183
+ 'error': str(e)
184
+ })
185
+
186
+ def _submit_to_mysql(self):
187
+ """同步提交统计数据到MySQL"""
188
+ if not self.mysql_pool:
189
+ return
127
190
 
128
- Returns:
129
- int: TTL值(秒)
130
- """
131
- return cls.TTL_STRATEGIES.get(namespace, default)
191
+ stats_data = self.get_stats()
192
+ if not stats_data.get('enabled'):
193
+ return
194
+
195
+ try:
196
+ connection = self.mysql_pool.connection()
197
+ with connection.cursor() as cursor:
198
+ # 选择数据库
199
+ cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
200
+
201
+ # 插入统计数据
202
+ table_name = self.config.get('table_name', 'cache_performance')
203
+ insert_sql = f"""
204
+ INSERT INTO `{table_name}` (
205
+ `日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
206
+ `缓存命中`, `缓存未命中`, `缓存设置`, `缓存删除`, `缓存错误`, `总操作数`,
207
+ `命中率`, `平均响应时间`, `运行时间`, `命名空间统计`
208
+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
209
+ """
210
+
211
+ insert_data = (
212
+ datetime.now().strftime('%Y-%m-%d'),
213
+ f"{self.instance_name}_pid_{self.process_id}",
214
+ socket.gethostname(),
215
+ self.process_id,
216
+ datetime.now(),
217
+ stats_data['hits'],
218
+ stats_data['misses'],
219
+ stats_data['sets'],
220
+ stats_data['deletes'],
221
+ stats_data['errors'],
222
+ stats_data['total_operations'],
223
+ stats_data['hit_rate_percent'],
224
+ stats_data['avg_response_time_ms'],
225
+ stats_data['uptime_seconds'],
226
+ json.dumps(stats_data['namespace_stats'], ensure_ascii=False)
227
+ )
228
+
229
+ cursor.execute(insert_sql, insert_data)
230
+ connection.commit()
231
+
232
+ connection.close()
233
+
234
+ except Exception as e:
235
+ logger.error("MySQL同步提交失败", {
236
+ 'instance_name': self.instance_name,
237
+ 'process_id': self.process_id,
238
+ 'error': str(e)
239
+ })
240
+ raise
132
241
 
133
- @classmethod
134
- def add_strategy(cls, namespace: str, ttl: int):
135
- """
136
- 添加新的TTL策略
242
+ def get_stats(self) -> Dict[str, Any]:
243
+ """获取统计数据"""
244
+ if not self.enabled:
245
+ return {'enabled': False, 'message': '统计功能已禁用'}
137
246
 
138
- Args:
139
- namespace: 命名空间
140
- ttl: TTL值(秒)
141
- """
142
- cls.TTL_STRATEGIES[namespace] = ttl
247
+ with self._lock:
248
+ uptime = time.time() - self.stats['start_time']
249
+ total_cache_ops = self.stats['hits'] + self.stats['misses']
250
+ hit_rate = (self.stats['hits'] / total_cache_ops * 100) if total_cache_ops > 0 else 0
251
+ avg_response_time = statistics.mean(self.response_times) if self.response_times else 0
252
+
253
+ return {
254
+ 'enabled': True,
255
+ 'hits': self.stats['hits'],
256
+ 'misses': self.stats['misses'],
257
+ 'sets': self.stats['sets'],
258
+ 'deletes': self.stats['deletes'],
259
+ 'errors': self.stats['errors'],
260
+ 'total_operations': self.stats['total_operations'],
261
+ 'hit_rate_percent': round(hit_rate, 2),
262
+ 'avg_response_time_ms': round(avg_response_time, 2),
263
+ 'uptime_seconds': round(uptime, 2),
264
+ 'namespace_stats': dict(self.namespace_stats),
265
+ 'last_updated': datetime.now().isoformat(),
266
+ 'process_id': self.process_id
267
+ }
143
268
 
144
- @classmethod
145
- def get_all_strategies(cls) -> Dict[str, int]:
146
- """获取所有TTL策略"""
147
- return cls.TTL_STRATEGIES.copy()
269
+ def reset_stats(self):
270
+ """重置统计数据"""
271
+ if not self.enabled:
272
+ return
273
+
274
+ with self._lock:
275
+ self.stats = {
276
+ 'hits': 0,
277
+ 'misses': 0,
278
+ 'sets': 0,
279
+ 'deletes': 0,
280
+ 'errors': 0,
281
+ 'total_operations': 0,
282
+ 'start_time': time.time()
283
+ }
284
+ self.response_times.clear()
285
+ self.namespace_stats.clear()
286
+ self.last_submit_time = time.time()
287
+
288
+
289
+ class CacheSystemState(enum.Enum):
290
+ """缓存系统状态枚举"""
291
+ INITIALIZING = "initializing"
292
+ READY = "ready"
293
+ MYSQL_READY = "mysql_ready"
294
+ ERROR = "error"
295
+ SHUTDOWN = "shutdown"
148
296
 
149
297
 
150
298
  class CacheConfig:
151
- """缓存系统配置类"""
152
-
153
- def __init__(self, db_name: str = "redis统计", table_name: str = "dpflask路由分析"):
154
- # TTL配置(秒)
155
- self.default_ttl = 3600 # 1小时
156
- self.short_ttl = 300 # 5分钟
157
- self.medium_ttl = 1800 # 30分钟
158
- self.long_ttl = 7200 # 2小时
159
- self.very_long_ttl = 86400 # 24小时
160
-
161
- # 缓存键前缀
162
- self.cache_prefix = "smart_cache:"
163
- self.stats_prefix = "cache_stats:"
164
- self.lock_prefix = "cache_lock:"
165
-
166
- # 统计配置
167
- self.stats_interval = 1800 # 统计间隔(秒), 自动提交统计信息到MySQL的间隔
168
- self.stats_retention = 7 # MySQL统计数据保留天数,超过此天数的数据将被自动删除
169
-
170
- # 性能配置
171
- self.max_key_length = 250
172
- self.max_value_size = 1024 * 1024 # 1MB
173
- self.batch_size = 100
174
-
175
- # 热点键配置
176
- self.max_hot_keys = 1000 # 最大热点键数量
177
- self.hot_keys_cleanup_threshold = 800 # 热点键清理阈值
178
-
179
- # MySQL数据库配置
180
- self.db_name = db_name
181
- self.table_name = table_name
182
-
183
- # 锁配置
184
- self.lock_timeout = 30 # 分布式锁超时时间
185
- self.lock_retry_delay = 0.1 # 锁重试延迟
299
+ """缓存配置类"""
300
+ def __init__(self, **kwargs):
301
+ # 基础配置
302
+ self.default_ttl = kwargs.get('default_ttl', 3600) # 默认过期时间(秒)
303
+ self.stats_interval = kwargs.get('stats_interval', 300) # 每N次操作提交一次统计数据
304
+ self.stats_initial_delay = kwargs.get('stats_initial_delay', 300) # 最少N秒提交一次
305
+ self.enable_stats = kwargs.get('enable_stats', True) # 是否启用统计功能
306
+ self.max_value_size = kwargs.get('max_value_size', 10 * 1024 * 1024) # 最大值大小(字节)
307
+ self.cache_prefix = kwargs.get('cache_prefix', 'cache') # 缓存键前缀
308
+ self.enable_compression = kwargs.get('enable_compression', True)
309
+
310
+ # 数据库配置
311
+ self.db_name = kwargs.get('db_name', 'redis_stats')
312
+ self.table_name = kwargs.get('table_name', 'cache_performance')
313
+
314
+ # 智能TTL策略配置
315
+ self.enable_smart_ttl = kwargs.get('enable_smart_ttl', True) # 启用智能TTL
316
+ self.ttl_min = kwargs.get('ttl_min', 60) # 最小TTL(秒)
317
+ self.ttl_max = kwargs.get('ttl_max', 7200) # 最大TTL(秒)
318
+ self.debug_ttl = kwargs.get('debug_ttl', False) # TTL调试模式
319
+
320
+ # 自定义TTL模式
321
+ self.custom_namespace_patterns = kwargs.get('custom_namespace_patterns', {})
322
+ self.custom_key_patterns = kwargs.get('custom_key_patterns', {})
186
323
 
187
324
 
188
325
  class SmartCacheSystem:
189
- """智能缓存系统核心类"""
326
+ """智能缓存系统 - 单线程"""
190
327
 
191
- def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
192
- config: CacheConfig = None, db_name: str = None, table_name: str = None):
328
+ def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
193
329
  self.redis_client = redis_client
194
330
  self.mysql_pool = mysql_pool
195
331
  self.instance_name = instance_name
332
+ self.config = CacheConfig(**config)
196
333
 
197
- # 配置优先级:传入的config > 自定义db_name/table_name > 默认配置
198
- if config:
199
- self.config = config
200
- elif db_name or table_name:
201
- self.config = CacheConfig(
202
- db_name=db_name or "redis统计",
203
- table_name=table_name or "dpflask路由分析"
204
- )
205
- else:
206
- self.config = CacheConfig()
207
-
208
- self.logger = logger
209
-
210
- # 状态管理
334
+ # 系统状态管理
211
335
  self._state = CacheSystemState.INITIALIZING
212
336
  self._ready_event = Event()
213
- self._mysql_ready_event = Event()
214
-
215
- # 统计数据
216
- self.stats = {
217
- 'hits': 0,
218
- 'misses': 0,
219
- 'sets': 0,
220
- 'deletes': 0,
221
- 'errors': 0,
222
- 'total_operations': 0,
223
- 'start_time': time.time(),
224
- 'response_times': []
225
- }
226
337
 
227
- # 热点键统计
228
- self.hot_keys = {}
229
- self.hot_keys_lock = threading.RLock()
230
-
231
- # 统计线程控制
232
- self._stats_running = False
233
- self._stats_thread = None
234
- self._stats_lock = threading.RLock()
235
-
236
- # 使用线程池异步初始化
237
- self._executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix=f"cache_{instance_name}")
338
+ # 直接初始化统计系统
339
+ self.stats_collector = None
340
+ if self.config.enable_stats:
341
+ self.stats_collector = CacheStatsCollector(
342
+ enabled=True,
343
+ mysql_pool=self.mysql_pool,
344
+ config={
345
+ 'instance_name': self.instance_name,
346
+ 'submit_interval': self.config.stats_interval, # 每N次操作提交一次
347
+ 'min_submit_interval': 300, # 最少N秒提交一次
348
+ 'table_name': self.config.table_name,
349
+ 'db_name': self.config.db_name
350
+ }
351
+ )
238
352
 
239
- # 立即设置为基础就绪状态(Redis缓存可用)
353
+ # 初始化系统
354
+ self._initialize()
355
+
356
+ def _initialize(self):
357
+ """初始化缓存系统"""
358
+ # 测试Redis连接
240
359
  if self._test_redis_connection():
241
360
  self._state = CacheSystemState.READY
242
361
  self._ready_event.set()
243
-
244
- # 异步初始化MySQL相关功能
245
- self._executor.submit(self._initialize_mysql_features)
246
-
247
- self.logger.info("智能缓存系统初始化完成", {
248
- 'instance_name': self.instance_name,
249
- 'state': self._state.value,
250
- 'redis_ready': self._ready_event.is_set(),
251
- 'mysql_enabled': self.mysql_pool is not None
252
- })
253
-
254
- def _initialize_mysql_features(self):
255
- """在后台初始化MySQL相关功能"""
256
- try:
257
- if self.mysql_pool:
258
- # 延迟初始化MySQL(给系统一些启动时间)
259
- time.sleep(5) # 等待5秒让主系统完全启动
260
-
261
- if self._init_mysql_db():
362
+
363
+ # 如果启用统计且有MySQL连接,创建统计表
364
+ if self.config.enable_stats and self.mysql_pool:
365
+ try:
366
+ self._create_simple_stats_table()
262
367
  self._state = CacheSystemState.MYSQL_READY
263
- self._mysql_ready_event.set()
264
-
265
- # MySQL就绪后再启动统计线程
266
- self._start_stats_worker()
267
-
268
- self.logger.info("MySQL功能初始化完成", {
368
+ logger.info("统计功能已启用", {
269
369
  'instance_name': self.instance_name,
270
- 'state': self._state.value
370
+ 'process_id': os.getpid()
271
371
  })
272
- else:
273
- self.logger.warning("MySQL初始化失败,缓存功能仍可正常使用")
274
- else:
275
- self.logger.info("未配置MySQL,跳过统计功能")
276
-
277
- except Exception as e:
372
+ except Exception as e:
373
+ logger.error("统计表创建失败", {
374
+ 'instance_name': self.instance_name,
375
+ 'error': str(e)
376
+ })
377
+ else:
278
378
  self._state = CacheSystemState.ERROR
279
- self.logger.error(f"MySQL功能初始化异常: {e}")
280
-
281
- @property
282
- def is_ready(self) -> bool:
283
- """检查缓存系统是否就绪(Redis可用即为就绪)"""
284
- return self._ready_event.is_set()
285
-
286
- @property
287
- def is_mysql_ready(self) -> bool:
288
- """检查MySQL功能是否就绪"""
289
- return self._mysql_ready_event.is_set()
290
-
291
- def wait_for_ready(self, timeout: float = 1.0) -> bool:
292
- """等待缓存系统就绪(非阻塞,有超时)"""
293
- return self._ready_event.wait(timeout)
294
-
295
- def wait_for_mysql_ready(self, timeout: float = 5.0) -> bool:
296
- """等待MySQL功能就绪(非阻塞,有超时)"""
297
- return self._mysql_ready_event.wait(timeout)
379
+ logger.error("Redis连接失败", {'instance_name': self.instance_name})
298
380
 
299
381
  def _test_redis_connection(self) -> bool:
300
382
  """测试Redis连接"""
@@ -302,652 +384,470 @@ class SmartCacheSystem:
302
384
  self.redis_client.ping()
303
385
  return True
304
386
  except Exception as e:
305
- self.logger.error(f"Redis连接测试失败: {e}")
387
+ logger.error("Redis连接测试失败", {'error': str(e)})
306
388
  return False
307
389
 
308
- def _init_mysql_db(self) -> bool:
309
- """初始化MySQL数据库和表"""
390
+ def _create_simple_stats_table(self):
391
+ """创建统计表"""
310
392
  if not self.mysql_pool:
311
- self.logger.warning("MySQL连接池未提供,统计功能将被禁用")
312
- return False
393
+ return
313
394
 
314
395
  try:
315
- # 设置连接超时,避免长时间阻塞
316
396
  connection = self.mysql_pool.connection()
317
- # 设置查询超时(如果支持)
318
- try:
319
- connection.autocommit(False) # 确保事务控制
320
- except:
321
- pass # 忽略不支持的操作
397
+ with connection.cursor() as cursor:
398
+ # 创建数据库
399
+ cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci")
400
+ cursor.execute(f"USE `{self.config.db_name}`")
322
401
 
323
- try:
324
- with connection.cursor() as cursor:
325
- # 设置会话超时
326
- try:
327
- cursor.execute("SET SESSION wait_timeout = 30")
328
- cursor.execute("SET SESSION interactive_timeout = 30")
329
- except:
330
- pass # 忽略不支持的操作
402
+ # 创建统计表
403
+ create_table_sql = f"""
404
+ CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
405
+ `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
406
+ `日期` date NOT NULL COMMENT '统计日期',
407
+ `实例标识` varchar(64) NOT NULL COMMENT '缓存实例标识',
408
+ `主机名` varchar(100) NOT NULL COMMENT '服务器主机名',
409
+ `进程ID` int NOT NULL COMMENT '进程ID',
410
+ `统计时间` timestamp NOT NULL COMMENT '统计时间',
331
411
 
332
- # 创建数据库
333
- cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
334
- cursor.execute(f"USE `{self.config.db_name}`")
412
+ -- 基础操作统计
413
+ `缓存命中` bigint DEFAULT 0 COMMENT '缓存命中次数',
414
+ `缓存未命中` bigint DEFAULT 0 COMMENT '缓存未命中次数',
415
+ `缓存设置` bigint DEFAULT 0 COMMENT '缓存设置次数',
416
+ `缓存删除` bigint DEFAULT 0 COMMENT '缓存删除次数',
417
+ `缓存错误` bigint DEFAULT 0 COMMENT '缓存错误次数',
418
+ `总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
335
419
 
336
- # 创建表(MySQL 8.4+兼容语法)
337
- create_table_sql = f"""
338
- CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
339
- `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
340
- `日期` date NOT NULL COMMENT '统计日期',
341
- `统计时间` datetime NOT NULL COMMENT '统计时间',
342
- `时间段` varchar(20) NOT NULL COMMENT '时间段标识',
343
- `缓存命中数` bigint DEFAULT 0 COMMENT '缓存命中次数',
344
- `缓存未命中数` bigint DEFAULT 0 COMMENT '缓存未命中次数',
345
- `缓存设置数` bigint DEFAULT 0 COMMENT '缓存设置次数',
346
- `缓存删除数` bigint DEFAULT 0 COMMENT '缓存删除次数',
347
- `缓存错误数` bigint DEFAULT 0 COMMENT '缓存错误次数',
348
- `命中率` decimal(5,2) DEFAULT 0.00 COMMENT '缓存命中率(%)',
349
- `总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
350
- `平均响应时间` decimal(10,4) DEFAULT 0.0000 COMMENT '平均响应时间(ms)',
351
- `每秒操作数` decimal(10,2) DEFAULT 0.00 COMMENT '每秒操作数',
352
- `唯一键数量` int DEFAULT 0 COMMENT '唯一键数量',
353
- `系统运行时间` bigint DEFAULT 0 COMMENT '系统运行时间(秒)',
354
- `热点键统计` json DEFAULT NULL COMMENT '热点键统计信息',
355
- `服务器主机` varchar(100) DEFAULT NULL COMMENT '服务器主机名',
356
- `实例名称` varchar(100) DEFAULT NULL COMMENT '缓存实例名称',
357
- `创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
358
- `更新时间` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
359
- PRIMARY KEY (`id`),
360
- KEY `idx_stats_date` (`日期`),
361
- KEY `idx_stats_time` (`统计时间`),
362
- KEY `idx_time_period` (`时间段`),
363
- KEY `idx_hit_rate` (`命中率`),
364
- KEY `idx_instance` (`实例名称`),
365
- KEY `idx_create_time` (`创建时间`)
366
- ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存系统统计分析表'
367
- """
420
+ -- 性能指标
421
+ `命中率` decimal(5,2) DEFAULT 0.00 COMMENT '命中率百分比',
422
+ `平均响应时间` decimal(10,2) DEFAULT 0.00 COMMENT '平均响应时间(毫秒)',
423
+ `运行时间` bigint DEFAULT 0 COMMENT '运行时间(秒)',
368
424
 
369
- cursor.execute(create_table_sql)
370
- connection.commit()
425
+ -- 命名空间统计
426
+ `命名空间统计` json COMMENT '命名空间统计详情',
371
427
 
372
- self.logger.info("MySQL数据库表初始化成功", {
373
- 'database': self.config.db_name,
374
- 'table': self.config.table_name
375
- })
376
- return True
428
+ -- 时间戳
429
+ `创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
377
430
 
378
- finally:
379
- connection.close()
431
+ PRIMARY KEY (`id`),
432
+ KEY `idx_日期` (`日期`),
433
+ KEY `idx_实例时间` (`实例标识`, `统计时间`),
434
+ KEY `idx_创建时间` (`创建时间`)
435
+ ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存统计表'
436
+ """
380
437
 
381
- except Exception as e:
382
- self.logger.error(f"MySQL数据库初始化失败: {e}")
383
- # 不抛出异常,让系统继续运行
384
- return False
385
-
386
- def _generate_cache_key(self, key: str, namespace: str = "") -> str:
387
- """生成缓存键"""
388
- if namespace:
389
- return f"{self.config.cache_prefix}{namespace}:{key}"
390
- return f"{self.config.cache_prefix}{key}"
391
-
392
- def _record_operation(self, operation: str, response_time: float = 0):
393
- """记录操作统计"""
394
- with self._stats_lock:
395
- self.stats['total_operations'] += 1
396
- if operation in self.stats:
397
- self.stats[operation] += 1
398
- if response_time > 0:
399
- self.stats['response_times'].append(response_time)
400
- # 只保留最近1000次操作的响应时间
401
- if len(self.stats['response_times']) > 1000:
402
- self.stats['response_times'] = self.stats['response_times'][-1000:]
403
-
404
- def _record_hot_key(self, key: str, namespace: str = ""):
405
- """记录热点键"""
406
- cache_key = self._generate_cache_key(key, namespace)
407
- with self.hot_keys_lock:
408
- # 检查是否需要清理热点键(防止内存泄漏)
409
- if len(self.hot_keys) >= self.config.hot_keys_cleanup_threshold:
410
- # 保留访问次数最高的键,移除访问次数最少的键
411
- sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
412
- # 保留前600个热点键,为新键留出空间
413
- self.hot_keys = dict(sorted_keys[:600])
438
+ cursor.execute(create_table_sql)
439
+ connection.commit()
414
440
 
415
- self.logger.info("热点键内存清理", {
416
- '清理前数量': len(sorted_keys),
417
- '清理后数量': len(self.hot_keys),
418
- '清理阈值': self.config.hot_keys_cleanup_threshold
419
- })
441
+ connection.close()
420
442
 
421
- # 记录或更新热点键访问次数
422
- self.hot_keys[cache_key] = self.hot_keys.get(cache_key, 0) + 1
443
+ except Exception as e:
444
+ logger.error("统计表初始化失败", {'error': str(e)})
445
+ raise
423
446
 
424
- def _cache_operation(operation_name: str):
425
- """缓存操作装饰器工厂"""
426
- def decorator(func):
427
- @functools.wraps(func)
428
- def wrapper(self, *args, **kwargs):
429
- start_time = time.time()
430
-
431
- # 检查系统是否就绪
432
- if not self.is_ready:
433
- self._record_operation('errors')
434
- self.logger.warning(f"缓存系统未就绪,{operation_name}操作失败", {
435
- 'state': self._state.value
436
- })
437
- return None if operation_name == 'get' else False
438
-
439
- try:
440
- result = func(self, *args, **kwargs)
441
- response_time = (time.time() - start_time) * 1000
442
- self._record_operation(operation_name, response_time)
443
- return result
444
-
445
- except Exception as e:
446
- self._record_operation('errors')
447
- self.logger.error(f"缓存{operation_name}操作失败: {e}", {
448
- 'args': args[:2] if len(args) > 2 else args, # 避免记录过长的参数
449
- 'operation': operation_name
450
- })
451
- return None if operation_name == 'get' else False
452
-
453
- return wrapper
454
- return decorator
447
+ @property
448
+ def is_ready(self) -> bool:
449
+ """检查系统是否就绪"""
450
+ return self._ready_event.is_set()
451
+
452
+ @property
453
+ def is_mysql_ready(self) -> bool:
454
+ """检查MySQL是否就绪"""
455
+ return self._state == CacheSystemState.MYSQL_READY
455
456
 
456
- @_cache_operation('get')
457
457
  def get(self, key: str, namespace: str = "", default=None) -> Any:
458
458
  """获取缓存值"""
459
- cache_key = self._generate_cache_key(key, namespace)
460
-
461
- # 获取缓存值
462
- value = self.redis_client.get(cache_key)
459
+ return self._get_with_stats(key, namespace, default)
460
+
461
+ def _get_with_stats(self, key: str, namespace: str = "", default=None) -> Any:
462
+ """带统计的获取缓存值"""
463
+ if not self.is_ready:
464
+ return default
463
465
 
464
- if value is not None:
465
- # 缓存命中
466
- self._record_hot_key(key, namespace)
466
+ start_time = time.time()
467
+ try:
468
+ cache_key = self._generate_cache_key(key, namespace)
469
+ value = self.redis_client.get(cache_key)
470
+ response_time = (time.time() - start_time) * 1000
467
471
 
468
- try:
469
- return json.loads(value.decode('utf-8'))
470
- except (json.JSONDecodeError, UnicodeDecodeError):
471
- return value.decode('utf-8')
472
- else:
473
- # 缓存未命中
472
+ if value is not None:
473
+ # 缓存命中
474
+ if self.stats_collector:
475
+ self.stats_collector.record_operation('hits', response_time, namespace)
476
+ try:
477
+ return json.loads(value.decode('utf-8'))
478
+ except (json.JSONDecodeError, UnicodeDecodeError):
479
+ return value.decode('utf-8')
480
+ else:
481
+ # 缓存未命中
482
+ if self.stats_collector:
483
+ self.stats_collector.record_operation('misses', response_time, namespace)
484
+ return default
485
+
486
+ except Exception as e:
487
+ response_time = (time.time() - start_time) * 1000
488
+ if self.stats_collector:
489
+ self.stats_collector.record_operation('errors', response_time, namespace)
490
+ logger.error("缓存获取失败", {
491
+ 'key': key,
492
+ 'namespace': namespace,
493
+ 'error': str(e)
494
+ })
474
495
  return default
475
496
 
476
- @_cache_operation('set')
477
- def set(self, key: str, value: Any, ttl: int = None, namespace: str = "") -> bool:
497
+ def set(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
478
498
  """设置缓存值"""
479
- cache_key = self._generate_cache_key(key, namespace)
480
-
481
- # 使用智能TTL策略
482
- if ttl is None:
483
- ttl = SmartTTLConfig.get_ttl(namespace, self.config.default_ttl)
484
-
485
- # 序列化值
486
- if isinstance(value, (dict, list, tuple)):
487
- serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
488
- else:
489
- serialized_value = str(value)
490
-
491
- # 检查值大小
492
- if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
493
- self.logger.warning(f"缓存值过大,跳过设置: {len(serialized_value)} bytes")
499
+ return self._set_with_stats(key, value, ttl, namespace)
500
+
501
+ def _set_with_stats(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
502
+ """带统计的设置缓存值"""
503
+ if not self.is_ready:
494
504
  return False
495
505
 
496
- # 设置缓存
497
- result = self.redis_client.setex(cache_key, ttl, serialized_value)
498
-
499
- # 记录TTL策略使用情况
500
- if namespace and ttl != self.config.default_ttl:
501
- self.logger.debug("使用智能TTL策略", {
502
- 'namespace': namespace,
503
- 'ttl': ttl,
504
- 'key': key[:50] + "..." if len(key) > 50 else key
505
- })
506
-
507
- return bool(result)
508
-
509
- def delete(self, key: str, namespace: str = "") -> bool:
510
- """删除缓存值"""
511
506
  start_time = time.time()
512
-
513
507
  try:
514
508
  cache_key = self._generate_cache_key(key, namespace)
515
- result = self.redis_client.delete(cache_key)
509
+
510
+ # 智能TTL策略
511
+ if ttl is None:
512
+ ttl = self._get_smart_ttl(namespace, key, len(json.dumps(value, ensure_ascii=False, default=str)))
513
+
514
+ # 序列化值
515
+ if isinstance(value, (dict, list)):
516
+ serialized_value = json.dumps(value, ensure_ascii=False, default=str)
517
+ else:
518
+ serialized_value = str(value)
519
+
520
+ # 检查值大小
521
+ value_size = len(serialized_value.encode('utf-8'))
522
+ if value_size > self.config.max_value_size:
523
+ if self.stats_collector:
524
+ self.stats_collector.record_operation('errors', 0, namespace)
525
+ logger.warning("缓存值过大,跳过设置", {
526
+ 'key': key,
527
+ 'size': len(serialized_value),
528
+ 'max_size': self.config.max_value_size
529
+ })
530
+ return False
531
+
532
+ result = self.redis_client.setex(cache_key, ttl, serialized_value)
516
533
  response_time = (time.time() - start_time) * 1000
517
534
 
518
- self._record_operation('deletes', response_time)
535
+ if self.stats_collector:
536
+ self.stats_collector.record_operation('sets', response_time, namespace)
519
537
  return bool(result)
520
538
 
521
539
  except Exception as e:
522
- self._record_operation('errors')
523
- self.logger.error(f"缓存删除失败: {e}", {
540
+ response_time = (time.time() - start_time) * 1000
541
+ if self.stats_collector:
542
+ self.stats_collector.record_operation('errors', response_time, namespace)
543
+ logger.error("缓存设置失败", {
524
544
  'key': key,
525
- 'namespace': namespace
545
+ 'namespace': namespace,
546
+ 'error': str(e)
526
547
  })
527
548
  return False
528
549
 
529
- def exists(self, key: str, namespace: str = "") -> bool:
530
- """检查缓存键是否存在"""
550
+ def delete(self, key: str, namespace: str = "") -> bool:
551
+ """删除缓存值"""
552
+ return self._delete_with_stats(key, namespace)
553
+
554
+ def _delete_with_stats(self, key: str, namespace: str = "") -> bool:
555
+ """带统计的删除缓存值"""
556
+ if not self.is_ready:
557
+ return False
558
+
559
+ start_time = time.time()
531
560
  try:
532
561
  cache_key = self._generate_cache_key(key, namespace)
533
- return bool(self.redis_client.exists(cache_key))
562
+ result = self.redis_client.delete(cache_key)
563
+ response_time = (time.time() - start_time) * 1000
564
+ if self.stats_collector:
565
+ self.stats_collector.record_operation('deletes', response_time, namespace)
566
+ return bool(result)
567
+
534
568
  except Exception as e:
535
- self.logger.error(f"缓存存在性检查失败: {e}")
569
+ response_time = (time.time() - start_time) * 1000
570
+ if self.stats_collector:
571
+ self.stats_collector.record_operation('errors', response_time, namespace)
572
+ logger.error("缓存删除失败", {
573
+ 'key': key,
574
+ 'namespace': namespace,
575
+ 'error': str(e)
576
+ })
536
577
  return False
537
578
 
538
579
  def clear_namespace(self, namespace: str) -> int:
539
580
  """清除指定命名空间的所有缓存"""
581
+ if not self.is_ready:
582
+ return 0
583
+
540
584
  try:
541
- pattern = f"{self.config.cache_prefix}{namespace}:*"
585
+ pattern = f"{self.config.cache_prefix}:{namespace}:*"
542
586
  keys = self.redis_client.keys(pattern)
543
587
 
544
588
  if keys:
545
589
  deleted = self.redis_client.delete(*keys)
546
- self.logger.info(f"清除命名空间缓存: {namespace}, 删除键数: {deleted}")
547
590
  return deleted
548
591
  return 0
549
592
 
550
593
  except Exception as e:
551
- self.logger.error(f"清除命名空间缓存失败: {e}")
594
+ logger.error("清除命名空间失败", {
595
+ 'namespace': namespace,
596
+ 'error': str(e)
597
+ })
552
598
  return 0
553
599
 
554
- def get_stats(self) -> Dict[str, Any]:
555
- """获取缓存统计信息"""
556
- with self._stats_lock:
557
- total_ops = self.stats['total_operations']
558
- hits = self.stats['hits']
600
+ def _generate_cache_key(self, key: str, namespace: str = "") -> str:
601
+ """生成缓存键"""
602
+ if namespace:
603
+ return f"{self.config.cache_prefix}:{namespace}:{key}"
604
+ return f"{self.config.cache_prefix}:{key}"
605
+
606
+ def _get_smart_ttl(self, namespace: str, key: str = "", data_size: int = 0) -> int:
607
+ """智能TTL策略 - 通用版本
608
+
609
+ 基于多种因素智能计算TTL:
610
+ 1. 命名空间模式匹配
611
+ 2. 数据类型推断
612
+ 3. 数据大小考虑
613
+ 4. 访问频率预测
614
+ """
615
+
616
+ # 如果禁用智能TTL,直接返回默认值
617
+ if not self.config.enable_smart_ttl:
618
+ return self.config.default_ttl
619
+
620
+ # 1. 基于命名空间模式的智能匹配
621
+ namespace_patterns = {
622
+ # 数据库相关 - 变化频率低,TTL较长
623
+ r'.*database.*|.*db.*|.*schema.*': 1800, # 30分钟
559
624
 
560
- # 计算命中率
561
- hit_rate = (hits / total_ops * 100) if total_ops > 0 else 0
625
+ # 表结构相关 - 变化频率低,TTL较长
626
+ r'.*table.*|.*column.*|.*field.*': 1200, # 20分钟
562
627
 
563
- # 计算平均响应时间
564
- response_times = self.stats['response_times']
565
- avg_response_time = sum(response_times) / len(response_times) if response_times else 0
628
+ # 数据查询相关 - 变化频率中等,TTL中等
629
+ r'.*data.*|.*query.*|.*result.*': 300, # 5分钟
566
630
 
567
- # 计算运行时间
568
- uptime = time.time() - self.stats['start_time']
631
+ # 用户会话相关 - 变化频率高,TTL较短
632
+ r'.*session.*|.*user.*|.*auth.*': 900, # 15分钟
569
633
 
570
- # 计算每秒操作数
571
- ops_per_second = total_ops / uptime if uptime > 0 else 0
634
+ # 配置相关 - 变化频率很低,TTL很长
635
+ r'.*config.*|.*setting.*|.*param.*': 3600, # 1小时
572
636
 
573
- # 获取热点键(前10个)
574
- with self.hot_keys_lock:
575
- top_hot_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)[:10]
637
+ # 统计相关 - 变化频率中等,TTL中等
638
+ r'.*stats.*|.*metric.*|.*count.*': 600, # 10分钟
576
639
 
577
- return {
578
- 'hits': hits,
579
- 'misses': self.stats['misses'],
580
- 'sets': self.stats['sets'],
581
- 'deletes': self.stats['deletes'],
582
- 'errors': self.stats['errors'],
583
- 'total_operations': total_ops,
584
- 'hit_rate': round(hit_rate, 2),
585
- 'avg_response_time': round(avg_response_time, 4),
586
- 'ops_per_second': round(ops_per_second, 2),
587
- 'uptime_seconds': int(uptime),
588
- 'hot_keys': dict(top_hot_keys),
589
- 'unique_keys_count': len(self.hot_keys)
590
- }
591
-
592
- def health_check(self) -> Dict[str, Any]:
593
- """健康检查"""
594
- health_info = {
595
- 'redis_connected': False,
596
- 'mysql_available': False,
597
- 'stats_worker_running': self._stats_running,
598
- 'instance_name': self.instance_name,
599
- 'timestamp': datetime.now().isoformat()
640
+ # 缓存相关 - 变化频率高,TTL较短
641
+ r'.*cache.*|.*temp.*|.*tmp.*': 180, # 3分钟
642
+
643
+ # 列表相关 - 变化频率中等,TTL中等
644
+ r'.*list.*|.*index.*|.*catalog.*': 450, # 7.5分钟
600
645
  }
601
646
 
602
- # 检查Redis连接
603
- try:
604
- self.redis_client.ping()
605
- health_info['redis_connected'] = True
606
- except Exception as e:
607
- health_info['redis_error'] = str(e)
647
+ # 合并用户自定义模式(优先级更高)
648
+ namespace_patterns.update(self.config.custom_namespace_patterns)
608
649
 
609
- # 检查MySQL连接
610
- if self.mysql_pool:
611
- try:
612
- connection = self.mysql_pool.connection()
613
- connection.close()
614
- health_info['mysql_available'] = True
615
- except Exception as e:
616
- health_info['mysql_error'] = str(e)
650
+ # 2. 基于键名模式的智能匹配
651
+ key_patterns = {
652
+ # ID相关 - 相对稳定
653
+ r'.*_id$|^id_.*': 1.2, # TTL倍数
654
+
655
+ # 详情相关 - 相对稳定
656
+ r'.*detail.*|.*info.*': 1.1,
657
+
658
+ # 列表相关 - 变化较频繁
659
+ r'.*list.*|.*items.*': 0.8,
660
+
661
+ # 计数相关 - 变化频繁
662
+ r'.*count.*|.*num.*|.*total.*': 0.6,
663
+
664
+ # 状态相关 - 变化很频繁
665
+ r'.*status.*|.*state.*': 0.5,
666
+ }
617
667
 
618
- return health_info
619
-
620
- def _start_stats_worker(self):
621
- """启动统计工作线程"""
622
- if not self._stats_running and self.mysql_pool:
623
- self._stats_running = True
624
- self._stats_thread = threading.Thread(
625
- target=self._stats_worker,
626
- daemon=True,
627
- name=f"stats_worker_{self.instance_name}"
628
- )
629
- self._stats_thread.start()
630
- self.logger.info("统计工作线程已启动", {
631
- 'instance_name': self.instance_name,
632
- 'delay_first_run': True
668
+ # 合并用户自定义键模式
669
+ key_patterns.update(self.config.custom_key_patterns)
670
+
671
+ # 3. 基于数据大小的调整
672
+ size_factor = 1.0
673
+ if data_size > 0:
674
+ if data_size > 1024 * 1024: # > 1MB,延长TTL减少重建开销
675
+ size_factor = 1.5
676
+ elif data_size > 100 * 1024: # > 100KB
677
+ size_factor = 1.2
678
+ elif data_size < 1024: # < 1KB,缩短TTL减少内存占用
679
+ size_factor = 0.8
680
+
681
+ # 4. 计算基础TTL
682
+ base_ttl = self.config.default_ttl
683
+
684
+ # 匹配命名空间模式
685
+ for pattern, ttl in namespace_patterns.items():
686
+ if re.match(pattern, namespace.lower()):
687
+ base_ttl = ttl
688
+ break
689
+
690
+ # 5. 应用键名模式调整
691
+ key_factor = 1.0
692
+ for pattern, factor in key_patterns.items():
693
+ if re.match(pattern, key.lower()):
694
+ key_factor = factor
695
+ break
696
+
697
+ # 6. 计算最终TTL
698
+ final_ttl = int(base_ttl * key_factor * size_factor)
699
+
700
+ # 7. TTL边界限制(使用配置值)
701
+ final_ttl = max(self.config.ttl_min, min(self.config.ttl_max, final_ttl))
702
+
703
+ # 8. 记录智能TTL决策(使用配置开关)
704
+ if self.config.debug_ttl:
705
+ logger.debug("智能TTL计算", {
706
+ 'namespace': namespace,
707
+ 'key': key[:50] + "..." if len(key) > 50 else key,
708
+ 'data_size': data_size,
709
+ 'base_ttl': base_ttl,
710
+ 'key_factor': key_factor,
711
+ 'size_factor': size_factor,
712
+ 'final_ttl': final_ttl
633
713
  })
714
+
715
+ return final_ttl
634
716
 
635
- def _stats_worker(self):
636
- """后台统计工作线程"""
637
- cleanup_counter = 0
717
+ def get_stats(self) -> Dict[str, Any]:
718
+ """获取统计信息"""
719
+ # 确保统计系统已初始化
720
+ if self.stats_collector:
721
+ return self.stats_collector.get_stats()
638
722
 
639
- # 延迟启动:等待一个完整的统计间隔
640
- self.logger.info("统计线程启动,等待首个统计间隔", {
723
+ logger.debug("统计系统未初始化,返回空统计信息", {
641
724
  'instance_name': self.instance_name,
642
- 'interval': self.config.stats_interval,
643
- 'delay_reason': '避免初始化时的阻塞操作'
725
+ 'process_id': os.getpid()
644
726
  })
645
-
646
- # 使用可中断的等待
647
- if not self._interruptible_sleep(self.config.stats_interval):
648
- return # 如果被中断则退出
649
-
650
- while self._stats_running:
651
- try:
652
- # 收集统计数据
653
- stats_data = self.get_stats()
654
-
655
- # 提交到MySQL(带状态检查)
656
- self._submit_stats_to_mysql(stats_data)
657
-
658
- # 清理过期的热点键统计
659
- self._cleanup_hot_keys()
660
-
661
- # 定期清理过期数据
662
- cleanup_counter += 1
663
- if cleanup_counter >= 24: # 24 * 1800秒 = 12小时
664
- self._cleanup_expired_mysql_data()
665
- cleanup_counter = 0
666
-
667
- except Exception as e:
668
- self.logger.error(f"统计工作线程异常: {e}")
669
-
670
- # 可中断的等待下一个统计间隔
671
- if not self._interruptible_sleep(self.config.stats_interval):
672
- break
727
+ return {
728
+ 'enabled': False,
729
+ 'message': '统计系统未初始化',
730
+ 'process_id': os.getpid()
731
+ }
673
732
 
674
- def _interruptible_sleep(self, duration: float) -> bool:
675
- """可中断的睡眠"""
676
- sleep_interval = 1.0 # 每秒检查一次
677
- elapsed = 0.0
678
-
679
- while elapsed < duration and self._stats_running:
680
- time.sleep(min(sleep_interval, duration - elapsed))
681
- elapsed += sleep_interval
733
+ def health_check(self) -> Dict[str, Any]:
734
+ """健康检查"""
735
+ try:
736
+ # Redis连接检查
737
+ redis_latency = time.time()
738
+ self.redis_client.ping()
739
+ redis_latency = (time.time() - redis_latency) * 1000
682
740
 
683
- return self._stats_running
684
-
685
- def _submit_stats_to_mysql(self, stats_data: Dict[str, Any]):
686
- """提交统计数据到MySQL"""
687
- if not self.mysql_pool:
688
- self.logger.debug("MySQL连接池不可用,跳过统计数据提交")
689
- return
741
+ # 获取Redis信息
742
+ redis_info = self.redis_client.info()
690
743
 
691
- # 使用状态检查而不是属性检查
692
- if not self.is_mysql_ready:
693
- self.logger.debug("MySQL尚未就绪,跳过统计数据提交", {
694
- 'state': self._state.value,
695
- 'instance_name': self.instance_name
696
- })
697
- return
744
+ return {
745
+ 'status': 'healthy',
746
+ 'redis_connected': True,
747
+ 'redis_latency_ms': round(redis_latency, 2),
748
+ 'mysql_enabled': self.mysql_pool is not None,
749
+ 'mysql_ready': self.is_mysql_ready,
750
+ 'system_state': self._state.value,
751
+ 'uptime_seconds': time.time() - (self.stats_collector.stats['start_time'] if self.stats_collector else time.time()),
752
+ 'redis_memory_used': redis_info.get('used_memory', 0),
753
+ 'redis_connected_clients': redis_info.get('connected_clients', 0),
754
+ 'process_id': os.getpid()
755
+ }
698
756
 
699
- logger.info("统计数据", {'stats_data': stats_data})
700
-
701
- try:
702
- # 使用上下文管理器确保连接正确关闭
703
- connection = self.mysql_pool.connection()
704
- try:
705
- with connection.cursor() as cursor:
706
- cursor.execute(f"USE `{self.config.db_name}`")
707
-
708
- # 准备数据
709
- now = datetime.now()
710
- date_str = now.strftime("%Y-%m-%d")
711
- time_period = now.strftime("%Y%m%d_%H%M")
712
-
713
- insert_sql = f"""
714
- INSERT INTO `{self.config.table_name}` (
715
- `日期`, `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
716
- `缓存删除数`, `缓存错误数`, `命中率`, `总操作数`, `平均响应时间`,
717
- `每秒操作数`, `唯一键数量`, `系统运行时间`, `热点键统计`,
718
- `服务器主机`, `实例名称`
719
- ) VALUES (
720
- %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
721
- )
722
- """
723
-
724
- cursor.execute(insert_sql, (
725
- date_str,
726
- now,
727
- time_period,
728
- stats_data['hits'],
729
- stats_data['misses'],
730
- stats_data['sets'],
731
- stats_data['deletes'],
732
- stats_data['errors'],
733
- stats_data['hit_rate'],
734
- stats_data['total_operations'],
735
- stats_data['avg_response_time'],
736
- stats_data['ops_per_second'],
737
- stats_data['unique_keys_count'],
738
- stats_data['uptime_seconds'],
739
- json.dumps(stats_data['hot_keys'], ensure_ascii=False, cls=MySQLDataEncoder),
740
- socket.gethostname(),
741
- self.instance_name
742
- ))
743
-
744
- connection.commit()
745
-
746
- self.logger.info("统计数据已提交到MySQL", {
747
- 'time_period': time_period,
748
- 'total_operations': stats_data['total_operations'],
749
- 'hit_rate': stats_data['hit_rate'],
750
- 'instance_name': self.instance_name
751
- })
752
-
753
- finally:
754
- connection.close()
755
-
756
757
  except Exception as e:
757
- self.logger.error(f"提交统计数据到MySQL失败: {e}", {
758
- 'instance_name': self.instance_name,
759
- 'error_type': type(e).__name__
760
- })
761
-
762
- def _cleanup_hot_keys(self):
763
- """清理热点键统计 """
764
- with self.hot_keys_lock:
765
- current_count = len(self.hot_keys)
766
-
767
- # 如果热点键数量超过最大限制,进行清理
768
- if current_count > self.config.max_hot_keys:
769
- # 保留访问次数最高的键
770
- sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
771
- # 保留80%的热点键
772
- keep_count = int(self.config.max_hot_keys * 0.8)
773
- self.hot_keys = dict(sorted_keys[:keep_count])
774
-
775
- self.logger.info("定期热点键清理", {
776
- '清理前数量': current_count,
777
- '清理后数量': len(self.hot_keys),
778
- '保留比例': '80%'
779
- })
758
+ return {
759
+ 'status': 'unhealthy',
760
+ 'error': str(e),
761
+ 'redis_connected': False,
762
+ 'system_state': self._state.value
763
+ }
780
764
 
781
- def _cleanup_expired_mysql_data(self):
782
- """清理过期的MySQL统计数据"""
783
- if not self.mysql_pool:
765
+ def _record_operation(self, operation: str, response_time: float = 0, namespace: str = "", key: str = ""):
766
+ """记录操作统计 """
767
+ # 如果禁用统计功能,直接返回
768
+ if not self.config.enable_stats:
784
769
  return
770
+
771
+ logger.debug("调用操作记录", {
772
+ 'operation': operation,
773
+ 'response_time_ms': round(response_time, 2),
774
+ 'namespace': namespace,
775
+ 'key': key[:50] + "..." if len(key) > 50 else key,
776
+ 'instance_name': self.instance_name
777
+ })
785
778
 
786
- try:
787
- connection = self.mysql_pool.connection()
788
- try:
789
- with connection.cursor() as cursor:
790
- cursor.execute(f"USE `{self.config.db_name}`")
791
-
792
- # 计算过期时间点
793
- from datetime import timedelta
794
- expire_date = datetime.now() - timedelta(days=self.config.stats_retention)
795
-
796
- # 删除过期数据
797
- delete_sql = f"""
798
- DELETE FROM `{self.config.table_name}`
799
- WHERE `统计时间` < %s
800
- """
801
-
802
- cursor.execute(delete_sql, (expire_date,))
803
- deleted_rows = cursor.rowcount
804
- connection.commit()
805
-
806
- if deleted_rows > 0:
807
- self.logger.info("清理过期MySQL统计数据", {
808
- 'deleted_rows': deleted_rows,
809
- 'expire_date': expire_date.strftime('%Y-%m-%d %H:%M:%S'),
810
- 'retention_days': self.config.stats_retention
811
- })
812
-
813
- finally:
814
- connection.close()
815
-
816
- except Exception as e:
817
- self.logger.error(f"清理过期MySQL统计数据失败: {e}")
818
-
819
779
  def shutdown(self):
820
780
  """关闭缓存系统"""
821
- self.logger.info("正在关闭缓存系统...", {
822
- 'instance_name': self.instance_name,
823
- 'state': self._state.value
824
- })
781
+ self._state = CacheSystemState.SHUTDOWN
825
782
 
826
- # 停止统计线程
827
- self._stats_running = False
828
- if self._stats_thread and self._stats_thread.is_alive():
829
- self.logger.info("等待统计线程结束...")
830
- self._stats_thread.join(timeout=10) # 最多等待10秒
831
-
832
- if self._stats_thread.is_alive():
833
- self.logger.warning("统计线程未能在超时时间内结束")
834
-
835
- # 关闭线程池
836
- if hasattr(self, '_executor'):
837
- self.logger.info("关闭线程池...")
838
- self._executor.shutdown(wait=True, timeout=5)
839
-
840
- # 最后一次提交统计数据(如果MySQL可用)
841
- if self.is_mysql_ready:
842
- try:
843
- self.logger.info("提交最终统计数据...")
844
- stats_data = self.get_stats()
845
- self._submit_stats_to_mysql(stats_data)
846
- except Exception as e:
847
- self.logger.error(f"关闭时提交统计数据失败: {e}")
783
+ if self.stats_collector:
784
+ # The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
785
+ self.stats_collector.reset_stats()
848
786
 
849
- self.logger.info("缓存系统已关闭", {
850
- 'instance_name': self.instance_name
851
- })
852
-
853
- def __enter__(self):
854
- """上下文管理器支持"""
855
- return self
856
-
857
- def __exit__(self, exc_type, exc_val, exc_tb):
858
- """上下文管理器退出时自动关闭"""
859
- self.shutdown()
860
- return False
787
+ logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
861
788
 
862
789
 
863
790
  class CacheManager:
864
- """缓存管理器 - 单例模式"""
791
+ """缓存管理器"""
865
792
 
866
793
  _instance = None
867
- _lock = threading.RLock()
794
+ _lock = threading.Lock()
868
795
 
869
796
  def __new__(cls):
870
- with cls._lock:
871
- if cls._instance is None:
872
- cls._instance = super().__new__(cls)
873
- cls._instance._initialized = False
874
- return cls._instance
797
+ if cls._instance is None:
798
+ with cls._lock:
799
+ if cls._instance is None:
800
+ cls._instance = super().__new__(cls)
801
+ return cls._instance
875
802
 
876
803
  def __init__(self):
877
- if self._initialized:
804
+ if not hasattr(self, '_initialized'):
805
+ self.cache_instance = None
806
+ self._initialized = True
807
+
808
+ def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
809
+ """初始化缓存系统"""
810
+ if self.cache_instance is not None:
811
+ logger.warning("缓存系统已初始化,跳过重复初始化", {
812
+ 'existing_instance': self.cache_instance.instance_name,
813
+ 'new_instance': instance_name
814
+ })
878
815
  return
879
816
 
880
- self.cache_instance = None
881
- self.enabled = True
882
- self.initialization_error = None
883
- self._initialized = True
884
- self.logger = logger
885
-
886
- def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
887
- config: CacheConfig = None, db_name: str = None, table_name: str = None):
888
- """初始化缓存系统(非阻塞)"""
889
- try:
890
- # 立即创建缓存实例,内部会异步初始化MySQL和统计线程
891
- self.cache_instance = SmartCacheSystem(
892
- redis_client=redis_client,
893
- mysql_pool=mysql_pool,
894
- instance_name=instance_name,
895
- config=config,
896
- db_name=db_name,
897
- table_name=table_name
898
- )
899
- self.initialization_error = None
900
- self.logger.info("缓存管理器初始化成功(异步模式)", {
901
- 'instance_name': instance_name,
902
- 'mysql_enabled': mysql_pool is not None,
903
- 'db_name': self.cache_instance.config.db_name,
904
- 'table_name': self.cache_instance.config.table_name,
905
- 'note': 'MySQL初始化将在后台异步完成'
906
- })
907
- return self # 支持链式调用
908
-
909
- except Exception as e:
910
- self.initialization_error = str(e)
911
- self.cache_instance = None
912
- self.logger.error(f"缓存管理器初始化失败: {e}")
913
- return self
817
+ self.cache_instance = SmartCacheSystem(
818
+ redis_client=redis_client,
819
+ mysql_pool=mysql_pool,
820
+ instance_name=instance_name,
821
+ **config
822
+ )
914
823
 
915
824
  def get_cache(self) -> Optional[SmartCacheSystem]:
916
825
  """获取缓存实例"""
917
- return self.cache_instance if self.enabled else None
826
+ return self.cache_instance
918
827
 
919
828
  def is_available(self) -> bool:
920
829
  """检查缓存是否可用"""
921
- return self.cache_instance is not None and self.enabled
922
-
923
- def enable(self):
924
- """启用缓存"""
925
- self.enabled = True
926
- self.logger.info("缓存系统已启用")
927
- return self # 支持链式调用
928
-
929
- def disable(self):
930
- """禁用缓存"""
931
- self.enabled = False
932
- self.logger.info("缓存系统已禁用")
933
- return self # 支持链式调用
830
+ return self.cache_instance is not None and self.cache_instance.is_ready
934
831
 
935
832
  def get_status(self) -> Dict[str, Any]:
936
833
  """获取缓存状态"""
834
+ if self.cache_instance is None:
835
+ return {
836
+ 'enabled': False,
837
+ 'available': False,
838
+ 'stats_enabled': False,
839
+ 'initialization_error': 'Cache not initialized'
840
+ }
841
+
937
842
  return {
938
- 'enabled': self.enabled,
939
- 'available': self.cache_instance is not None,
940
- 'initialization_error': self.initialization_error,
941
- 'instance_name': getattr(self.cache_instance, 'instance_name', None) if self.cache_instance else None
843
+ 'enabled': True,
844
+ 'available': self.cache_instance.is_ready,
845
+ 'mysql_ready': self.cache_instance.is_mysql_ready,
846
+ 'stats_enabled': self.cache_instance.config.enable_stats,
847
+ 'state': self.cache_instance._state.value,
848
+ 'instance_name': self.cache_instance.instance_name
942
849
  }
943
-
944
- def shutdown(self):
945
- """关闭缓存系统"""
946
- if self.cache_instance:
947
- self.cache_instance.shutdown()
948
- self.cache_instance = None
949
- self.logger.info("缓存管理器已关闭")
950
850
 
951
851
 
952
- # 导出单例实例
852
+ # 全局缓存管理器实例
953
853
  cache_manager = CacheManager()