mdbq 4.0.113__py3-none-any.whl → 4.0.115__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- mdbq/__version__.py +1 -1
- mdbq/redis/redis_cache.py +658 -779
- {mdbq-4.0.113.dist-info → mdbq-4.0.115.dist-info}/METADATA +1 -1
- {mdbq-4.0.113.dist-info → mdbq-4.0.115.dist-info}/RECORD +6 -6
- {mdbq-4.0.113.dist-info → mdbq-4.0.115.dist-info}/WHEEL +0 -0
- {mdbq-4.0.113.dist-info → mdbq-4.0.115.dist-info}/top_level.txt +0 -0
mdbq/redis/redis_cache.py
CHANGED
|
@@ -5,30 +5,31 @@ Redis智能缓存系统
|
|
|
5
5
|
主要功能:
|
|
6
6
|
1. Redis缓存的CRUD操作
|
|
7
7
|
2. 命名空间隔离
|
|
8
|
-
3.
|
|
9
|
-
4.
|
|
8
|
+
3. 智能TTL策略
|
|
9
|
+
4. 专业统计分析并提交到MySQL
|
|
10
10
|
5. 缓存健康检查和监控
|
|
11
|
-
|
|
11
|
+
6. 热点键分析
|
|
12
|
+
7. 响应时间分析
|
|
13
|
+
8. 业务指标计算
|
|
12
14
|
"""
|
|
13
15
|
|
|
14
16
|
import json
|
|
15
17
|
import time
|
|
16
18
|
import threading
|
|
17
19
|
import socket
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
import os
|
|
21
|
+
import statistics
|
|
22
|
+
import enum
|
|
23
|
+
import re
|
|
24
|
+
from datetime import datetime, timedelta
|
|
25
|
+
from typing import Optional, Dict, Any, List, Callable
|
|
26
|
+
from threading import Event
|
|
27
|
+
from collections import defaultdict, deque
|
|
22
28
|
import redis
|
|
23
29
|
from mdbq.log import mylogger
|
|
24
30
|
|
|
25
|
-
import enum
|
|
26
|
-
from typing import Optional, Dict, Any, List, Union, Callable
|
|
27
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
28
|
-
import asyncio
|
|
29
|
-
from threading import Event
|
|
30
|
-
import functools
|
|
31
31
|
|
|
32
|
+
# 全局日志器
|
|
32
33
|
logger = mylogger.MyLogger(
|
|
33
34
|
logging_mode='file',
|
|
34
35
|
log_level='info',
|
|
@@ -42,259 +43,319 @@ logger = mylogger.MyLogger(
|
|
|
42
43
|
)
|
|
43
44
|
|
|
44
45
|
|
|
45
|
-
class
|
|
46
|
-
"""
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
46
|
+
class CacheStatsCollector:
|
|
47
|
+
"""缓存统计收集器 - 单线程同步版本"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
|
|
50
|
+
self.enabled = enabled
|
|
51
|
+
self.mysql_pool = mysql_pool
|
|
52
|
+
self.config = config or {}
|
|
53
|
+
self.process_id = os.getpid()
|
|
54
|
+
self.instance_name = self.config.get('instance_name', 'default')
|
|
55
|
+
|
|
56
|
+
# 统计数据
|
|
57
|
+
self.stats = {
|
|
58
|
+
'hits': 0,
|
|
59
|
+
'misses': 0,
|
|
60
|
+
'sets': 0,
|
|
61
|
+
'deletes': 0,
|
|
62
|
+
'errors': 0,
|
|
63
|
+
'total_operations': 0,
|
|
64
|
+
'start_time': time.time()
|
|
65
|
+
}
|
|
66
|
+
self._lock = threading.RLock()
|
|
67
|
+
self.response_times = deque(maxlen=1000)
|
|
68
|
+
self.namespace_stats = defaultdict(int)
|
|
69
|
+
|
|
70
|
+
# 同步提交控制
|
|
71
|
+
self.submit_interval = self.config.get('submit_interval', 600) # 每隔N秒定时提交一次(期间有新操作时)
|
|
72
|
+
self.last_submit_time = time.time()
|
|
73
|
+
self.last_operation_count = 0 # 上次提交时的操作总数
|
|
52
74
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
75
|
+
def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
|
|
76
|
+
"""记录操作统计"""
|
|
77
|
+
if not self.enabled:
|
|
78
|
+
return
|
|
79
|
+
|
|
56
80
|
try:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
with self._lock:
|
|
82
|
+
old_total = self.stats['total_operations']
|
|
83
|
+
|
|
84
|
+
self.stats['total_operations'] += 1
|
|
85
|
+
self.stats[operation] = self.stats.get(operation, 0) + 1
|
|
86
|
+
|
|
87
|
+
if response_time > 0:
|
|
88
|
+
self.response_times.append(response_time)
|
|
89
|
+
|
|
90
|
+
if namespace:
|
|
91
|
+
self.namespace_stats[namespace] += 1
|
|
92
|
+
|
|
93
|
+
# 检查是否需要提交统计数据(容错处理)
|
|
94
|
+
try:
|
|
95
|
+
self._check_and_submit()
|
|
96
|
+
except Exception as submit_error:
|
|
97
|
+
# 统计提交失败不应影响统计记录
|
|
98
|
+
logger.error("统计数据提交检查失败,但统计记录继续", {
|
|
99
|
+
'instance_name': self.instance_name,
|
|
100
|
+
'process_id': self.process_id,
|
|
101
|
+
'operation': operation,
|
|
102
|
+
'submit_error': str(submit_error)
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
logger.debug("缓存操作统计已记录", {
|
|
106
|
+
'operation': operation,
|
|
107
|
+
'response_time_ms': round(response_time, 2),
|
|
108
|
+
'namespace': namespace,
|
|
109
|
+
'process_id': self.process_id,
|
|
110
|
+
'old_total_operations': old_total,
|
|
111
|
+
'new_total_operations': self.stats['total_operations']
|
|
112
|
+
})
|
|
85
113
|
except Exception as e:
|
|
86
|
-
#
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
"""智能TTL配置策略"""
|
|
94
|
-
|
|
95
|
-
# TTL策略映射表(秒)
|
|
96
|
-
TTL_STRATEGIES = {
|
|
97
|
-
# 数据库相关缓存
|
|
98
|
-
'sycm_database': 1800, # 数据库列表:30分钟
|
|
99
|
-
'sycm_tables': 1200, # 表列表:20分钟
|
|
100
|
-
'sycm_table_data': 300, # 表数据:5分钟
|
|
101
|
-
|
|
102
|
-
# 用户会话相关
|
|
103
|
-
'user_session': 3600, # 用户会话:1小时
|
|
104
|
-
'user_info': 1800, # 用户信息:30分钟
|
|
105
|
-
|
|
106
|
-
# 静态配置
|
|
107
|
-
'static_config': 86400, # 静态配置:24小时
|
|
108
|
-
'system_config': 43200, # 系统配置:12小时
|
|
109
|
-
|
|
110
|
-
# API相关
|
|
111
|
-
'api_response': 600, # API响应:10分钟
|
|
112
|
-
'api_cache': 900, # API缓存:15分钟
|
|
113
|
-
|
|
114
|
-
# 临时数据
|
|
115
|
-
'temp_data': 300, # 临时数据:5分钟
|
|
116
|
-
'short_cache': 60, # 短期缓存:1分钟
|
|
117
|
-
}
|
|
114
|
+
# 统计记录失败不应影响缓存操作
|
|
115
|
+
logger.error("统计记录失败,但缓存操作继续", {
|
|
116
|
+
'instance_name': self.instance_name,
|
|
117
|
+
'process_id': self.process_id,
|
|
118
|
+
'operation': operation,
|
|
119
|
+
'error': str(e)
|
|
120
|
+
})
|
|
118
121
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
122
|
+
def _check_and_submit(self):
|
|
123
|
+
"""检查并提交统计数据 """
|
|
124
|
+
if not self.mysql_pool:
|
|
125
|
+
return
|
|
126
|
+
|
|
127
|
+
current_time = time.time()
|
|
128
|
+
time_since_last_submit = current_time - self.last_submit_time
|
|
123
129
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
130
|
+
# 提交逻辑:每隔固定秒数且期间有新操作则提交
|
|
131
|
+
if time_since_last_submit >= self.submit_interval:
|
|
132
|
+
# 检查是否有新的操作(与上次提交时相比)
|
|
133
|
+
new_operations = self.stats['total_operations'] - self.last_operation_count
|
|
127
134
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
135
|
+
if new_operations > 0:
|
|
136
|
+
# 有新操作,提交统计数据
|
|
137
|
+
try:
|
|
138
|
+
self._submit_to_mysql()
|
|
139
|
+
self.last_submit_time = current_time
|
|
140
|
+
self.last_operation_count = self.stats['total_operations']
|
|
141
|
+
|
|
142
|
+
logger.info("统计数据定时提交", {
|
|
143
|
+
'instance_name': self.instance_name,
|
|
144
|
+
'process_id': self.process_id,
|
|
145
|
+
'total_operations': self.stats['total_operations'],
|
|
146
|
+
'new_operations': new_operations,
|
|
147
|
+
'time_since_last_submit': round(time_since_last_submit, 2),
|
|
148
|
+
'submit_interval': self.submit_interval
|
|
149
|
+
})
|
|
150
|
+
except Exception as e:
|
|
151
|
+
logger.error("统计数据提交失败", {
|
|
152
|
+
'instance_name': self.instance_name,
|
|
153
|
+
'process_id': self.process_id,
|
|
154
|
+
'error': str(e)
|
|
155
|
+
})
|
|
156
|
+
else:
|
|
157
|
+
# 无新操作,跳过提交但更新时间
|
|
158
|
+
self.last_submit_time = current_time
|
|
159
|
+
logger.debug("定时检查:期间无新操作,跳过提交", {
|
|
160
|
+
'instance_name': self.instance_name,
|
|
161
|
+
'process_id': self.process_id,
|
|
162
|
+
'total_operations': self.stats['total_operations'],
|
|
163
|
+
'time_since_last_submit': round(time_since_last_submit, 2)
|
|
164
|
+
})
|
|
132
165
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
166
|
+
def _submit_to_mysql(self):
|
|
167
|
+
"""同步提交统计数据到MySQL"""
|
|
168
|
+
if not self.mysql_pool:
|
|
169
|
+
return
|
|
170
|
+
|
|
171
|
+
stats_data = self.get_stats()
|
|
172
|
+
if not stats_data.get('enabled'):
|
|
173
|
+
return
|
|
174
|
+
|
|
175
|
+
try:
|
|
176
|
+
connection = self.mysql_pool.connection()
|
|
177
|
+
with connection.cursor() as cursor:
|
|
178
|
+
# 选择数据库
|
|
179
|
+
cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
|
|
180
|
+
|
|
181
|
+
# 插入统计数据
|
|
182
|
+
table_name = self.config.get('table_name', 'cache_performance')
|
|
183
|
+
insert_sql = f"""
|
|
184
|
+
INSERT INTO `{table_name}` (
|
|
185
|
+
`日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
|
|
186
|
+
`缓存命中`, `缓存未命中`, `缓存设置`, `缓存删除`, `缓存错误`, `总操作数`,
|
|
187
|
+
`命中率`, `平均响应时间`, `运行时间`, `命名空间统计`
|
|
188
|
+
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
189
|
+
"""
|
|
190
|
+
|
|
191
|
+
insert_data = (
|
|
192
|
+
datetime.now().strftime('%Y-%m-%d'),
|
|
193
|
+
f"{self.instance_name}_pid_{self.process_id}",
|
|
194
|
+
socket.gethostname(),
|
|
195
|
+
self.process_id,
|
|
196
|
+
datetime.now(),
|
|
197
|
+
stats_data['hits'],
|
|
198
|
+
stats_data['misses'],
|
|
199
|
+
stats_data['sets'],
|
|
200
|
+
stats_data['deletes'],
|
|
201
|
+
stats_data['errors'],
|
|
202
|
+
stats_data['total_operations'],
|
|
203
|
+
stats_data['hit_rate_percent'],
|
|
204
|
+
stats_data['avg_response_time_ms'],
|
|
205
|
+
stats_data['uptime_seconds'],
|
|
206
|
+
json.dumps(stats_data['namespace_stats'], ensure_ascii=False)
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
cursor.execute(insert_sql, insert_data)
|
|
210
|
+
connection.commit()
|
|
211
|
+
|
|
212
|
+
connection.close()
|
|
213
|
+
|
|
214
|
+
except Exception as e:
|
|
215
|
+
logger.error("MySQL同步提交失败", {
|
|
216
|
+
'instance_name': self.instance_name,
|
|
217
|
+
'process_id': self.process_id,
|
|
218
|
+
'error': str(e)
|
|
219
|
+
})
|
|
220
|
+
raise
|
|
221
|
+
|
|
222
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
223
|
+
"""获取统计数据"""
|
|
224
|
+
if not self.enabled:
|
|
225
|
+
return {'enabled': False, 'message': '统计功能已禁用'}
|
|
137
226
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
227
|
+
with self._lock:
|
|
228
|
+
uptime = time.time() - self.stats['start_time']
|
|
229
|
+
total_cache_ops = self.stats['hits'] + self.stats['misses']
|
|
230
|
+
hit_rate = (self.stats['hits'] / total_cache_ops * 100) if total_cache_ops > 0 else 0
|
|
231
|
+
avg_response_time = statistics.mean(self.response_times) if self.response_times else 0
|
|
232
|
+
|
|
233
|
+
return {
|
|
234
|
+
'enabled': True,
|
|
235
|
+
'hits': self.stats['hits'],
|
|
236
|
+
'misses': self.stats['misses'],
|
|
237
|
+
'sets': self.stats['sets'],
|
|
238
|
+
'deletes': self.stats['deletes'],
|
|
239
|
+
'errors': self.stats['errors'],
|
|
240
|
+
'total_operations': self.stats['total_operations'],
|
|
241
|
+
'hit_rate_percent': round(hit_rate, 2),
|
|
242
|
+
'avg_response_time_ms': round(avg_response_time, 2),
|
|
243
|
+
'uptime_seconds': round(uptime, 2),
|
|
244
|
+
'namespace_stats': dict(self.namespace_stats),
|
|
245
|
+
'last_updated': datetime.now().isoformat(),
|
|
246
|
+
'process_id': self.process_id
|
|
247
|
+
}
|
|
143
248
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
249
|
+
def reset_stats(self):
|
|
250
|
+
"""重置统计数据"""
|
|
251
|
+
if not self.enabled:
|
|
252
|
+
return
|
|
253
|
+
|
|
254
|
+
with self._lock:
|
|
255
|
+
self.stats = {
|
|
256
|
+
'hits': 0,
|
|
257
|
+
'misses': 0,
|
|
258
|
+
'sets': 0,
|
|
259
|
+
'deletes': 0,
|
|
260
|
+
'errors': 0,
|
|
261
|
+
'total_operations': 0,
|
|
262
|
+
'start_time': time.time()
|
|
263
|
+
}
|
|
264
|
+
self.response_times.clear()
|
|
265
|
+
self.namespace_stats.clear()
|
|
266
|
+
self.last_submit_time = time.time()
|
|
267
|
+
self.last_operation_count = 0
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
class CacheSystemState(enum.Enum):
|
|
271
|
+
"""缓存系统状态枚举"""
|
|
272
|
+
INITIALIZING = "initializing"
|
|
273
|
+
READY = "ready"
|
|
274
|
+
MYSQL_READY = "mysql_ready"
|
|
275
|
+
ERROR = "error"
|
|
276
|
+
SHUTDOWN = "shutdown"
|
|
148
277
|
|
|
149
278
|
|
|
150
279
|
class CacheConfig:
|
|
151
|
-
"""
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
#
|
|
155
|
-
self.
|
|
156
|
-
self.
|
|
157
|
-
self.
|
|
158
|
-
self.
|
|
159
|
-
self.
|
|
160
|
-
|
|
161
|
-
#
|
|
162
|
-
self.
|
|
163
|
-
self.
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
#
|
|
167
|
-
self.
|
|
168
|
-
self.
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
self.
|
|
173
|
-
self.
|
|
174
|
-
|
|
175
|
-
# 热点键配置
|
|
176
|
-
self.max_hot_keys = 1000 # 最大热点键数量
|
|
177
|
-
self.hot_keys_cleanup_threshold = 800 # 热点键清理阈值
|
|
178
|
-
|
|
179
|
-
# MySQL数据库配置
|
|
180
|
-
self.db_name = db_name
|
|
181
|
-
self.table_name = table_name
|
|
182
|
-
|
|
183
|
-
# 锁配置
|
|
184
|
-
self.lock_timeout = 30 # 分布式锁超时时间
|
|
185
|
-
self.lock_retry_delay = 0.1 # 锁重试延迟
|
|
280
|
+
"""缓存配置类"""
|
|
281
|
+
def __init__(self, **kwargs):
|
|
282
|
+
# 基础配置
|
|
283
|
+
self.default_ttl = kwargs.get('default_ttl', 3600) # 默认过期时间(秒)
|
|
284
|
+
self.stats_submit_interval = kwargs.get('stats_submit_interval', 600) # 每隔N秒定时提交一次(期间有新操作时)
|
|
285
|
+
self.enable_stats = kwargs.get('enable_stats', True) # 是否启用统计功能
|
|
286
|
+
self.max_value_size = kwargs.get('max_value_size', 10 * 1024 * 1024) # 最大值大小(字节)
|
|
287
|
+
self.cache_prefix = kwargs.get('cache_prefix', 'cache') # 缓存键前缀
|
|
288
|
+
self.enable_compression = kwargs.get('enable_compression', True)
|
|
289
|
+
|
|
290
|
+
# 数据库配置
|
|
291
|
+
self.db_name = kwargs.get('db_name', 'redis_stats')
|
|
292
|
+
self.table_name = kwargs.get('table_name', 'cache_performance')
|
|
293
|
+
|
|
294
|
+
# 智能TTL策略配置
|
|
295
|
+
self.enable_smart_ttl = kwargs.get('enable_smart_ttl', True) # 启用智能TTL
|
|
296
|
+
self.ttl_min = kwargs.get('ttl_min', 60) # 最小TTL(秒)
|
|
297
|
+
self.ttl_max = kwargs.get('ttl_max', 7200) # 最大TTL(秒)
|
|
298
|
+
self.debug_ttl = kwargs.get('debug_ttl', False) # TTL调试模式
|
|
299
|
+
|
|
300
|
+
# 自定义TTL模式
|
|
301
|
+
self.custom_namespace_patterns = kwargs.get('custom_namespace_patterns', {})
|
|
302
|
+
self.custom_key_patterns = kwargs.get('custom_key_patterns', {})
|
|
186
303
|
|
|
187
304
|
|
|
188
305
|
class SmartCacheSystem:
|
|
189
|
-
"""
|
|
306
|
+
"""智能缓存系统 - 单线程"""
|
|
190
307
|
|
|
191
|
-
def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
|
|
192
|
-
config: CacheConfig = None, db_name: str = None, table_name: str = None):
|
|
308
|
+
def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
|
|
193
309
|
self.redis_client = redis_client
|
|
194
310
|
self.mysql_pool = mysql_pool
|
|
195
311
|
self.instance_name = instance_name
|
|
312
|
+
self.config = CacheConfig(**config)
|
|
196
313
|
|
|
197
|
-
#
|
|
198
|
-
if config:
|
|
199
|
-
self.config = config
|
|
200
|
-
elif db_name or table_name:
|
|
201
|
-
self.config = CacheConfig(
|
|
202
|
-
db_name=db_name or "redis统计",
|
|
203
|
-
table_name=table_name or "dpflask路由分析"
|
|
204
|
-
)
|
|
205
|
-
else:
|
|
206
|
-
self.config = CacheConfig()
|
|
207
|
-
|
|
208
|
-
self.logger = logger
|
|
209
|
-
|
|
210
|
-
# 状态管理
|
|
314
|
+
# 系统状态管理
|
|
211
315
|
self._state = CacheSystemState.INITIALIZING
|
|
212
316
|
self._ready_event = Event()
|
|
213
|
-
self._mysql_ready_event = Event()
|
|
214
|
-
|
|
215
|
-
# 统计数据
|
|
216
|
-
self.stats = {
|
|
217
|
-
'hits': 0,
|
|
218
|
-
'misses': 0,
|
|
219
|
-
'sets': 0,
|
|
220
|
-
'deletes': 0,
|
|
221
|
-
'errors': 0,
|
|
222
|
-
'total_operations': 0,
|
|
223
|
-
'start_time': time.time(),
|
|
224
|
-
'response_times': []
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
# 热点键统计
|
|
228
|
-
self.hot_keys = {}
|
|
229
|
-
self.hot_keys_lock = threading.RLock()
|
|
230
317
|
|
|
231
|
-
#
|
|
232
|
-
self.
|
|
233
|
-
self.
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
318
|
+
# 直接初始化统计系统
|
|
319
|
+
self.stats_collector = None
|
|
320
|
+
if self.config.enable_stats:
|
|
321
|
+
self.stats_collector = CacheStatsCollector(
|
|
322
|
+
enabled=True,
|
|
323
|
+
mysql_pool=self.mysql_pool,
|
|
324
|
+
config={
|
|
325
|
+
'instance_name': self.instance_name,
|
|
326
|
+
'submit_interval': self.config.stats_submit_interval, # 每隔N秒定时提交一次(期间有新操作时)
|
|
327
|
+
'table_name': self.config.table_name,
|
|
328
|
+
'db_name': self.config.db_name
|
|
329
|
+
}
|
|
330
|
+
)
|
|
238
331
|
|
|
239
|
-
#
|
|
332
|
+
# 初始化系统
|
|
333
|
+
self._initialize()
|
|
334
|
+
|
|
335
|
+
def _initialize(self):
|
|
336
|
+
"""初始化缓存系统"""
|
|
337
|
+
# 测试Redis连接
|
|
240
338
|
if self._test_redis_connection():
|
|
241
339
|
self._state = CacheSystemState.READY
|
|
242
340
|
self._ready_event.set()
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
'instance_name': self.instance_name,
|
|
249
|
-
'state': self._state.value,
|
|
250
|
-
'redis_ready': self._ready_event.is_set(),
|
|
251
|
-
'mysql_enabled': self.mysql_pool is not None
|
|
252
|
-
})
|
|
253
|
-
|
|
254
|
-
def _initialize_mysql_features(self):
|
|
255
|
-
"""在后台初始化MySQL相关功能"""
|
|
256
|
-
try:
|
|
257
|
-
if self.mysql_pool:
|
|
258
|
-
# 延迟初始化MySQL(给系统一些启动时间)
|
|
259
|
-
time.sleep(5) # 等待5秒让主系统完全启动
|
|
260
|
-
|
|
261
|
-
if self._init_mysql_db():
|
|
341
|
+
|
|
342
|
+
# 如果启用统计且有MySQL连接,创建统计表
|
|
343
|
+
if self.config.enable_stats and self.mysql_pool:
|
|
344
|
+
try:
|
|
345
|
+
self._create_simple_stats_table()
|
|
262
346
|
self._state = CacheSystemState.MYSQL_READY
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
# MySQL就绪后再启动统计线程
|
|
266
|
-
self._start_stats_worker()
|
|
267
|
-
|
|
268
|
-
self.logger.info("MySQL功能初始化完成", {
|
|
347
|
+
logger.info("统计功能已启用", {
|
|
269
348
|
'instance_name': self.instance_name,
|
|
270
|
-
'
|
|
349
|
+
'process_id': os.getpid()
|
|
271
350
|
})
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
351
|
+
except Exception as e:
|
|
352
|
+
logger.error("统计表创建失败", {
|
|
353
|
+
'instance_name': self.instance_name,
|
|
354
|
+
'error': str(e)
|
|
355
|
+
})
|
|
356
|
+
else:
|
|
278
357
|
self._state = CacheSystemState.ERROR
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
@property
|
|
282
|
-
def is_ready(self) -> bool:
|
|
283
|
-
"""检查缓存系统是否就绪(Redis可用即为就绪)"""
|
|
284
|
-
return self._ready_event.is_set()
|
|
285
|
-
|
|
286
|
-
@property
|
|
287
|
-
def is_mysql_ready(self) -> bool:
|
|
288
|
-
"""检查MySQL功能是否就绪"""
|
|
289
|
-
return self._mysql_ready_event.is_set()
|
|
290
|
-
|
|
291
|
-
def wait_for_ready(self, timeout: float = 1.0) -> bool:
|
|
292
|
-
"""等待缓存系统就绪(非阻塞,有超时)"""
|
|
293
|
-
return self._ready_event.wait(timeout)
|
|
294
|
-
|
|
295
|
-
def wait_for_mysql_ready(self, timeout: float = 5.0) -> bool:
|
|
296
|
-
"""等待MySQL功能就绪(非阻塞,有超时)"""
|
|
297
|
-
return self._mysql_ready_event.wait(timeout)
|
|
358
|
+
logger.error("Redis连接失败", {'instance_name': self.instance_name})
|
|
298
359
|
|
|
299
360
|
def _test_redis_connection(self) -> bool:
|
|
300
361
|
"""测试Redis连接"""
|
|
@@ -302,652 +363,470 @@ class SmartCacheSystem:
|
|
|
302
363
|
self.redis_client.ping()
|
|
303
364
|
return True
|
|
304
365
|
except Exception as e:
|
|
305
|
-
|
|
366
|
+
logger.error("Redis连接测试失败", {'error': str(e)})
|
|
306
367
|
return False
|
|
307
368
|
|
|
308
|
-
def
|
|
309
|
-
"""
|
|
369
|
+
def _create_simple_stats_table(self):
|
|
370
|
+
"""创建统计表"""
|
|
310
371
|
if not self.mysql_pool:
|
|
311
|
-
|
|
312
|
-
return False
|
|
372
|
+
return
|
|
313
373
|
|
|
314
374
|
try:
|
|
315
|
-
# 设置连接超时,避免长时间阻塞
|
|
316
375
|
connection = self.mysql_pool.connection()
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
pass # 忽略不支持的操作
|
|
376
|
+
with connection.cursor() as cursor:
|
|
377
|
+
# 创建数据库
|
|
378
|
+
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci")
|
|
379
|
+
cursor.execute(f"USE `{self.config.db_name}`")
|
|
322
380
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
381
|
+
# 创建统计表
|
|
382
|
+
create_table_sql = f"""
|
|
383
|
+
CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
|
|
384
|
+
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
|
|
385
|
+
`日期` date NOT NULL COMMENT '统计日期',
|
|
386
|
+
`实例标识` varchar(64) NOT NULL COMMENT '缓存实例标识',
|
|
387
|
+
`主机名` varchar(100) NOT NULL COMMENT '服务器主机名',
|
|
388
|
+
`进程ID` int NOT NULL COMMENT '进程ID',
|
|
389
|
+
`统计时间` timestamp NOT NULL COMMENT '统计时间',
|
|
331
390
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
391
|
+
-- 基础操作统计
|
|
392
|
+
`缓存命中` bigint DEFAULT 0 COMMENT '缓存命中次数',
|
|
393
|
+
`缓存未命中` bigint DEFAULT 0 COMMENT '缓存未命中次数',
|
|
394
|
+
`缓存设置` bigint DEFAULT 0 COMMENT '缓存设置次数',
|
|
395
|
+
`缓存删除` bigint DEFAULT 0 COMMENT '缓存删除次数',
|
|
396
|
+
`缓存错误` bigint DEFAULT 0 COMMENT '缓存错误次数',
|
|
397
|
+
`总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
|
|
335
398
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
`日期` date NOT NULL COMMENT '统计日期',
|
|
341
|
-
`统计时间` datetime NOT NULL COMMENT '统计时间',
|
|
342
|
-
`时间段` varchar(20) NOT NULL COMMENT '时间段标识',
|
|
343
|
-
`缓存命中数` bigint DEFAULT 0 COMMENT '缓存命中次数',
|
|
344
|
-
`缓存未命中数` bigint DEFAULT 0 COMMENT '缓存未命中次数',
|
|
345
|
-
`缓存设置数` bigint DEFAULT 0 COMMENT '缓存设置次数',
|
|
346
|
-
`缓存删除数` bigint DEFAULT 0 COMMENT '缓存删除次数',
|
|
347
|
-
`缓存错误数` bigint DEFAULT 0 COMMENT '缓存错误次数',
|
|
348
|
-
`命中率` decimal(5,2) DEFAULT 0.00 COMMENT '缓存命中率(%)',
|
|
349
|
-
`总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
|
|
350
|
-
`平均响应时间` decimal(10,4) DEFAULT 0.0000 COMMENT '平均响应时间(ms)',
|
|
351
|
-
`每秒操作数` decimal(10,2) DEFAULT 0.00 COMMENT '每秒操作数',
|
|
352
|
-
`唯一键数量` int DEFAULT 0 COMMENT '唯一键数量',
|
|
353
|
-
`系统运行时间` bigint DEFAULT 0 COMMENT '系统运行时间(秒)',
|
|
354
|
-
`热点键统计` json DEFAULT NULL COMMENT '热点键统计信息',
|
|
355
|
-
`服务器主机` varchar(100) DEFAULT NULL COMMENT '服务器主机名',
|
|
356
|
-
`实例名称` varchar(100) DEFAULT NULL COMMENT '缓存实例名称',
|
|
357
|
-
`创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
358
|
-
`更新时间` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
359
|
-
PRIMARY KEY (`id`),
|
|
360
|
-
KEY `idx_stats_date` (`日期`),
|
|
361
|
-
KEY `idx_stats_time` (`统计时间`),
|
|
362
|
-
KEY `idx_time_period` (`时间段`),
|
|
363
|
-
KEY `idx_hit_rate` (`命中率`),
|
|
364
|
-
KEY `idx_instance` (`实例名称`),
|
|
365
|
-
KEY `idx_create_time` (`创建时间`)
|
|
366
|
-
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存系统统计分析表'
|
|
367
|
-
"""
|
|
399
|
+
-- 性能指标
|
|
400
|
+
`命中率` decimal(5,2) DEFAULT 0.00 COMMENT '命中率百分比',
|
|
401
|
+
`平均响应时间` decimal(10,2) DEFAULT 0.00 COMMENT '平均响应时间(毫秒)',
|
|
402
|
+
`运行时间` bigint DEFAULT 0 COMMENT '运行时间(秒)',
|
|
368
403
|
|
|
369
|
-
|
|
370
|
-
|
|
404
|
+
-- 命名空间统计
|
|
405
|
+
`命名空间统计` json COMMENT '命名空间统计详情',
|
|
371
406
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
'table': self.config.table_name
|
|
375
|
-
})
|
|
376
|
-
return True
|
|
407
|
+
-- 时间戳
|
|
408
|
+
`创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
377
409
|
|
|
378
|
-
|
|
379
|
-
|
|
410
|
+
PRIMARY KEY (`id`),
|
|
411
|
+
KEY `idx_日期` (`日期`),
|
|
412
|
+
KEY `idx_实例时间` (`实例标识`, `统计时间`),
|
|
413
|
+
KEY `idx_创建时间` (`创建时间`)
|
|
414
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存统计表'
|
|
415
|
+
"""
|
|
380
416
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
# 不抛出异常,让系统继续运行
|
|
384
|
-
return False
|
|
385
|
-
|
|
386
|
-
def _generate_cache_key(self, key: str, namespace: str = "") -> str:
|
|
387
|
-
"""生成缓存键"""
|
|
388
|
-
if namespace:
|
|
389
|
-
return f"{self.config.cache_prefix}{namespace}:{key}"
|
|
390
|
-
return f"{self.config.cache_prefix}{key}"
|
|
391
|
-
|
|
392
|
-
def _record_operation(self, operation: str, response_time: float = 0):
|
|
393
|
-
"""记录操作统计"""
|
|
394
|
-
with self._stats_lock:
|
|
395
|
-
self.stats['total_operations'] += 1
|
|
396
|
-
if operation in self.stats:
|
|
397
|
-
self.stats[operation] += 1
|
|
398
|
-
if response_time > 0:
|
|
399
|
-
self.stats['response_times'].append(response_time)
|
|
400
|
-
# 只保留最近1000次操作的响应时间
|
|
401
|
-
if len(self.stats['response_times']) > 1000:
|
|
402
|
-
self.stats['response_times'] = self.stats['response_times'][-1000:]
|
|
403
|
-
|
|
404
|
-
def _record_hot_key(self, key: str, namespace: str = ""):
|
|
405
|
-
"""记录热点键"""
|
|
406
|
-
cache_key = self._generate_cache_key(key, namespace)
|
|
407
|
-
with self.hot_keys_lock:
|
|
408
|
-
# 检查是否需要清理热点键(防止内存泄漏)
|
|
409
|
-
if len(self.hot_keys) >= self.config.hot_keys_cleanup_threshold:
|
|
410
|
-
# 保留访问次数最高的键,移除访问次数最少的键
|
|
411
|
-
sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
|
|
412
|
-
# 保留前600个热点键,为新键留出空间
|
|
413
|
-
self.hot_keys = dict(sorted_keys[:600])
|
|
417
|
+
cursor.execute(create_table_sql)
|
|
418
|
+
connection.commit()
|
|
414
419
|
|
|
415
|
-
|
|
416
|
-
'清理前数量': len(sorted_keys),
|
|
417
|
-
'清理后数量': len(self.hot_keys),
|
|
418
|
-
'清理阈值': self.config.hot_keys_cleanup_threshold
|
|
419
|
-
})
|
|
420
|
+
connection.close()
|
|
420
421
|
|
|
421
|
-
|
|
422
|
-
|
|
422
|
+
except Exception as e:
|
|
423
|
+
logger.error("统计表初始化失败", {'error': str(e)})
|
|
424
|
+
raise
|
|
423
425
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
self._record_operation('errors')
|
|
434
|
-
self.logger.warning(f"缓存系统未就绪,{operation_name}操作失败", {
|
|
435
|
-
'state': self._state.value
|
|
436
|
-
})
|
|
437
|
-
return None if operation_name == 'get' else False
|
|
438
|
-
|
|
439
|
-
try:
|
|
440
|
-
result = func(self, *args, **kwargs)
|
|
441
|
-
response_time = (time.time() - start_time) * 1000
|
|
442
|
-
self._record_operation(operation_name, response_time)
|
|
443
|
-
return result
|
|
444
|
-
|
|
445
|
-
except Exception as e:
|
|
446
|
-
self._record_operation('errors')
|
|
447
|
-
self.logger.error(f"缓存{operation_name}操作失败: {e}", {
|
|
448
|
-
'args': args[:2] if len(args) > 2 else args, # 避免记录过长的参数
|
|
449
|
-
'operation': operation_name
|
|
450
|
-
})
|
|
451
|
-
return None if operation_name == 'get' else False
|
|
452
|
-
|
|
453
|
-
return wrapper
|
|
454
|
-
return decorator
|
|
426
|
+
@property
|
|
427
|
+
def is_ready(self) -> bool:
|
|
428
|
+
"""检查系统是否就绪"""
|
|
429
|
+
return self._ready_event.is_set()
|
|
430
|
+
|
|
431
|
+
@property
|
|
432
|
+
def is_mysql_ready(self) -> bool:
|
|
433
|
+
"""检查MySQL是否就绪"""
|
|
434
|
+
return self._state == CacheSystemState.MYSQL_READY
|
|
455
435
|
|
|
456
|
-
@_cache_operation('get')
|
|
457
436
|
def get(self, key: str, namespace: str = "", default=None) -> Any:
|
|
458
437
|
"""获取缓存值"""
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
438
|
+
return self._get_with_stats(key, namespace, default)
|
|
439
|
+
|
|
440
|
+
def _get_with_stats(self, key: str, namespace: str = "", default=None) -> Any:
|
|
441
|
+
"""带统计的获取缓存值"""
|
|
442
|
+
if not self.is_ready:
|
|
443
|
+
return default
|
|
463
444
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
self.
|
|
445
|
+
start_time = time.time()
|
|
446
|
+
try:
|
|
447
|
+
cache_key = self._generate_cache_key(key, namespace)
|
|
448
|
+
value = self.redis_client.get(cache_key)
|
|
449
|
+
response_time = (time.time() - start_time) * 1000
|
|
467
450
|
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
451
|
+
if value is not None:
|
|
452
|
+
# 缓存命中
|
|
453
|
+
if self.stats_collector:
|
|
454
|
+
self.stats_collector.record_operation('hits', response_time, namespace)
|
|
455
|
+
try:
|
|
456
|
+
return json.loads(value.decode('utf-8'))
|
|
457
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
458
|
+
return value.decode('utf-8')
|
|
459
|
+
else:
|
|
460
|
+
# 缓存未命中
|
|
461
|
+
if self.stats_collector:
|
|
462
|
+
self.stats_collector.record_operation('misses', response_time, namespace)
|
|
463
|
+
return default
|
|
464
|
+
|
|
465
|
+
except Exception as e:
|
|
466
|
+
response_time = (time.time() - start_time) * 1000
|
|
467
|
+
if self.stats_collector:
|
|
468
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
469
|
+
logger.error("缓存获取失败", {
|
|
470
|
+
'key': key,
|
|
471
|
+
'namespace': namespace,
|
|
472
|
+
'error': str(e)
|
|
473
|
+
})
|
|
474
474
|
return default
|
|
475
475
|
|
|
476
|
-
|
|
477
|
-
def set(self, key: str, value: Any, ttl: int = None, namespace: str = "") -> bool:
|
|
476
|
+
def set(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
|
|
478
477
|
"""设置缓存值"""
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
# 序列化值
|
|
486
|
-
if isinstance(value, (dict, list, tuple)):
|
|
487
|
-
serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
|
|
488
|
-
else:
|
|
489
|
-
serialized_value = str(value)
|
|
490
|
-
|
|
491
|
-
# 检查值大小
|
|
492
|
-
if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
|
|
493
|
-
self.logger.warning(f"缓存值过大,跳过设置: {len(serialized_value)} bytes")
|
|
478
|
+
return self._set_with_stats(key, value, ttl, namespace)
|
|
479
|
+
|
|
480
|
+
def _set_with_stats(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
|
|
481
|
+
"""带统计的设置缓存值"""
|
|
482
|
+
if not self.is_ready:
|
|
494
483
|
return False
|
|
495
484
|
|
|
496
|
-
# 设置缓存
|
|
497
|
-
result = self.redis_client.setex(cache_key, ttl, serialized_value)
|
|
498
|
-
|
|
499
|
-
# 记录TTL策略使用情况
|
|
500
|
-
if namespace and ttl != self.config.default_ttl:
|
|
501
|
-
self.logger.debug("使用智能TTL策略", {
|
|
502
|
-
'namespace': namespace,
|
|
503
|
-
'ttl': ttl,
|
|
504
|
-
'key': key[:50] + "..." if len(key) > 50 else key
|
|
505
|
-
})
|
|
506
|
-
|
|
507
|
-
return bool(result)
|
|
508
|
-
|
|
509
|
-
def delete(self, key: str, namespace: str = "") -> bool:
|
|
510
|
-
"""删除缓存值"""
|
|
511
485
|
start_time = time.time()
|
|
512
|
-
|
|
513
486
|
try:
|
|
514
487
|
cache_key = self._generate_cache_key(key, namespace)
|
|
515
|
-
|
|
488
|
+
|
|
489
|
+
# 智能TTL策略
|
|
490
|
+
if ttl is None:
|
|
491
|
+
ttl = self._get_smart_ttl(namespace, key, len(json.dumps(value, ensure_ascii=False, default=str)))
|
|
492
|
+
|
|
493
|
+
# 序列化值
|
|
494
|
+
if isinstance(value, (dict, list)):
|
|
495
|
+
serialized_value = json.dumps(value, ensure_ascii=False, default=str)
|
|
496
|
+
else:
|
|
497
|
+
serialized_value = str(value)
|
|
498
|
+
|
|
499
|
+
# 检查值大小
|
|
500
|
+
value_size = len(serialized_value.encode('utf-8'))
|
|
501
|
+
if value_size > self.config.max_value_size:
|
|
502
|
+
if self.stats_collector:
|
|
503
|
+
self.stats_collector.record_operation('errors', 0, namespace)
|
|
504
|
+
logger.warning("缓存值过大,跳过设置", {
|
|
505
|
+
'key': key,
|
|
506
|
+
'size': len(serialized_value),
|
|
507
|
+
'max_size': self.config.max_value_size
|
|
508
|
+
})
|
|
509
|
+
return False
|
|
510
|
+
|
|
511
|
+
result = self.redis_client.setex(cache_key, ttl, serialized_value)
|
|
516
512
|
response_time = (time.time() - start_time) * 1000
|
|
517
513
|
|
|
518
|
-
self.
|
|
514
|
+
if self.stats_collector:
|
|
515
|
+
self.stats_collector.record_operation('sets', response_time, namespace)
|
|
519
516
|
return bool(result)
|
|
520
517
|
|
|
521
518
|
except Exception as e:
|
|
522
|
-
|
|
523
|
-
self.
|
|
519
|
+
response_time = (time.time() - start_time) * 1000
|
|
520
|
+
if self.stats_collector:
|
|
521
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
522
|
+
logger.error("缓存设置失败", {
|
|
524
523
|
'key': key,
|
|
525
|
-
'namespace': namespace
|
|
524
|
+
'namespace': namespace,
|
|
525
|
+
'error': str(e)
|
|
526
526
|
})
|
|
527
527
|
return False
|
|
528
528
|
|
|
529
|
-
def
|
|
530
|
-
"""
|
|
529
|
+
def delete(self, key: str, namespace: str = "") -> bool:
|
|
530
|
+
"""删除缓存值"""
|
|
531
|
+
return self._delete_with_stats(key, namespace)
|
|
532
|
+
|
|
533
|
+
def _delete_with_stats(self, key: str, namespace: str = "") -> bool:
|
|
534
|
+
"""带统计的删除缓存值"""
|
|
535
|
+
if not self.is_ready:
|
|
536
|
+
return False
|
|
537
|
+
|
|
538
|
+
start_time = time.time()
|
|
531
539
|
try:
|
|
532
540
|
cache_key = self._generate_cache_key(key, namespace)
|
|
533
|
-
|
|
541
|
+
result = self.redis_client.delete(cache_key)
|
|
542
|
+
response_time = (time.time() - start_time) * 1000
|
|
543
|
+
if self.stats_collector:
|
|
544
|
+
self.stats_collector.record_operation('deletes', response_time, namespace)
|
|
545
|
+
return bool(result)
|
|
546
|
+
|
|
534
547
|
except Exception as e:
|
|
535
|
-
|
|
548
|
+
response_time = (time.time() - start_time) * 1000
|
|
549
|
+
if self.stats_collector:
|
|
550
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
551
|
+
logger.error("缓存删除失败", {
|
|
552
|
+
'key': key,
|
|
553
|
+
'namespace': namespace,
|
|
554
|
+
'error': str(e)
|
|
555
|
+
})
|
|
536
556
|
return False
|
|
537
557
|
|
|
538
558
|
def clear_namespace(self, namespace: str) -> int:
|
|
539
559
|
"""清除指定命名空间的所有缓存"""
|
|
560
|
+
if not self.is_ready:
|
|
561
|
+
return 0
|
|
562
|
+
|
|
540
563
|
try:
|
|
541
|
-
pattern = f"{self.config.cache_prefix}{namespace}:*"
|
|
564
|
+
pattern = f"{self.config.cache_prefix}:{namespace}:*"
|
|
542
565
|
keys = self.redis_client.keys(pattern)
|
|
543
566
|
|
|
544
567
|
if keys:
|
|
545
568
|
deleted = self.redis_client.delete(*keys)
|
|
546
|
-
self.logger.info(f"清除命名空间缓存: {namespace}, 删除键数: {deleted}")
|
|
547
569
|
return deleted
|
|
548
570
|
return 0
|
|
549
571
|
|
|
550
572
|
except Exception as e:
|
|
551
|
-
|
|
573
|
+
logger.error("清除命名空间失败", {
|
|
574
|
+
'namespace': namespace,
|
|
575
|
+
'error': str(e)
|
|
576
|
+
})
|
|
552
577
|
return 0
|
|
553
578
|
|
|
554
|
-
def
|
|
555
|
-
"""
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
579
|
+
def _generate_cache_key(self, key: str, namespace: str = "") -> str:
|
|
580
|
+
"""生成缓存键"""
|
|
581
|
+
if namespace:
|
|
582
|
+
return f"{self.config.cache_prefix}:{namespace}:{key}"
|
|
583
|
+
return f"{self.config.cache_prefix}:{key}"
|
|
584
|
+
|
|
585
|
+
def _get_smart_ttl(self, namespace: str, key: str = "", data_size: int = 0) -> int:
|
|
586
|
+
"""智能TTL策略 - 通用版本
|
|
587
|
+
|
|
588
|
+
基于多种因素智能计算TTL:
|
|
589
|
+
1. 命名空间模式匹配
|
|
590
|
+
2. 数据类型推断
|
|
591
|
+
3. 数据大小考虑
|
|
592
|
+
4. 访问频率预测
|
|
593
|
+
"""
|
|
594
|
+
|
|
595
|
+
# 如果禁用智能TTL,直接返回默认值
|
|
596
|
+
if not self.config.enable_smart_ttl:
|
|
597
|
+
return self.config.default_ttl
|
|
598
|
+
|
|
599
|
+
# 1. 基于命名空间模式的智能匹配
|
|
600
|
+
namespace_patterns = {
|
|
601
|
+
# 数据库相关 - 变化频率低,TTL较长
|
|
602
|
+
r'.*database.*|.*db.*|.*schema.*': 1800, # 30分钟
|
|
559
603
|
|
|
560
|
-
#
|
|
561
|
-
|
|
604
|
+
# 表结构相关 - 变化频率低,TTL较长
|
|
605
|
+
r'.*table.*|.*column.*|.*field.*': 1200, # 20分钟
|
|
562
606
|
|
|
563
|
-
#
|
|
564
|
-
|
|
565
|
-
avg_response_time = sum(response_times) / len(response_times) if response_times else 0
|
|
607
|
+
# 数据查询相关 - 变化频率中等,TTL中等
|
|
608
|
+
r'.*data.*|.*query.*|.*result.*': 300, # 5分钟
|
|
566
609
|
|
|
567
|
-
#
|
|
568
|
-
|
|
610
|
+
# 用户会话相关 - 变化频率高,TTL较短
|
|
611
|
+
r'.*session.*|.*user.*|.*auth.*': 900, # 15分钟
|
|
569
612
|
|
|
570
|
-
#
|
|
571
|
-
|
|
613
|
+
# 配置相关 - 变化频率很低,TTL很长
|
|
614
|
+
r'.*config.*|.*setting.*|.*param.*': 3600, # 1小时
|
|
572
615
|
|
|
573
|
-
#
|
|
574
|
-
|
|
575
|
-
top_hot_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)[:10]
|
|
616
|
+
# 统计相关 - 变化频率中等,TTL中等
|
|
617
|
+
r'.*stats.*|.*metric.*|.*count.*': 600, # 10分钟
|
|
576
618
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
'errors': self.stats['errors'],
|
|
583
|
-
'total_operations': total_ops,
|
|
584
|
-
'hit_rate': round(hit_rate, 2),
|
|
585
|
-
'avg_response_time': round(avg_response_time, 4),
|
|
586
|
-
'ops_per_second': round(ops_per_second, 2),
|
|
587
|
-
'uptime_seconds': int(uptime),
|
|
588
|
-
'hot_keys': dict(top_hot_keys),
|
|
589
|
-
'unique_keys_count': len(self.hot_keys)
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
def health_check(self) -> Dict[str, Any]:
|
|
593
|
-
"""健康检查"""
|
|
594
|
-
health_info = {
|
|
595
|
-
'redis_connected': False,
|
|
596
|
-
'mysql_available': False,
|
|
597
|
-
'stats_worker_running': self._stats_running,
|
|
598
|
-
'instance_name': self.instance_name,
|
|
599
|
-
'timestamp': datetime.now().isoformat()
|
|
619
|
+
# 缓存相关 - 变化频率高,TTL较短
|
|
620
|
+
r'.*cache.*|.*temp.*|.*tmp.*': 180, # 3分钟
|
|
621
|
+
|
|
622
|
+
# 列表相关 - 变化频率中等,TTL中等
|
|
623
|
+
r'.*list.*|.*index.*|.*catalog.*': 450, # 7.5分钟
|
|
600
624
|
}
|
|
601
625
|
|
|
602
|
-
#
|
|
603
|
-
|
|
604
|
-
self.redis_client.ping()
|
|
605
|
-
health_info['redis_connected'] = True
|
|
606
|
-
except Exception as e:
|
|
607
|
-
health_info['redis_error'] = str(e)
|
|
608
|
-
|
|
609
|
-
# 检查MySQL连接
|
|
610
|
-
if self.mysql_pool:
|
|
611
|
-
try:
|
|
612
|
-
connection = self.mysql_pool.connection()
|
|
613
|
-
connection.close()
|
|
614
|
-
health_info['mysql_available'] = True
|
|
615
|
-
except Exception as e:
|
|
616
|
-
health_info['mysql_error'] = str(e)
|
|
617
|
-
|
|
618
|
-
return health_info
|
|
619
|
-
|
|
620
|
-
def _start_stats_worker(self):
|
|
621
|
-
"""启动统计工作线程"""
|
|
622
|
-
if not self._stats_running and self.mysql_pool:
|
|
623
|
-
self._stats_running = True
|
|
624
|
-
self._stats_thread = threading.Thread(
|
|
625
|
-
target=self._stats_worker,
|
|
626
|
-
daemon=True,
|
|
627
|
-
name=f"stats_worker_{self.instance_name}"
|
|
628
|
-
)
|
|
629
|
-
self._stats_thread.start()
|
|
630
|
-
self.logger.info("统计工作线程已启动", {
|
|
631
|
-
'instance_name': self.instance_name,
|
|
632
|
-
'delay_first_run': True
|
|
633
|
-
})
|
|
634
|
-
|
|
635
|
-
def _stats_worker(self):
|
|
636
|
-
"""后台统计工作线程"""
|
|
637
|
-
cleanup_counter = 0
|
|
626
|
+
# 合并用户自定义模式(优先级更高)
|
|
627
|
+
namespace_patterns.update(self.config.custom_namespace_patterns)
|
|
638
628
|
|
|
639
|
-
#
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
'
|
|
643
|
-
|
|
644
|
-
|
|
629
|
+
# 2. 基于键名模式的智能匹配
|
|
630
|
+
key_patterns = {
|
|
631
|
+
# ID相关 - 相对稳定
|
|
632
|
+
r'.*_id$|^id_.*': 1.2, # TTL倍数
|
|
633
|
+
|
|
634
|
+
# 详情相关 - 相对稳定
|
|
635
|
+
r'.*detail.*|.*info.*': 1.1,
|
|
636
|
+
|
|
637
|
+
# 列表相关 - 变化较频繁
|
|
638
|
+
r'.*list.*|.*items.*': 0.8,
|
|
639
|
+
|
|
640
|
+
# 计数相关 - 变化频繁
|
|
641
|
+
r'.*count.*|.*num.*|.*total.*': 0.6,
|
|
642
|
+
|
|
643
|
+
# 状态相关 - 变化很频繁
|
|
644
|
+
r'.*status.*|.*state.*': 0.5,
|
|
645
|
+
}
|
|
645
646
|
|
|
646
|
-
#
|
|
647
|
-
|
|
648
|
-
|
|
647
|
+
# 合并用户自定义键模式
|
|
648
|
+
key_patterns.update(self.config.custom_key_patterns)
|
|
649
|
+
|
|
650
|
+
# 3. 基于数据大小的调整
|
|
651
|
+
size_factor = 1.0
|
|
652
|
+
if data_size > 0:
|
|
653
|
+
if data_size > 1024 * 1024: # > 1MB,延长TTL减少重建开销
|
|
654
|
+
size_factor = 1.5
|
|
655
|
+
elif data_size > 100 * 1024: # > 100KB
|
|
656
|
+
size_factor = 1.2
|
|
657
|
+
elif data_size < 1024: # < 1KB,缩短TTL减少内存占用
|
|
658
|
+
size_factor = 0.8
|
|
659
|
+
|
|
660
|
+
# 4. 计算基础TTL
|
|
661
|
+
base_ttl = self.config.default_ttl
|
|
662
|
+
|
|
663
|
+
# 匹配命名空间模式
|
|
664
|
+
for pattern, ttl in namespace_patterns.items():
|
|
665
|
+
if re.match(pattern, namespace.lower()):
|
|
666
|
+
base_ttl = ttl
|
|
667
|
+
break
|
|
649
668
|
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
# 提交到MySQL(带状态检查)
|
|
656
|
-
self._submit_stats_to_mysql(stats_data)
|
|
657
|
-
|
|
658
|
-
# 清理过期的热点键统计
|
|
659
|
-
self._cleanup_hot_keys()
|
|
660
|
-
|
|
661
|
-
# 定期清理过期数据
|
|
662
|
-
cleanup_counter += 1
|
|
663
|
-
if cleanup_counter >= 24: # 24 * 1800秒 = 12小时
|
|
664
|
-
self._cleanup_expired_mysql_data()
|
|
665
|
-
cleanup_counter = 0
|
|
666
|
-
|
|
667
|
-
except Exception as e:
|
|
668
|
-
self.logger.error(f"统计工作线程异常: {e}")
|
|
669
|
-
|
|
670
|
-
# 可中断的等待下一个统计间隔
|
|
671
|
-
if not self._interruptible_sleep(self.config.stats_interval):
|
|
669
|
+
# 5. 应用键名模式调整
|
|
670
|
+
key_factor = 1.0
|
|
671
|
+
for pattern, factor in key_patterns.items():
|
|
672
|
+
if re.match(pattern, key.lower()):
|
|
673
|
+
key_factor = factor
|
|
672
674
|
break
|
|
675
|
+
|
|
676
|
+
# 6. 计算最终TTL
|
|
677
|
+
final_ttl = int(base_ttl * key_factor * size_factor)
|
|
678
|
+
|
|
679
|
+
# 7. TTL边界限制(使用配置值)
|
|
680
|
+
final_ttl = max(self.config.ttl_min, min(self.config.ttl_max, final_ttl))
|
|
681
|
+
|
|
682
|
+
# 8. 记录智能TTL决策(使用配置开关)
|
|
683
|
+
if self.config.debug_ttl:
|
|
684
|
+
logger.debug("智能TTL计算", {
|
|
685
|
+
'namespace': namespace,
|
|
686
|
+
'key': key[:50] + "..." if len(key) > 50 else key,
|
|
687
|
+
'data_size': data_size,
|
|
688
|
+
'base_ttl': base_ttl,
|
|
689
|
+
'key_factor': key_factor,
|
|
690
|
+
'size_factor': size_factor,
|
|
691
|
+
'final_ttl': final_ttl
|
|
692
|
+
})
|
|
693
|
+
|
|
694
|
+
return final_ttl
|
|
673
695
|
|
|
674
|
-
def
|
|
675
|
-
"""
|
|
676
|
-
|
|
677
|
-
|
|
696
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
697
|
+
"""获取统计信息"""
|
|
698
|
+
# 确保统计系统已初始化
|
|
699
|
+
if self.stats_collector:
|
|
700
|
+
return self.stats_collector.get_stats()
|
|
678
701
|
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
return
|
|
702
|
+
logger.debug("统计系统未初始化,返回空统计信息", {
|
|
703
|
+
'instance_name': self.instance_name,
|
|
704
|
+
'process_id': os.getpid()
|
|
705
|
+
})
|
|
706
|
+
return {
|
|
707
|
+
'enabled': False,
|
|
708
|
+
'message': '统计系统未初始化',
|
|
709
|
+
'process_id': os.getpid()
|
|
710
|
+
}
|
|
684
711
|
|
|
685
|
-
def
|
|
686
|
-
"""
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
712
|
+
def health_check(self) -> Dict[str, Any]:
|
|
713
|
+
"""健康检查"""
|
|
714
|
+
try:
|
|
715
|
+
# Redis连接检查
|
|
716
|
+
redis_latency = time.time()
|
|
717
|
+
self.redis_client.ping()
|
|
718
|
+
redis_latency = (time.time() - redis_latency) * 1000
|
|
690
719
|
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
'
|
|
696
|
-
|
|
697
|
-
|
|
720
|
+
# 获取Redis信息
|
|
721
|
+
redis_info = self.redis_client.info()
|
|
722
|
+
|
|
723
|
+
return {
|
|
724
|
+
'status': 'healthy',
|
|
725
|
+
'redis_connected': True,
|
|
726
|
+
'redis_latency_ms': round(redis_latency, 2),
|
|
727
|
+
'mysql_enabled': self.mysql_pool is not None,
|
|
728
|
+
'mysql_ready': self.is_mysql_ready,
|
|
729
|
+
'system_state': self._state.value,
|
|
730
|
+
'uptime_seconds': time.time() - (self.stats_collector.stats['start_time'] if self.stats_collector else time.time()),
|
|
731
|
+
'redis_memory_used': redis_info.get('used_memory', 0),
|
|
732
|
+
'redis_connected_clients': redis_info.get('connected_clients', 0),
|
|
733
|
+
'process_id': os.getpid()
|
|
734
|
+
}
|
|
698
735
|
|
|
699
|
-
logger.info("统计数据", {'stats_data': stats_data})
|
|
700
|
-
|
|
701
|
-
try:
|
|
702
|
-
# 使用上下文管理器确保连接正确关闭
|
|
703
|
-
connection = self.mysql_pool.connection()
|
|
704
|
-
try:
|
|
705
|
-
with connection.cursor() as cursor:
|
|
706
|
-
cursor.execute(f"USE `{self.config.db_name}`")
|
|
707
|
-
|
|
708
|
-
# 准备数据
|
|
709
|
-
now = datetime.now()
|
|
710
|
-
date_str = now.strftime("%Y-%m-%d")
|
|
711
|
-
time_period = now.strftime("%Y%m%d_%H%M")
|
|
712
|
-
|
|
713
|
-
insert_sql = f"""
|
|
714
|
-
INSERT INTO `{self.config.table_name}` (
|
|
715
|
-
`日期`, `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
|
|
716
|
-
`缓存删除数`, `缓存错误数`, `命中率`, `总操作数`, `平均响应时间`,
|
|
717
|
-
`每秒操作数`, `唯一键数量`, `系统运行时间`, `热点键统计`,
|
|
718
|
-
`服务器主机`, `实例名称`
|
|
719
|
-
) VALUES (
|
|
720
|
-
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
721
|
-
)
|
|
722
|
-
"""
|
|
723
|
-
|
|
724
|
-
cursor.execute(insert_sql, (
|
|
725
|
-
date_str,
|
|
726
|
-
now,
|
|
727
|
-
time_period,
|
|
728
|
-
stats_data['hits'],
|
|
729
|
-
stats_data['misses'],
|
|
730
|
-
stats_data['sets'],
|
|
731
|
-
stats_data['deletes'],
|
|
732
|
-
stats_data['errors'],
|
|
733
|
-
stats_data['hit_rate'],
|
|
734
|
-
stats_data['total_operations'],
|
|
735
|
-
stats_data['avg_response_time'],
|
|
736
|
-
stats_data['ops_per_second'],
|
|
737
|
-
stats_data['unique_keys_count'],
|
|
738
|
-
stats_data['uptime_seconds'],
|
|
739
|
-
json.dumps(stats_data['hot_keys'], ensure_ascii=False, cls=MySQLDataEncoder),
|
|
740
|
-
socket.gethostname(),
|
|
741
|
-
self.instance_name
|
|
742
|
-
))
|
|
743
|
-
|
|
744
|
-
connection.commit()
|
|
745
|
-
|
|
746
|
-
self.logger.info("统计数据已提交到MySQL", {
|
|
747
|
-
'time_period': time_period,
|
|
748
|
-
'total_operations': stats_data['total_operations'],
|
|
749
|
-
'hit_rate': stats_data['hit_rate'],
|
|
750
|
-
'instance_name': self.instance_name
|
|
751
|
-
})
|
|
752
|
-
|
|
753
|
-
finally:
|
|
754
|
-
connection.close()
|
|
755
|
-
|
|
756
736
|
except Exception as e:
|
|
757
|
-
|
|
758
|
-
'
|
|
759
|
-
'
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
"""清理热点键统计 """
|
|
764
|
-
with self.hot_keys_lock:
|
|
765
|
-
current_count = len(self.hot_keys)
|
|
766
|
-
|
|
767
|
-
# 如果热点键数量超过最大限制,进行清理
|
|
768
|
-
if current_count > self.config.max_hot_keys:
|
|
769
|
-
# 保留访问次数最高的键
|
|
770
|
-
sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
|
|
771
|
-
# 保留80%的热点键
|
|
772
|
-
keep_count = int(self.config.max_hot_keys * 0.8)
|
|
773
|
-
self.hot_keys = dict(sorted_keys[:keep_count])
|
|
774
|
-
|
|
775
|
-
self.logger.info("定期热点键清理", {
|
|
776
|
-
'清理前数量': current_count,
|
|
777
|
-
'清理后数量': len(self.hot_keys),
|
|
778
|
-
'保留比例': '80%'
|
|
779
|
-
})
|
|
737
|
+
return {
|
|
738
|
+
'status': 'unhealthy',
|
|
739
|
+
'error': str(e),
|
|
740
|
+
'redis_connected': False,
|
|
741
|
+
'system_state': self._state.value
|
|
742
|
+
}
|
|
780
743
|
|
|
781
|
-
def
|
|
782
|
-
"""
|
|
783
|
-
|
|
744
|
+
def _record_operation(self, operation: str, response_time: float = 0, namespace: str = "", key: str = ""):
|
|
745
|
+
"""记录操作统计 """
|
|
746
|
+
# 如果禁用统计功能,直接返回
|
|
747
|
+
if not self.config.enable_stats:
|
|
784
748
|
return
|
|
749
|
+
|
|
750
|
+
logger.debug("调用操作记录", {
|
|
751
|
+
'operation': operation,
|
|
752
|
+
'response_time_ms': round(response_time, 2),
|
|
753
|
+
'namespace': namespace,
|
|
754
|
+
'key': key[:50] + "..." if len(key) > 50 else key,
|
|
755
|
+
'instance_name': self.instance_name
|
|
756
|
+
})
|
|
785
757
|
|
|
786
|
-
try:
|
|
787
|
-
connection = self.mysql_pool.connection()
|
|
788
|
-
try:
|
|
789
|
-
with connection.cursor() as cursor:
|
|
790
|
-
cursor.execute(f"USE `{self.config.db_name}`")
|
|
791
|
-
|
|
792
|
-
# 计算过期时间点
|
|
793
|
-
from datetime import timedelta
|
|
794
|
-
expire_date = datetime.now() - timedelta(days=self.config.stats_retention)
|
|
795
|
-
|
|
796
|
-
# 删除过期数据
|
|
797
|
-
delete_sql = f"""
|
|
798
|
-
DELETE FROM `{self.config.table_name}`
|
|
799
|
-
WHERE `统计时间` < %s
|
|
800
|
-
"""
|
|
801
|
-
|
|
802
|
-
cursor.execute(delete_sql, (expire_date,))
|
|
803
|
-
deleted_rows = cursor.rowcount
|
|
804
|
-
connection.commit()
|
|
805
|
-
|
|
806
|
-
if deleted_rows > 0:
|
|
807
|
-
self.logger.info("清理过期MySQL统计数据", {
|
|
808
|
-
'deleted_rows': deleted_rows,
|
|
809
|
-
'expire_date': expire_date.strftime('%Y-%m-%d %H:%M:%S'),
|
|
810
|
-
'retention_days': self.config.stats_retention
|
|
811
|
-
})
|
|
812
|
-
|
|
813
|
-
finally:
|
|
814
|
-
connection.close()
|
|
815
|
-
|
|
816
|
-
except Exception as e:
|
|
817
|
-
self.logger.error(f"清理过期MySQL统计数据失败: {e}")
|
|
818
|
-
|
|
819
758
|
def shutdown(self):
|
|
820
759
|
"""关闭缓存系统"""
|
|
821
|
-
self.
|
|
822
|
-
'instance_name': self.instance_name,
|
|
823
|
-
'state': self._state.value
|
|
824
|
-
})
|
|
760
|
+
self._state = CacheSystemState.SHUTDOWN
|
|
825
761
|
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
if self._stats_thread.is_alive():
|
|
833
|
-
self.logger.warning("统计线程未能在超时时间内结束")
|
|
834
|
-
|
|
835
|
-
# 关闭线程池
|
|
836
|
-
if hasattr(self, '_executor'):
|
|
837
|
-
self.logger.info("关闭线程池...")
|
|
838
|
-
self._executor.shutdown(wait=True, timeout=5)
|
|
839
|
-
|
|
840
|
-
# 最后一次提交统计数据(如果MySQL可用)
|
|
841
|
-
if self.is_mysql_ready:
|
|
842
|
-
try:
|
|
843
|
-
self.logger.info("提交最终统计数据...")
|
|
844
|
-
stats_data = self.get_stats()
|
|
845
|
-
self._submit_stats_to_mysql(stats_data)
|
|
846
|
-
except Exception as e:
|
|
847
|
-
self.logger.error(f"关闭时提交统计数据失败: {e}")
|
|
848
|
-
|
|
849
|
-
self.logger.info("缓存系统已关闭", {
|
|
850
|
-
'instance_name': self.instance_name
|
|
851
|
-
})
|
|
852
|
-
|
|
853
|
-
def __enter__(self):
|
|
854
|
-
"""上下文管理器支持"""
|
|
855
|
-
return self
|
|
856
|
-
|
|
857
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
858
|
-
"""上下文管理器退出时自动关闭"""
|
|
859
|
-
self.shutdown()
|
|
860
|
-
return False
|
|
762
|
+
if self.stats_collector:
|
|
763
|
+
# The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
|
|
764
|
+
self.stats_collector.reset_stats()
|
|
765
|
+
|
|
766
|
+
logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
|
|
861
767
|
|
|
862
768
|
|
|
863
769
|
class CacheManager:
|
|
864
|
-
"""缓存管理器
|
|
770
|
+
"""缓存管理器"""
|
|
865
771
|
|
|
866
772
|
_instance = None
|
|
867
|
-
_lock = threading.
|
|
773
|
+
_lock = threading.Lock()
|
|
868
774
|
|
|
869
775
|
def __new__(cls):
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
cls._instance
|
|
873
|
-
|
|
874
|
-
|
|
776
|
+
if cls._instance is None:
|
|
777
|
+
with cls._lock:
|
|
778
|
+
if cls._instance is None:
|
|
779
|
+
cls._instance = super().__new__(cls)
|
|
780
|
+
return cls._instance
|
|
875
781
|
|
|
876
782
|
def __init__(self):
|
|
877
|
-
if self
|
|
783
|
+
if not hasattr(self, '_initialized'):
|
|
784
|
+
self.cache_instance = None
|
|
785
|
+
self._initialized = True
|
|
786
|
+
|
|
787
|
+
def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
|
|
788
|
+
"""初始化缓存系统"""
|
|
789
|
+
if self.cache_instance is not None:
|
|
790
|
+
logger.warning("缓存系统已初始化,跳过重复初始化", {
|
|
791
|
+
'existing_instance': self.cache_instance.instance_name,
|
|
792
|
+
'new_instance': instance_name
|
|
793
|
+
})
|
|
878
794
|
return
|
|
879
795
|
|
|
880
|
-
self.cache_instance =
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
|
|
887
|
-
config: CacheConfig = None, db_name: str = None, table_name: str = None):
|
|
888
|
-
"""初始化缓存系统(非阻塞)"""
|
|
889
|
-
try:
|
|
890
|
-
# 立即创建缓存实例,内部会异步初始化MySQL和统计线程
|
|
891
|
-
self.cache_instance = SmartCacheSystem(
|
|
892
|
-
redis_client=redis_client,
|
|
893
|
-
mysql_pool=mysql_pool,
|
|
894
|
-
instance_name=instance_name,
|
|
895
|
-
config=config,
|
|
896
|
-
db_name=db_name,
|
|
897
|
-
table_name=table_name
|
|
898
|
-
)
|
|
899
|
-
self.initialization_error = None
|
|
900
|
-
self.logger.info("缓存管理器初始化成功(异步模式)", {
|
|
901
|
-
'instance_name': instance_name,
|
|
902
|
-
'mysql_enabled': mysql_pool is not None,
|
|
903
|
-
'db_name': self.cache_instance.config.db_name,
|
|
904
|
-
'table_name': self.cache_instance.config.table_name,
|
|
905
|
-
'note': 'MySQL初始化将在后台异步完成'
|
|
906
|
-
})
|
|
907
|
-
return self # 支持链式调用
|
|
908
|
-
|
|
909
|
-
except Exception as e:
|
|
910
|
-
self.initialization_error = str(e)
|
|
911
|
-
self.cache_instance = None
|
|
912
|
-
self.logger.error(f"缓存管理器初始化失败: {e}")
|
|
913
|
-
return self
|
|
796
|
+
self.cache_instance = SmartCacheSystem(
|
|
797
|
+
redis_client=redis_client,
|
|
798
|
+
mysql_pool=mysql_pool,
|
|
799
|
+
instance_name=instance_name,
|
|
800
|
+
**config
|
|
801
|
+
)
|
|
914
802
|
|
|
915
803
|
def get_cache(self) -> Optional[SmartCacheSystem]:
|
|
916
804
|
"""获取缓存实例"""
|
|
917
|
-
return self.cache_instance
|
|
805
|
+
return self.cache_instance
|
|
918
806
|
|
|
919
807
|
def is_available(self) -> bool:
|
|
920
808
|
"""检查缓存是否可用"""
|
|
921
|
-
return self.cache_instance is not None and self.
|
|
922
|
-
|
|
923
|
-
def enable(self):
|
|
924
|
-
"""启用缓存"""
|
|
925
|
-
self.enabled = True
|
|
926
|
-
self.logger.info("缓存系统已启用")
|
|
927
|
-
return self # 支持链式调用
|
|
928
|
-
|
|
929
|
-
def disable(self):
|
|
930
|
-
"""禁用缓存"""
|
|
931
|
-
self.enabled = False
|
|
932
|
-
self.logger.info("缓存系统已禁用")
|
|
933
|
-
return self # 支持链式调用
|
|
809
|
+
return self.cache_instance is not None and self.cache_instance.is_ready
|
|
934
810
|
|
|
935
811
|
def get_status(self) -> Dict[str, Any]:
|
|
936
812
|
"""获取缓存状态"""
|
|
813
|
+
if self.cache_instance is None:
|
|
814
|
+
return {
|
|
815
|
+
'enabled': False,
|
|
816
|
+
'available': False,
|
|
817
|
+
'stats_enabled': False,
|
|
818
|
+
'initialization_error': 'Cache not initialized'
|
|
819
|
+
}
|
|
820
|
+
|
|
937
821
|
return {
|
|
938
|
-
'enabled':
|
|
939
|
-
'available': self.cache_instance
|
|
940
|
-
'
|
|
941
|
-
'
|
|
822
|
+
'enabled': True,
|
|
823
|
+
'available': self.cache_instance.is_ready,
|
|
824
|
+
'mysql_ready': self.cache_instance.is_mysql_ready,
|
|
825
|
+
'stats_enabled': self.cache_instance.config.enable_stats,
|
|
826
|
+
'state': self.cache_instance._state.value,
|
|
827
|
+
'instance_name': self.cache_instance.instance_name
|
|
942
828
|
}
|
|
943
|
-
|
|
944
|
-
def shutdown(self):
|
|
945
|
-
"""关闭缓存系统"""
|
|
946
|
-
if self.cache_instance:
|
|
947
|
-
self.cache_instance.shutdown()
|
|
948
|
-
self.cache_instance = None
|
|
949
|
-
self.logger.info("缓存管理器已关闭")
|
|
950
829
|
|
|
951
830
|
|
|
952
|
-
#
|
|
831
|
+
# 全局缓存管理器实例
|
|
953
832
|
cache_manager = CacheManager()
|