mdbq 4.0.112__py3-none-any.whl → 4.0.114__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- mdbq/__version__.py +1 -1
- mdbq/auth/auth_backend.py +14 -5
- mdbq/redis/redis_cache.py +676 -776
- {mdbq-4.0.112.dist-info → mdbq-4.0.114.dist-info}/METADATA +1 -1
- {mdbq-4.0.112.dist-info → mdbq-4.0.114.dist-info}/RECORD +7 -7
- {mdbq-4.0.112.dist-info → mdbq-4.0.114.dist-info}/WHEEL +0 -0
- {mdbq-4.0.112.dist-info → mdbq-4.0.114.dist-info}/top_level.txt +0 -0
mdbq/redis/redis_cache.py
CHANGED
|
@@ -5,30 +5,31 @@ Redis智能缓存系统
|
|
|
5
5
|
主要功能:
|
|
6
6
|
1. Redis缓存的CRUD操作
|
|
7
7
|
2. 命名空间隔离
|
|
8
|
-
3.
|
|
9
|
-
4.
|
|
8
|
+
3. 智能TTL策略
|
|
9
|
+
4. 专业统计分析并提交到MySQL
|
|
10
10
|
5. 缓存健康检查和监控
|
|
11
|
-
|
|
11
|
+
6. 热点键分析
|
|
12
|
+
7. 响应时间分析
|
|
13
|
+
8. 业务指标计算
|
|
12
14
|
"""
|
|
13
15
|
|
|
14
16
|
import json
|
|
15
17
|
import time
|
|
16
18
|
import threading
|
|
17
19
|
import socket
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
import os
|
|
21
|
+
import statistics
|
|
22
|
+
import enum
|
|
23
|
+
import re
|
|
24
|
+
from datetime import datetime, timedelta
|
|
25
|
+
from typing import Optional, Dict, Any, List, Callable
|
|
26
|
+
from threading import Event
|
|
27
|
+
from collections import defaultdict, deque
|
|
22
28
|
import redis
|
|
23
29
|
from mdbq.log import mylogger
|
|
24
30
|
|
|
25
|
-
import enum
|
|
26
|
-
from typing import Optional, Dict, Any, List, Union, Callable
|
|
27
|
-
from concurrent.futures import ThreadPoolExecutor
|
|
28
|
-
import asyncio
|
|
29
|
-
from threading import Event
|
|
30
|
-
import functools
|
|
31
31
|
|
|
32
|
+
# 全局日志器
|
|
32
33
|
logger = mylogger.MyLogger(
|
|
33
34
|
logging_mode='file',
|
|
34
35
|
log_level='info',
|
|
@@ -42,259 +43,340 @@ logger = mylogger.MyLogger(
|
|
|
42
43
|
)
|
|
43
44
|
|
|
44
45
|
|
|
45
|
-
class
|
|
46
|
-
"""
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
46
|
+
class CacheStatsCollector:
|
|
47
|
+
"""缓存统计收集器 - 单线程同步版本"""
|
|
48
|
+
|
|
49
|
+
def __init__(self, enabled: bool = True, mysql_pool=None, config: dict = None):
|
|
50
|
+
self.enabled = enabled
|
|
51
|
+
self.mysql_pool = mysql_pool
|
|
52
|
+
self.config = config or {}
|
|
53
|
+
self.process_id = os.getpid()
|
|
54
|
+
self.instance_name = self.config.get('instance_name', 'default')
|
|
55
|
+
|
|
56
|
+
# 统计数据
|
|
57
|
+
self.stats = {
|
|
58
|
+
'hits': 0,
|
|
59
|
+
'misses': 0,
|
|
60
|
+
'sets': 0,
|
|
61
|
+
'deletes': 0,
|
|
62
|
+
'errors': 0,
|
|
63
|
+
'total_operations': 0,
|
|
64
|
+
'start_time': time.time()
|
|
65
|
+
}
|
|
66
|
+
self._lock = threading.RLock()
|
|
67
|
+
self.response_times = deque(maxlen=1000)
|
|
68
|
+
self.namespace_stats = defaultdict(int)
|
|
69
|
+
|
|
70
|
+
# 同步提交控制
|
|
71
|
+
self.submit_interval = self.config.get('submit_interval', 300) # 每N次操作提交一次统计数据
|
|
72
|
+
self.last_submit_time = time.time()
|
|
73
|
+
self.min_submit_interval = self.config.get('min_submit_interval', 300) # 最少N秒提交一次
|
|
52
74
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
75
|
+
def record_operation(self, operation: str, response_time: float = 0, namespace: str = ""):
|
|
76
|
+
"""记录操作统计"""
|
|
77
|
+
if not self.enabled:
|
|
78
|
+
return
|
|
79
|
+
|
|
56
80
|
try:
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
81
|
+
with self._lock:
|
|
82
|
+
old_total = self.stats['total_operations']
|
|
83
|
+
|
|
84
|
+
self.stats['total_operations'] += 1
|
|
85
|
+
self.stats[operation] = self.stats.get(operation, 0) + 1
|
|
86
|
+
|
|
87
|
+
if response_time > 0:
|
|
88
|
+
self.response_times.append(response_time)
|
|
89
|
+
|
|
90
|
+
if namespace:
|
|
91
|
+
self.namespace_stats[namespace] += 1
|
|
92
|
+
|
|
93
|
+
# 检查是否需要提交统计数据(容错处理)
|
|
94
|
+
try:
|
|
95
|
+
self._check_and_submit()
|
|
96
|
+
except Exception as submit_error:
|
|
97
|
+
# 统计提交失败不应影响统计记录
|
|
98
|
+
logger.error("统计数据提交检查失败,但统计记录继续", {
|
|
99
|
+
'instance_name': self.instance_name,
|
|
100
|
+
'process_id': self.process_id,
|
|
101
|
+
'operation': operation,
|
|
102
|
+
'submit_error': str(submit_error)
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
logger.debug("缓存操作统计已记录", {
|
|
106
|
+
'operation': operation,
|
|
107
|
+
'response_time_ms': round(response_time, 2),
|
|
108
|
+
'namespace': namespace,
|
|
109
|
+
'process_id': self.process_id,
|
|
110
|
+
'old_total_operations': old_total,
|
|
111
|
+
'new_total_operations': self.stats['total_operations']
|
|
112
|
+
})
|
|
85
113
|
except Exception as e:
|
|
86
|
-
#
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
"""智能TTL配置策略"""
|
|
94
|
-
|
|
95
|
-
# TTL策略映射表(秒)
|
|
96
|
-
TTL_STRATEGIES = {
|
|
97
|
-
# 数据库相关缓存
|
|
98
|
-
'sycm_database': 1800, # 数据库列表:30分钟
|
|
99
|
-
'sycm_tables': 1200, # 表列表:20分钟
|
|
100
|
-
'sycm_table_data': 300, # 表数据:5分钟
|
|
101
|
-
|
|
102
|
-
# 用户会话相关
|
|
103
|
-
'user_session': 3600, # 用户会话:1小时
|
|
104
|
-
'user_info': 1800, # 用户信息:30分钟
|
|
105
|
-
|
|
106
|
-
# 静态配置
|
|
107
|
-
'static_config': 86400, # 静态配置:24小时
|
|
108
|
-
'system_config': 43200, # 系统配置:12小时
|
|
109
|
-
|
|
110
|
-
# API相关
|
|
111
|
-
'api_response': 600, # API响应:10分钟
|
|
112
|
-
'api_cache': 900, # API缓存:15分钟
|
|
113
|
-
|
|
114
|
-
# 临时数据
|
|
115
|
-
'temp_data': 300, # 临时数据:5分钟
|
|
116
|
-
'short_cache': 60, # 短期缓存:1分钟
|
|
117
|
-
}
|
|
114
|
+
# 统计记录失败不应影响缓存操作
|
|
115
|
+
logger.error("统计记录失败,但缓存操作继续", {
|
|
116
|
+
'instance_name': self.instance_name,
|
|
117
|
+
'process_id': self.process_id,
|
|
118
|
+
'operation': operation,
|
|
119
|
+
'error': str(e)
|
|
120
|
+
})
|
|
118
121
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
122
|
+
def _check_and_submit(self):
|
|
123
|
+
"""检查并提交统计数据(同步)"""
|
|
124
|
+
if not self.mysql_pool:
|
|
125
|
+
return
|
|
126
|
+
|
|
127
|
+
current_time = time.time()
|
|
128
|
+
time_since_last_submit = current_time - self.last_submit_time
|
|
129
|
+
should_submit = False
|
|
130
|
+
reason = ""
|
|
131
|
+
|
|
132
|
+
# 重构后的提交逻辑:
|
|
133
|
+
# 1. 每N次操作提交一次,但每X秒内只能提交一次(防止频繁提交)
|
|
134
|
+
# 2. 每隔X秒提交一次,但必须满足N次操作
|
|
135
|
+
|
|
136
|
+
if self.stats['total_operations'] % self.submit_interval == 0:
|
|
137
|
+
# 达到操作次数阈值
|
|
138
|
+
if time_since_last_submit >= self.min_submit_interval:
|
|
139
|
+
# 满足时间间隔要求,可以提交
|
|
140
|
+
should_submit = True
|
|
141
|
+
reason = f"达到操作次数阈值({self.submit_interval}次)且满足时间间隔({self.min_submit_interval}秒)"
|
|
142
|
+
else:
|
|
143
|
+
# 时间间隔不够,延迟提交
|
|
144
|
+
remaining_time = self.min_submit_interval - time_since_last_submit
|
|
145
|
+
logger.debug("操作次数达标但时间间隔不足,延迟提交", {
|
|
146
|
+
'instance_name': self.instance_name,
|
|
147
|
+
'process_id': self.process_id,
|
|
148
|
+
'total_operations': self.stats['total_operations'],
|
|
149
|
+
'time_since_last_submit': round(time_since_last_submit, 2),
|
|
150
|
+
'remaining_time': round(remaining_time, 2)
|
|
151
|
+
})
|
|
152
|
+
elif time_since_last_submit >= self.min_submit_interval:
|
|
153
|
+
# 达到时间间隔阈值
|
|
154
|
+
if self.stats['total_operations'] >= self.submit_interval:
|
|
155
|
+
# 满足操作次数要求,可以提交
|
|
156
|
+
should_submit = True
|
|
157
|
+
reason = f"达到时间间隔阈值({self.min_submit_interval}秒)且满足操作次数({self.submit_interval}次)"
|
|
158
|
+
else:
|
|
159
|
+
# 操作次数不够,不提交
|
|
160
|
+
logger.debug("时间间隔达标但操作次数不足,暂不提交", {
|
|
161
|
+
'instance_name': self.instance_name,
|
|
162
|
+
'process_id': self.process_id,
|
|
163
|
+
'total_operations': self.stats['total_operations'],
|
|
164
|
+
'required_operations': self.submit_interval,
|
|
165
|
+
'time_since_last_submit': round(time_since_last_submit, 2)
|
|
166
|
+
})
|
|
123
167
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
168
|
+
if should_submit and self.stats['total_operations'] > 0:
|
|
169
|
+
try:
|
|
170
|
+
self._submit_to_mysql()
|
|
171
|
+
self.last_submit_time = current_time
|
|
172
|
+
logger.info("统计数据已同步提交", {
|
|
173
|
+
'instance_name': self.instance_name,
|
|
174
|
+
'process_id': self.process_id,
|
|
175
|
+
'total_operations': self.stats['total_operations'],
|
|
176
|
+
'time_since_last_submit': round(time_since_last_submit, 2),
|
|
177
|
+
'reason': reason
|
|
178
|
+
})
|
|
179
|
+
except Exception as e:
|
|
180
|
+
logger.error("统计数据提交失败", {
|
|
181
|
+
'instance_name': self.instance_name,
|
|
182
|
+
'process_id': self.process_id,
|
|
183
|
+
'error': str(e)
|
|
184
|
+
})
|
|
185
|
+
|
|
186
|
+
def _submit_to_mysql(self):
|
|
187
|
+
"""同步提交统计数据到MySQL"""
|
|
188
|
+
if not self.mysql_pool:
|
|
189
|
+
return
|
|
127
190
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
191
|
+
stats_data = self.get_stats()
|
|
192
|
+
if not stats_data.get('enabled'):
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
connection = self.mysql_pool.connection()
|
|
197
|
+
with connection.cursor() as cursor:
|
|
198
|
+
# 选择数据库
|
|
199
|
+
cursor.execute(f"USE `{self.config.get('db_name', 'redis_stats')}`")
|
|
200
|
+
|
|
201
|
+
# 插入统计数据
|
|
202
|
+
table_name = self.config.get('table_name', 'cache_performance')
|
|
203
|
+
insert_sql = f"""
|
|
204
|
+
INSERT INTO `{table_name}` (
|
|
205
|
+
`日期`, `实例标识`, `主机名`, `进程ID`, `统计时间`,
|
|
206
|
+
`缓存命中`, `缓存未命中`, `缓存设置`, `缓存删除`, `缓存错误`, `总操作数`,
|
|
207
|
+
`命中率`, `平均响应时间`, `运行时间`, `命名空间统计`
|
|
208
|
+
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
|
209
|
+
"""
|
|
210
|
+
|
|
211
|
+
insert_data = (
|
|
212
|
+
datetime.now().strftime('%Y-%m-%d'),
|
|
213
|
+
f"{self.instance_name}_pid_{self.process_id}",
|
|
214
|
+
socket.gethostname(),
|
|
215
|
+
self.process_id,
|
|
216
|
+
datetime.now(),
|
|
217
|
+
stats_data['hits'],
|
|
218
|
+
stats_data['misses'],
|
|
219
|
+
stats_data['sets'],
|
|
220
|
+
stats_data['deletes'],
|
|
221
|
+
stats_data['errors'],
|
|
222
|
+
stats_data['total_operations'],
|
|
223
|
+
stats_data['hit_rate_percent'],
|
|
224
|
+
stats_data['avg_response_time_ms'],
|
|
225
|
+
stats_data['uptime_seconds'],
|
|
226
|
+
json.dumps(stats_data['namespace_stats'], ensure_ascii=False)
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
cursor.execute(insert_sql, insert_data)
|
|
230
|
+
connection.commit()
|
|
231
|
+
|
|
232
|
+
connection.close()
|
|
233
|
+
|
|
234
|
+
except Exception as e:
|
|
235
|
+
logger.error("MySQL同步提交失败", {
|
|
236
|
+
'instance_name': self.instance_name,
|
|
237
|
+
'process_id': self.process_id,
|
|
238
|
+
'error': str(e)
|
|
239
|
+
})
|
|
240
|
+
raise
|
|
132
241
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
242
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
243
|
+
"""获取统计数据"""
|
|
244
|
+
if not self.enabled:
|
|
245
|
+
return {'enabled': False, 'message': '统计功能已禁用'}
|
|
137
246
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
247
|
+
with self._lock:
|
|
248
|
+
uptime = time.time() - self.stats['start_time']
|
|
249
|
+
total_cache_ops = self.stats['hits'] + self.stats['misses']
|
|
250
|
+
hit_rate = (self.stats['hits'] / total_cache_ops * 100) if total_cache_ops > 0 else 0
|
|
251
|
+
avg_response_time = statistics.mean(self.response_times) if self.response_times else 0
|
|
252
|
+
|
|
253
|
+
return {
|
|
254
|
+
'enabled': True,
|
|
255
|
+
'hits': self.stats['hits'],
|
|
256
|
+
'misses': self.stats['misses'],
|
|
257
|
+
'sets': self.stats['sets'],
|
|
258
|
+
'deletes': self.stats['deletes'],
|
|
259
|
+
'errors': self.stats['errors'],
|
|
260
|
+
'total_operations': self.stats['total_operations'],
|
|
261
|
+
'hit_rate_percent': round(hit_rate, 2),
|
|
262
|
+
'avg_response_time_ms': round(avg_response_time, 2),
|
|
263
|
+
'uptime_seconds': round(uptime, 2),
|
|
264
|
+
'namespace_stats': dict(self.namespace_stats),
|
|
265
|
+
'last_updated': datetime.now().isoformat(),
|
|
266
|
+
'process_id': self.process_id
|
|
267
|
+
}
|
|
143
268
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
269
|
+
def reset_stats(self):
|
|
270
|
+
"""重置统计数据"""
|
|
271
|
+
if not self.enabled:
|
|
272
|
+
return
|
|
273
|
+
|
|
274
|
+
with self._lock:
|
|
275
|
+
self.stats = {
|
|
276
|
+
'hits': 0,
|
|
277
|
+
'misses': 0,
|
|
278
|
+
'sets': 0,
|
|
279
|
+
'deletes': 0,
|
|
280
|
+
'errors': 0,
|
|
281
|
+
'total_operations': 0,
|
|
282
|
+
'start_time': time.time()
|
|
283
|
+
}
|
|
284
|
+
self.response_times.clear()
|
|
285
|
+
self.namespace_stats.clear()
|
|
286
|
+
self.last_submit_time = time.time()
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
class CacheSystemState(enum.Enum):
|
|
290
|
+
"""缓存系统状态枚举"""
|
|
291
|
+
INITIALIZING = "initializing"
|
|
292
|
+
READY = "ready"
|
|
293
|
+
MYSQL_READY = "mysql_ready"
|
|
294
|
+
ERROR = "error"
|
|
295
|
+
SHUTDOWN = "shutdown"
|
|
148
296
|
|
|
149
297
|
|
|
150
298
|
class CacheConfig:
|
|
151
|
-
"""
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
#
|
|
155
|
-
self.
|
|
156
|
-
self.
|
|
157
|
-
self.
|
|
158
|
-
self.
|
|
159
|
-
self.
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
self.
|
|
164
|
-
self.
|
|
165
|
-
|
|
166
|
-
#
|
|
167
|
-
self.
|
|
168
|
-
self.
|
|
169
|
-
|
|
170
|
-
#
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
self.
|
|
174
|
-
|
|
175
|
-
# 热点键配置
|
|
176
|
-
self.max_hot_keys = 1000 # 最大热点键数量
|
|
177
|
-
self.hot_keys_cleanup_threshold = 800 # 热点键清理阈值
|
|
178
|
-
|
|
179
|
-
# MySQL数据库配置
|
|
180
|
-
self.db_name = db_name
|
|
181
|
-
self.table_name = table_name
|
|
182
|
-
|
|
183
|
-
# 锁配置
|
|
184
|
-
self.lock_timeout = 30 # 分布式锁超时时间
|
|
185
|
-
self.lock_retry_delay = 0.1 # 锁重试延迟
|
|
299
|
+
"""缓存配置类"""
|
|
300
|
+
def __init__(self, **kwargs):
|
|
301
|
+
# 基础配置
|
|
302
|
+
self.default_ttl = kwargs.get('default_ttl', 3600) # 默认过期时间(秒)
|
|
303
|
+
self.stats_interval = kwargs.get('stats_interval', 300) # 每N次操作提交一次统计数据
|
|
304
|
+
self.stats_initial_delay = kwargs.get('stats_initial_delay', 300) # 最少N秒提交一次
|
|
305
|
+
self.enable_stats = kwargs.get('enable_stats', True) # 是否启用统计功能
|
|
306
|
+
self.max_value_size = kwargs.get('max_value_size', 10 * 1024 * 1024) # 最大值大小(字节)
|
|
307
|
+
self.cache_prefix = kwargs.get('cache_prefix', 'cache') # 缓存键前缀
|
|
308
|
+
self.enable_compression = kwargs.get('enable_compression', True)
|
|
309
|
+
|
|
310
|
+
# 数据库配置
|
|
311
|
+
self.db_name = kwargs.get('db_name', 'redis_stats')
|
|
312
|
+
self.table_name = kwargs.get('table_name', 'cache_performance')
|
|
313
|
+
|
|
314
|
+
# 智能TTL策略配置
|
|
315
|
+
self.enable_smart_ttl = kwargs.get('enable_smart_ttl', True) # 启用智能TTL
|
|
316
|
+
self.ttl_min = kwargs.get('ttl_min', 60) # 最小TTL(秒)
|
|
317
|
+
self.ttl_max = kwargs.get('ttl_max', 7200) # 最大TTL(秒)
|
|
318
|
+
self.debug_ttl = kwargs.get('debug_ttl', False) # TTL调试模式
|
|
319
|
+
|
|
320
|
+
# 自定义TTL模式
|
|
321
|
+
self.custom_namespace_patterns = kwargs.get('custom_namespace_patterns', {})
|
|
322
|
+
self.custom_key_patterns = kwargs.get('custom_key_patterns', {})
|
|
186
323
|
|
|
187
324
|
|
|
188
325
|
class SmartCacheSystem:
|
|
189
|
-
"""
|
|
326
|
+
"""智能缓存系统 - 单线程"""
|
|
190
327
|
|
|
191
|
-
def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
|
|
192
|
-
config: CacheConfig = None, db_name: str = None, table_name: str = None):
|
|
328
|
+
def __init__(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
|
|
193
329
|
self.redis_client = redis_client
|
|
194
330
|
self.mysql_pool = mysql_pool
|
|
195
331
|
self.instance_name = instance_name
|
|
332
|
+
self.config = CacheConfig(**config)
|
|
196
333
|
|
|
197
|
-
#
|
|
198
|
-
if config:
|
|
199
|
-
self.config = config
|
|
200
|
-
elif db_name or table_name:
|
|
201
|
-
self.config = CacheConfig(
|
|
202
|
-
db_name=db_name or "redis统计",
|
|
203
|
-
table_name=table_name or "dpflask路由分析"
|
|
204
|
-
)
|
|
205
|
-
else:
|
|
206
|
-
self.config = CacheConfig()
|
|
207
|
-
|
|
208
|
-
self.logger = logger
|
|
209
|
-
|
|
210
|
-
# 状态管理
|
|
334
|
+
# 系统状态管理
|
|
211
335
|
self._state = CacheSystemState.INITIALIZING
|
|
212
336
|
self._ready_event = Event()
|
|
213
|
-
self._mysql_ready_event = Event()
|
|
214
|
-
|
|
215
|
-
# 统计数据
|
|
216
|
-
self.stats = {
|
|
217
|
-
'hits': 0,
|
|
218
|
-
'misses': 0,
|
|
219
|
-
'sets': 0,
|
|
220
|
-
'deletes': 0,
|
|
221
|
-
'errors': 0,
|
|
222
|
-
'total_operations': 0,
|
|
223
|
-
'start_time': time.time(),
|
|
224
|
-
'response_times': []
|
|
225
|
-
}
|
|
226
337
|
|
|
227
|
-
#
|
|
228
|
-
self.
|
|
229
|
-
self.
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
338
|
+
# 直接初始化统计系统
|
|
339
|
+
self.stats_collector = None
|
|
340
|
+
if self.config.enable_stats:
|
|
341
|
+
self.stats_collector = CacheStatsCollector(
|
|
342
|
+
enabled=True,
|
|
343
|
+
mysql_pool=self.mysql_pool,
|
|
344
|
+
config={
|
|
345
|
+
'instance_name': self.instance_name,
|
|
346
|
+
'submit_interval': self.config.stats_interval, # 每N次操作提交一次
|
|
347
|
+
'min_submit_interval': 300, # 最少N秒提交一次
|
|
348
|
+
'table_name': self.config.table_name,
|
|
349
|
+
'db_name': self.config.db_name
|
|
350
|
+
}
|
|
351
|
+
)
|
|
238
352
|
|
|
239
|
-
#
|
|
353
|
+
# 初始化系统
|
|
354
|
+
self._initialize()
|
|
355
|
+
|
|
356
|
+
def _initialize(self):
|
|
357
|
+
"""初始化缓存系统"""
|
|
358
|
+
# 测试Redis连接
|
|
240
359
|
if self._test_redis_connection():
|
|
241
360
|
self._state = CacheSystemState.READY
|
|
242
361
|
self._ready_event.set()
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
'instance_name': self.instance_name,
|
|
249
|
-
'state': self._state.value,
|
|
250
|
-
'redis_ready': self._ready_event.is_set(),
|
|
251
|
-
'mysql_enabled': self.mysql_pool is not None
|
|
252
|
-
})
|
|
253
|
-
|
|
254
|
-
def _initialize_mysql_features(self):
|
|
255
|
-
"""在后台初始化MySQL相关功能"""
|
|
256
|
-
try:
|
|
257
|
-
if self.mysql_pool:
|
|
258
|
-
# 延迟初始化MySQL(给系统一些启动时间)
|
|
259
|
-
time.sleep(5) # 等待5秒让主系统完全启动
|
|
260
|
-
|
|
261
|
-
if self._init_mysql_db():
|
|
362
|
+
|
|
363
|
+
# 如果启用统计且有MySQL连接,创建统计表
|
|
364
|
+
if self.config.enable_stats and self.mysql_pool:
|
|
365
|
+
try:
|
|
366
|
+
self._create_simple_stats_table()
|
|
262
367
|
self._state = CacheSystemState.MYSQL_READY
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
# MySQL就绪后再启动统计线程
|
|
266
|
-
self._start_stats_worker()
|
|
267
|
-
|
|
268
|
-
self.logger.info("MySQL功能初始化完成", {
|
|
368
|
+
logger.info("统计功能已启用", {
|
|
269
369
|
'instance_name': self.instance_name,
|
|
270
|
-
'
|
|
370
|
+
'process_id': os.getpid()
|
|
271
371
|
})
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
372
|
+
except Exception as e:
|
|
373
|
+
logger.error("统计表创建失败", {
|
|
374
|
+
'instance_name': self.instance_name,
|
|
375
|
+
'error': str(e)
|
|
376
|
+
})
|
|
377
|
+
else:
|
|
278
378
|
self._state = CacheSystemState.ERROR
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
@property
|
|
282
|
-
def is_ready(self) -> bool:
|
|
283
|
-
"""检查缓存系统是否就绪(Redis可用即为就绪)"""
|
|
284
|
-
return self._ready_event.is_set()
|
|
285
|
-
|
|
286
|
-
@property
|
|
287
|
-
def is_mysql_ready(self) -> bool:
|
|
288
|
-
"""检查MySQL功能是否就绪"""
|
|
289
|
-
return self._mysql_ready_event.is_set()
|
|
290
|
-
|
|
291
|
-
def wait_for_ready(self, timeout: float = 1.0) -> bool:
|
|
292
|
-
"""等待缓存系统就绪(非阻塞,有超时)"""
|
|
293
|
-
return self._ready_event.wait(timeout)
|
|
294
|
-
|
|
295
|
-
def wait_for_mysql_ready(self, timeout: float = 5.0) -> bool:
|
|
296
|
-
"""等待MySQL功能就绪(非阻塞,有超时)"""
|
|
297
|
-
return self._mysql_ready_event.wait(timeout)
|
|
379
|
+
logger.error("Redis连接失败", {'instance_name': self.instance_name})
|
|
298
380
|
|
|
299
381
|
def _test_redis_connection(self) -> bool:
|
|
300
382
|
"""测试Redis连接"""
|
|
@@ -302,652 +384,470 @@ class SmartCacheSystem:
|
|
|
302
384
|
self.redis_client.ping()
|
|
303
385
|
return True
|
|
304
386
|
except Exception as e:
|
|
305
|
-
|
|
387
|
+
logger.error("Redis连接测试失败", {'error': str(e)})
|
|
306
388
|
return False
|
|
307
389
|
|
|
308
|
-
def
|
|
309
|
-
"""
|
|
390
|
+
def _create_simple_stats_table(self):
|
|
391
|
+
"""创建统计表"""
|
|
310
392
|
if not self.mysql_pool:
|
|
311
|
-
|
|
312
|
-
return False
|
|
393
|
+
return
|
|
313
394
|
|
|
314
395
|
try:
|
|
315
|
-
# 设置连接超时,避免长时间阻塞
|
|
316
396
|
connection = self.mysql_pool.connection()
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
pass # 忽略不支持的操作
|
|
397
|
+
with connection.cursor() as cursor:
|
|
398
|
+
# 创建数据库
|
|
399
|
+
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.config.db_name}` DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci")
|
|
400
|
+
cursor.execute(f"USE `{self.config.db_name}`")
|
|
322
401
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
402
|
+
# 创建统计表
|
|
403
|
+
create_table_sql = f"""
|
|
404
|
+
CREATE TABLE IF NOT EXISTS `{self.config.table_name}` (
|
|
405
|
+
`id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID',
|
|
406
|
+
`日期` date NOT NULL COMMENT '统计日期',
|
|
407
|
+
`实例标识` varchar(64) NOT NULL COMMENT '缓存实例标识',
|
|
408
|
+
`主机名` varchar(100) NOT NULL COMMENT '服务器主机名',
|
|
409
|
+
`进程ID` int NOT NULL COMMENT '进程ID',
|
|
410
|
+
`统计时间` timestamp NOT NULL COMMENT '统计时间',
|
|
331
411
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
412
|
+
-- 基础操作统计
|
|
413
|
+
`缓存命中` bigint DEFAULT 0 COMMENT '缓存命中次数',
|
|
414
|
+
`缓存未命中` bigint DEFAULT 0 COMMENT '缓存未命中次数',
|
|
415
|
+
`缓存设置` bigint DEFAULT 0 COMMENT '缓存设置次数',
|
|
416
|
+
`缓存删除` bigint DEFAULT 0 COMMENT '缓存删除次数',
|
|
417
|
+
`缓存错误` bigint DEFAULT 0 COMMENT '缓存错误次数',
|
|
418
|
+
`总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
|
|
335
419
|
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
`日期` date NOT NULL COMMENT '统计日期',
|
|
341
|
-
`统计时间` datetime NOT NULL COMMENT '统计时间',
|
|
342
|
-
`时间段` varchar(20) NOT NULL COMMENT '时间段标识',
|
|
343
|
-
`缓存命中数` bigint DEFAULT 0 COMMENT '缓存命中次数',
|
|
344
|
-
`缓存未命中数` bigint DEFAULT 0 COMMENT '缓存未命中次数',
|
|
345
|
-
`缓存设置数` bigint DEFAULT 0 COMMENT '缓存设置次数',
|
|
346
|
-
`缓存删除数` bigint DEFAULT 0 COMMENT '缓存删除次数',
|
|
347
|
-
`缓存错误数` bigint DEFAULT 0 COMMENT '缓存错误次数',
|
|
348
|
-
`命中率` decimal(5,2) DEFAULT 0.00 COMMENT '缓存命中率(%)',
|
|
349
|
-
`总操作数` bigint DEFAULT 0 COMMENT '总操作次数',
|
|
350
|
-
`平均响应时间` decimal(10,4) DEFAULT 0.0000 COMMENT '平均响应时间(ms)',
|
|
351
|
-
`每秒操作数` decimal(10,2) DEFAULT 0.00 COMMENT '每秒操作数',
|
|
352
|
-
`唯一键数量` int DEFAULT 0 COMMENT '唯一键数量',
|
|
353
|
-
`系统运行时间` bigint DEFAULT 0 COMMENT '系统运行时间(秒)',
|
|
354
|
-
`热点键统计` json DEFAULT NULL COMMENT '热点键统计信息',
|
|
355
|
-
`服务器主机` varchar(100) DEFAULT NULL COMMENT '服务器主机名',
|
|
356
|
-
`实例名称` varchar(100) DEFAULT NULL COMMENT '缓存实例名称',
|
|
357
|
-
`创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
358
|
-
`更新时间` timestamp DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
359
|
-
PRIMARY KEY (`id`),
|
|
360
|
-
KEY `idx_stats_date` (`日期`),
|
|
361
|
-
KEY `idx_stats_time` (`统计时间`),
|
|
362
|
-
KEY `idx_time_period` (`时间段`),
|
|
363
|
-
KEY `idx_hit_rate` (`命中率`),
|
|
364
|
-
KEY `idx_instance` (`实例名称`),
|
|
365
|
-
KEY `idx_create_time` (`创建时间`)
|
|
366
|
-
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存系统统计分析表'
|
|
367
|
-
"""
|
|
420
|
+
-- 性能指标
|
|
421
|
+
`命中率` decimal(5,2) DEFAULT 0.00 COMMENT '命中率百分比',
|
|
422
|
+
`平均响应时间` decimal(10,2) DEFAULT 0.00 COMMENT '平均响应时间(毫秒)',
|
|
423
|
+
`运行时间` bigint DEFAULT 0 COMMENT '运行时间(秒)',
|
|
368
424
|
|
|
369
|
-
|
|
370
|
-
|
|
425
|
+
-- 命名空间统计
|
|
426
|
+
`命名空间统计` json COMMENT '命名空间统计详情',
|
|
371
427
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
'table': self.config.table_name
|
|
375
|
-
})
|
|
376
|
-
return True
|
|
428
|
+
-- 时间戳
|
|
429
|
+
`创建时间` timestamp DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
377
430
|
|
|
378
|
-
|
|
379
|
-
|
|
431
|
+
PRIMARY KEY (`id`),
|
|
432
|
+
KEY `idx_日期` (`日期`),
|
|
433
|
+
KEY `idx_实例时间` (`实例标识`, `统计时间`),
|
|
434
|
+
KEY `idx_创建时间` (`创建时间`)
|
|
435
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='Redis缓存统计表'
|
|
436
|
+
"""
|
|
380
437
|
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
# 不抛出异常,让系统继续运行
|
|
384
|
-
return False
|
|
385
|
-
|
|
386
|
-
def _generate_cache_key(self, key: str, namespace: str = "") -> str:
|
|
387
|
-
"""生成缓存键"""
|
|
388
|
-
if namespace:
|
|
389
|
-
return f"{self.config.cache_prefix}{namespace}:{key}"
|
|
390
|
-
return f"{self.config.cache_prefix}{key}"
|
|
391
|
-
|
|
392
|
-
def _record_operation(self, operation: str, response_time: float = 0):
|
|
393
|
-
"""记录操作统计"""
|
|
394
|
-
with self._stats_lock:
|
|
395
|
-
self.stats['total_operations'] += 1
|
|
396
|
-
if operation in self.stats:
|
|
397
|
-
self.stats[operation] += 1
|
|
398
|
-
if response_time > 0:
|
|
399
|
-
self.stats['response_times'].append(response_time)
|
|
400
|
-
# 只保留最近1000次操作的响应时间
|
|
401
|
-
if len(self.stats['response_times']) > 1000:
|
|
402
|
-
self.stats['response_times'] = self.stats['response_times'][-1000:]
|
|
403
|
-
|
|
404
|
-
def _record_hot_key(self, key: str, namespace: str = ""):
|
|
405
|
-
"""记录热点键"""
|
|
406
|
-
cache_key = self._generate_cache_key(key, namespace)
|
|
407
|
-
with self.hot_keys_lock:
|
|
408
|
-
# 检查是否需要清理热点键(防止内存泄漏)
|
|
409
|
-
if len(self.hot_keys) >= self.config.hot_keys_cleanup_threshold:
|
|
410
|
-
# 保留访问次数最高的键,移除访问次数最少的键
|
|
411
|
-
sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
|
|
412
|
-
# 保留前600个热点键,为新键留出空间
|
|
413
|
-
self.hot_keys = dict(sorted_keys[:600])
|
|
438
|
+
cursor.execute(create_table_sql)
|
|
439
|
+
connection.commit()
|
|
414
440
|
|
|
415
|
-
|
|
416
|
-
'清理前数量': len(sorted_keys),
|
|
417
|
-
'清理后数量': len(self.hot_keys),
|
|
418
|
-
'清理阈值': self.config.hot_keys_cleanup_threshold
|
|
419
|
-
})
|
|
441
|
+
connection.close()
|
|
420
442
|
|
|
421
|
-
|
|
422
|
-
|
|
443
|
+
except Exception as e:
|
|
444
|
+
logger.error("统计表初始化失败", {'error': str(e)})
|
|
445
|
+
raise
|
|
423
446
|
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
self._record_operation('errors')
|
|
434
|
-
self.logger.warning(f"缓存系统未就绪,{operation_name}操作失败", {
|
|
435
|
-
'state': self._state.value
|
|
436
|
-
})
|
|
437
|
-
return None if operation_name == 'get' else False
|
|
438
|
-
|
|
439
|
-
try:
|
|
440
|
-
result = func(self, *args, **kwargs)
|
|
441
|
-
response_time = (time.time() - start_time) * 1000
|
|
442
|
-
self._record_operation(operation_name, response_time)
|
|
443
|
-
return result
|
|
444
|
-
|
|
445
|
-
except Exception as e:
|
|
446
|
-
self._record_operation('errors')
|
|
447
|
-
self.logger.error(f"缓存{operation_name}操作失败: {e}", {
|
|
448
|
-
'args': args[:2] if len(args) > 2 else args, # 避免记录过长的参数
|
|
449
|
-
'operation': operation_name
|
|
450
|
-
})
|
|
451
|
-
return None if operation_name == 'get' else False
|
|
452
|
-
|
|
453
|
-
return wrapper
|
|
454
|
-
return decorator
|
|
447
|
+
@property
|
|
448
|
+
def is_ready(self) -> bool:
|
|
449
|
+
"""检查系统是否就绪"""
|
|
450
|
+
return self._ready_event.is_set()
|
|
451
|
+
|
|
452
|
+
@property
|
|
453
|
+
def is_mysql_ready(self) -> bool:
|
|
454
|
+
"""检查MySQL是否就绪"""
|
|
455
|
+
return self._state == CacheSystemState.MYSQL_READY
|
|
455
456
|
|
|
456
|
-
@_cache_operation('get')
|
|
457
457
|
def get(self, key: str, namespace: str = "", default=None) -> Any:
|
|
458
458
|
"""获取缓存值"""
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
459
|
+
return self._get_with_stats(key, namespace, default)
|
|
460
|
+
|
|
461
|
+
def _get_with_stats(self, key: str, namespace: str = "", default=None) -> Any:
|
|
462
|
+
"""带统计的获取缓存值"""
|
|
463
|
+
if not self.is_ready:
|
|
464
|
+
return default
|
|
463
465
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
self.
|
|
466
|
+
start_time = time.time()
|
|
467
|
+
try:
|
|
468
|
+
cache_key = self._generate_cache_key(key, namespace)
|
|
469
|
+
value = self.redis_client.get(cache_key)
|
|
470
|
+
response_time = (time.time() - start_time) * 1000
|
|
467
471
|
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
472
|
+
if value is not None:
|
|
473
|
+
# 缓存命中
|
|
474
|
+
if self.stats_collector:
|
|
475
|
+
self.stats_collector.record_operation('hits', response_time, namespace)
|
|
476
|
+
try:
|
|
477
|
+
return json.loads(value.decode('utf-8'))
|
|
478
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
479
|
+
return value.decode('utf-8')
|
|
480
|
+
else:
|
|
481
|
+
# 缓存未命中
|
|
482
|
+
if self.stats_collector:
|
|
483
|
+
self.stats_collector.record_operation('misses', response_time, namespace)
|
|
484
|
+
return default
|
|
485
|
+
|
|
486
|
+
except Exception as e:
|
|
487
|
+
response_time = (time.time() - start_time) * 1000
|
|
488
|
+
if self.stats_collector:
|
|
489
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
490
|
+
logger.error("缓存获取失败", {
|
|
491
|
+
'key': key,
|
|
492
|
+
'namespace': namespace,
|
|
493
|
+
'error': str(e)
|
|
494
|
+
})
|
|
474
495
|
return default
|
|
475
496
|
|
|
476
|
-
|
|
477
|
-
def set(self, key: str, value: Any, ttl: int = None, namespace: str = "") -> bool:
|
|
497
|
+
def set(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
|
|
478
498
|
"""设置缓存值"""
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
# 序列化值
|
|
486
|
-
if isinstance(value, (dict, list, tuple)):
|
|
487
|
-
serialized_value = json.dumps(value, ensure_ascii=False, sort_keys=True, cls=MySQLDataEncoder)
|
|
488
|
-
else:
|
|
489
|
-
serialized_value = str(value)
|
|
490
|
-
|
|
491
|
-
# 检查值大小
|
|
492
|
-
if len(serialized_value.encode('utf-8')) > self.config.max_value_size:
|
|
493
|
-
self.logger.warning(f"缓存值过大,跳过设置: {len(serialized_value)} bytes")
|
|
499
|
+
return self._set_with_stats(key, value, ttl, namespace)
|
|
500
|
+
|
|
501
|
+
def _set_with_stats(self, key: str, value: Any, ttl: Optional[int] = None, namespace: str = "") -> bool:
|
|
502
|
+
"""带统计的设置缓存值"""
|
|
503
|
+
if not self.is_ready:
|
|
494
504
|
return False
|
|
495
505
|
|
|
496
|
-
# 设置缓存
|
|
497
|
-
result = self.redis_client.setex(cache_key, ttl, serialized_value)
|
|
498
|
-
|
|
499
|
-
# 记录TTL策略使用情况
|
|
500
|
-
if namespace and ttl != self.config.default_ttl:
|
|
501
|
-
self.logger.debug("使用智能TTL策略", {
|
|
502
|
-
'namespace': namespace,
|
|
503
|
-
'ttl': ttl,
|
|
504
|
-
'key': key[:50] + "..." if len(key) > 50 else key
|
|
505
|
-
})
|
|
506
|
-
|
|
507
|
-
return bool(result)
|
|
508
|
-
|
|
509
|
-
def delete(self, key: str, namespace: str = "") -> bool:
|
|
510
|
-
"""删除缓存值"""
|
|
511
506
|
start_time = time.time()
|
|
512
|
-
|
|
513
507
|
try:
|
|
514
508
|
cache_key = self._generate_cache_key(key, namespace)
|
|
515
|
-
|
|
509
|
+
|
|
510
|
+
# 智能TTL策略
|
|
511
|
+
if ttl is None:
|
|
512
|
+
ttl = self._get_smart_ttl(namespace, key, len(json.dumps(value, ensure_ascii=False, default=str)))
|
|
513
|
+
|
|
514
|
+
# 序列化值
|
|
515
|
+
if isinstance(value, (dict, list)):
|
|
516
|
+
serialized_value = json.dumps(value, ensure_ascii=False, default=str)
|
|
517
|
+
else:
|
|
518
|
+
serialized_value = str(value)
|
|
519
|
+
|
|
520
|
+
# 检查值大小
|
|
521
|
+
value_size = len(serialized_value.encode('utf-8'))
|
|
522
|
+
if value_size > self.config.max_value_size:
|
|
523
|
+
if self.stats_collector:
|
|
524
|
+
self.stats_collector.record_operation('errors', 0, namespace)
|
|
525
|
+
logger.warning("缓存值过大,跳过设置", {
|
|
526
|
+
'key': key,
|
|
527
|
+
'size': len(serialized_value),
|
|
528
|
+
'max_size': self.config.max_value_size
|
|
529
|
+
})
|
|
530
|
+
return False
|
|
531
|
+
|
|
532
|
+
result = self.redis_client.setex(cache_key, ttl, serialized_value)
|
|
516
533
|
response_time = (time.time() - start_time) * 1000
|
|
517
534
|
|
|
518
|
-
self.
|
|
535
|
+
if self.stats_collector:
|
|
536
|
+
self.stats_collector.record_operation('sets', response_time, namespace)
|
|
519
537
|
return bool(result)
|
|
520
538
|
|
|
521
539
|
except Exception as e:
|
|
522
|
-
|
|
523
|
-
self.
|
|
540
|
+
response_time = (time.time() - start_time) * 1000
|
|
541
|
+
if self.stats_collector:
|
|
542
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
543
|
+
logger.error("缓存设置失败", {
|
|
524
544
|
'key': key,
|
|
525
|
-
'namespace': namespace
|
|
545
|
+
'namespace': namespace,
|
|
546
|
+
'error': str(e)
|
|
526
547
|
})
|
|
527
548
|
return False
|
|
528
549
|
|
|
529
|
-
def
|
|
530
|
-
"""
|
|
550
|
+
def delete(self, key: str, namespace: str = "") -> bool:
|
|
551
|
+
"""删除缓存值"""
|
|
552
|
+
return self._delete_with_stats(key, namespace)
|
|
553
|
+
|
|
554
|
+
def _delete_with_stats(self, key: str, namespace: str = "") -> bool:
|
|
555
|
+
"""带统计的删除缓存值"""
|
|
556
|
+
if not self.is_ready:
|
|
557
|
+
return False
|
|
558
|
+
|
|
559
|
+
start_time = time.time()
|
|
531
560
|
try:
|
|
532
561
|
cache_key = self._generate_cache_key(key, namespace)
|
|
533
|
-
|
|
562
|
+
result = self.redis_client.delete(cache_key)
|
|
563
|
+
response_time = (time.time() - start_time) * 1000
|
|
564
|
+
if self.stats_collector:
|
|
565
|
+
self.stats_collector.record_operation('deletes', response_time, namespace)
|
|
566
|
+
return bool(result)
|
|
567
|
+
|
|
534
568
|
except Exception as e:
|
|
535
|
-
|
|
569
|
+
response_time = (time.time() - start_time) * 1000
|
|
570
|
+
if self.stats_collector:
|
|
571
|
+
self.stats_collector.record_operation('errors', response_time, namespace)
|
|
572
|
+
logger.error("缓存删除失败", {
|
|
573
|
+
'key': key,
|
|
574
|
+
'namespace': namespace,
|
|
575
|
+
'error': str(e)
|
|
576
|
+
})
|
|
536
577
|
return False
|
|
537
578
|
|
|
538
579
|
def clear_namespace(self, namespace: str) -> int:
|
|
539
580
|
"""清除指定命名空间的所有缓存"""
|
|
581
|
+
if not self.is_ready:
|
|
582
|
+
return 0
|
|
583
|
+
|
|
540
584
|
try:
|
|
541
|
-
pattern = f"{self.config.cache_prefix}{namespace}:*"
|
|
585
|
+
pattern = f"{self.config.cache_prefix}:{namespace}:*"
|
|
542
586
|
keys = self.redis_client.keys(pattern)
|
|
543
587
|
|
|
544
588
|
if keys:
|
|
545
589
|
deleted = self.redis_client.delete(*keys)
|
|
546
|
-
self.logger.info(f"清除命名空间缓存: {namespace}, 删除键数: {deleted}")
|
|
547
590
|
return deleted
|
|
548
591
|
return 0
|
|
549
592
|
|
|
550
593
|
except Exception as e:
|
|
551
|
-
|
|
594
|
+
logger.error("清除命名空间失败", {
|
|
595
|
+
'namespace': namespace,
|
|
596
|
+
'error': str(e)
|
|
597
|
+
})
|
|
552
598
|
return 0
|
|
553
599
|
|
|
554
|
-
def
|
|
555
|
-
"""
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
600
|
+
def _generate_cache_key(self, key: str, namespace: str = "") -> str:
|
|
601
|
+
"""生成缓存键"""
|
|
602
|
+
if namespace:
|
|
603
|
+
return f"{self.config.cache_prefix}:{namespace}:{key}"
|
|
604
|
+
return f"{self.config.cache_prefix}:{key}"
|
|
605
|
+
|
|
606
|
+
def _get_smart_ttl(self, namespace: str, key: str = "", data_size: int = 0) -> int:
|
|
607
|
+
"""智能TTL策略 - 通用版本
|
|
608
|
+
|
|
609
|
+
基于多种因素智能计算TTL:
|
|
610
|
+
1. 命名空间模式匹配
|
|
611
|
+
2. 数据类型推断
|
|
612
|
+
3. 数据大小考虑
|
|
613
|
+
4. 访问频率预测
|
|
614
|
+
"""
|
|
615
|
+
|
|
616
|
+
# 如果禁用智能TTL,直接返回默认值
|
|
617
|
+
if not self.config.enable_smart_ttl:
|
|
618
|
+
return self.config.default_ttl
|
|
619
|
+
|
|
620
|
+
# 1. 基于命名空间模式的智能匹配
|
|
621
|
+
namespace_patterns = {
|
|
622
|
+
# 数据库相关 - 变化频率低,TTL较长
|
|
623
|
+
r'.*database.*|.*db.*|.*schema.*': 1800, # 30分钟
|
|
559
624
|
|
|
560
|
-
#
|
|
561
|
-
|
|
625
|
+
# 表结构相关 - 变化频率低,TTL较长
|
|
626
|
+
r'.*table.*|.*column.*|.*field.*': 1200, # 20分钟
|
|
562
627
|
|
|
563
|
-
#
|
|
564
|
-
|
|
565
|
-
avg_response_time = sum(response_times) / len(response_times) if response_times else 0
|
|
628
|
+
# 数据查询相关 - 变化频率中等,TTL中等
|
|
629
|
+
r'.*data.*|.*query.*|.*result.*': 300, # 5分钟
|
|
566
630
|
|
|
567
|
-
#
|
|
568
|
-
|
|
631
|
+
# 用户会话相关 - 变化频率高,TTL较短
|
|
632
|
+
r'.*session.*|.*user.*|.*auth.*': 900, # 15分钟
|
|
569
633
|
|
|
570
|
-
#
|
|
571
|
-
|
|
634
|
+
# 配置相关 - 变化频率很低,TTL很长
|
|
635
|
+
r'.*config.*|.*setting.*|.*param.*': 3600, # 1小时
|
|
572
636
|
|
|
573
|
-
#
|
|
574
|
-
|
|
575
|
-
top_hot_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)[:10]
|
|
637
|
+
# 统计相关 - 变化频率中等,TTL中等
|
|
638
|
+
r'.*stats.*|.*metric.*|.*count.*': 600, # 10分钟
|
|
576
639
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
'errors': self.stats['errors'],
|
|
583
|
-
'total_operations': total_ops,
|
|
584
|
-
'hit_rate': round(hit_rate, 2),
|
|
585
|
-
'avg_response_time': round(avg_response_time, 4),
|
|
586
|
-
'ops_per_second': round(ops_per_second, 2),
|
|
587
|
-
'uptime_seconds': int(uptime),
|
|
588
|
-
'hot_keys': dict(top_hot_keys),
|
|
589
|
-
'unique_keys_count': len(self.hot_keys)
|
|
590
|
-
}
|
|
591
|
-
|
|
592
|
-
def health_check(self) -> Dict[str, Any]:
|
|
593
|
-
"""健康检查"""
|
|
594
|
-
health_info = {
|
|
595
|
-
'redis_connected': False,
|
|
596
|
-
'mysql_available': False,
|
|
597
|
-
'stats_worker_running': self._stats_running,
|
|
598
|
-
'instance_name': self.instance_name,
|
|
599
|
-
'timestamp': datetime.now().isoformat()
|
|
640
|
+
# 缓存相关 - 变化频率高,TTL较短
|
|
641
|
+
r'.*cache.*|.*temp.*|.*tmp.*': 180, # 3分钟
|
|
642
|
+
|
|
643
|
+
# 列表相关 - 变化频率中等,TTL中等
|
|
644
|
+
r'.*list.*|.*index.*|.*catalog.*': 450, # 7.5分钟
|
|
600
645
|
}
|
|
601
646
|
|
|
602
|
-
#
|
|
603
|
-
|
|
604
|
-
self.redis_client.ping()
|
|
605
|
-
health_info['redis_connected'] = True
|
|
606
|
-
except Exception as e:
|
|
607
|
-
health_info['redis_error'] = str(e)
|
|
647
|
+
# 合并用户自定义模式(优先级更高)
|
|
648
|
+
namespace_patterns.update(self.config.custom_namespace_patterns)
|
|
608
649
|
|
|
609
|
-
#
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
650
|
+
# 2. 基于键名模式的智能匹配
|
|
651
|
+
key_patterns = {
|
|
652
|
+
# ID相关 - 相对稳定
|
|
653
|
+
r'.*_id$|^id_.*': 1.2, # TTL倍数
|
|
654
|
+
|
|
655
|
+
# 详情相关 - 相对稳定
|
|
656
|
+
r'.*detail.*|.*info.*': 1.1,
|
|
657
|
+
|
|
658
|
+
# 列表相关 - 变化较频繁
|
|
659
|
+
r'.*list.*|.*items.*': 0.8,
|
|
660
|
+
|
|
661
|
+
# 计数相关 - 变化频繁
|
|
662
|
+
r'.*count.*|.*num.*|.*total.*': 0.6,
|
|
663
|
+
|
|
664
|
+
# 状态相关 - 变化很频繁
|
|
665
|
+
r'.*status.*|.*state.*': 0.5,
|
|
666
|
+
}
|
|
617
667
|
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
668
|
+
# 合并用户自定义键模式
|
|
669
|
+
key_patterns.update(self.config.custom_key_patterns)
|
|
670
|
+
|
|
671
|
+
# 3. 基于数据大小的调整
|
|
672
|
+
size_factor = 1.0
|
|
673
|
+
if data_size > 0:
|
|
674
|
+
if data_size > 1024 * 1024: # > 1MB,延长TTL减少重建开销
|
|
675
|
+
size_factor = 1.5
|
|
676
|
+
elif data_size > 100 * 1024: # > 100KB
|
|
677
|
+
size_factor = 1.2
|
|
678
|
+
elif data_size < 1024: # < 1KB,缩短TTL减少内存占用
|
|
679
|
+
size_factor = 0.8
|
|
680
|
+
|
|
681
|
+
# 4. 计算基础TTL
|
|
682
|
+
base_ttl = self.config.default_ttl
|
|
683
|
+
|
|
684
|
+
# 匹配命名空间模式
|
|
685
|
+
for pattern, ttl in namespace_patterns.items():
|
|
686
|
+
if re.match(pattern, namespace.lower()):
|
|
687
|
+
base_ttl = ttl
|
|
688
|
+
break
|
|
689
|
+
|
|
690
|
+
# 5. 应用键名模式调整
|
|
691
|
+
key_factor = 1.0
|
|
692
|
+
for pattern, factor in key_patterns.items():
|
|
693
|
+
if re.match(pattern, key.lower()):
|
|
694
|
+
key_factor = factor
|
|
695
|
+
break
|
|
696
|
+
|
|
697
|
+
# 6. 计算最终TTL
|
|
698
|
+
final_ttl = int(base_ttl * key_factor * size_factor)
|
|
699
|
+
|
|
700
|
+
# 7. TTL边界限制(使用配置值)
|
|
701
|
+
final_ttl = max(self.config.ttl_min, min(self.config.ttl_max, final_ttl))
|
|
702
|
+
|
|
703
|
+
# 8. 记录智能TTL决策(使用配置开关)
|
|
704
|
+
if self.config.debug_ttl:
|
|
705
|
+
logger.debug("智能TTL计算", {
|
|
706
|
+
'namespace': namespace,
|
|
707
|
+
'key': key[:50] + "..." if len(key) > 50 else key,
|
|
708
|
+
'data_size': data_size,
|
|
709
|
+
'base_ttl': base_ttl,
|
|
710
|
+
'key_factor': key_factor,
|
|
711
|
+
'size_factor': size_factor,
|
|
712
|
+
'final_ttl': final_ttl
|
|
633
713
|
})
|
|
714
|
+
|
|
715
|
+
return final_ttl
|
|
634
716
|
|
|
635
|
-
def
|
|
636
|
-
"""
|
|
637
|
-
|
|
717
|
+
def get_stats(self) -> Dict[str, Any]:
|
|
718
|
+
"""获取统计信息"""
|
|
719
|
+
# 确保统计系统已初始化
|
|
720
|
+
if self.stats_collector:
|
|
721
|
+
return self.stats_collector.get_stats()
|
|
638
722
|
|
|
639
|
-
|
|
640
|
-
self.logger.info("统计线程启动,等待首个统计间隔", {
|
|
723
|
+
logger.debug("统计系统未初始化,返回空统计信息", {
|
|
641
724
|
'instance_name': self.instance_name,
|
|
642
|
-
'
|
|
643
|
-
'delay_reason': '避免初始化时的阻塞操作'
|
|
725
|
+
'process_id': os.getpid()
|
|
644
726
|
})
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
while self._stats_running:
|
|
651
|
-
try:
|
|
652
|
-
# 收集统计数据
|
|
653
|
-
stats_data = self.get_stats()
|
|
654
|
-
|
|
655
|
-
# 提交到MySQL(带状态检查)
|
|
656
|
-
self._submit_stats_to_mysql(stats_data)
|
|
657
|
-
|
|
658
|
-
# 清理过期的热点键统计
|
|
659
|
-
self._cleanup_hot_keys()
|
|
660
|
-
|
|
661
|
-
# 定期清理过期数据
|
|
662
|
-
cleanup_counter += 1
|
|
663
|
-
if cleanup_counter >= 24: # 24 * 1800秒 = 12小时
|
|
664
|
-
self._cleanup_expired_mysql_data()
|
|
665
|
-
cleanup_counter = 0
|
|
666
|
-
|
|
667
|
-
except Exception as e:
|
|
668
|
-
self.logger.error(f"统计工作线程异常: {e}")
|
|
669
|
-
|
|
670
|
-
# 可中断的等待下一个统计间隔
|
|
671
|
-
if not self._interruptible_sleep(self.config.stats_interval):
|
|
672
|
-
break
|
|
727
|
+
return {
|
|
728
|
+
'enabled': False,
|
|
729
|
+
'message': '统计系统未初始化',
|
|
730
|
+
'process_id': os.getpid()
|
|
731
|
+
}
|
|
673
732
|
|
|
674
|
-
def
|
|
675
|
-
"""
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
time.
|
|
681
|
-
elapsed += sleep_interval
|
|
733
|
+
def health_check(self) -> Dict[str, Any]:
|
|
734
|
+
"""健康检查"""
|
|
735
|
+
try:
|
|
736
|
+
# Redis连接检查
|
|
737
|
+
redis_latency = time.time()
|
|
738
|
+
self.redis_client.ping()
|
|
739
|
+
redis_latency = (time.time() - redis_latency) * 1000
|
|
682
740
|
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
def _submit_stats_to_mysql(self, stats_data: Dict[str, Any]):
|
|
686
|
-
"""提交统计数据到MySQL"""
|
|
687
|
-
if not self.mysql_pool:
|
|
688
|
-
self.logger.debug("MySQL连接池不可用,跳过统计数据提交")
|
|
689
|
-
return
|
|
741
|
+
# 获取Redis信息
|
|
742
|
+
redis_info = self.redis_client.info()
|
|
690
743
|
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
'
|
|
695
|
-
'
|
|
696
|
-
|
|
697
|
-
|
|
744
|
+
return {
|
|
745
|
+
'status': 'healthy',
|
|
746
|
+
'redis_connected': True,
|
|
747
|
+
'redis_latency_ms': round(redis_latency, 2),
|
|
748
|
+
'mysql_enabled': self.mysql_pool is not None,
|
|
749
|
+
'mysql_ready': self.is_mysql_ready,
|
|
750
|
+
'system_state': self._state.value,
|
|
751
|
+
'uptime_seconds': time.time() - (self.stats_collector.stats['start_time'] if self.stats_collector else time.time()),
|
|
752
|
+
'redis_memory_used': redis_info.get('used_memory', 0),
|
|
753
|
+
'redis_connected_clients': redis_info.get('connected_clients', 0),
|
|
754
|
+
'process_id': os.getpid()
|
|
755
|
+
}
|
|
698
756
|
|
|
699
|
-
logger.info("统计数据", {'stats_data': stats_data})
|
|
700
|
-
|
|
701
|
-
try:
|
|
702
|
-
# 使用上下文管理器确保连接正确关闭
|
|
703
|
-
connection = self.mysql_pool.connection()
|
|
704
|
-
try:
|
|
705
|
-
with connection.cursor() as cursor:
|
|
706
|
-
cursor.execute(f"USE `{self.config.db_name}`")
|
|
707
|
-
|
|
708
|
-
# 准备数据
|
|
709
|
-
now = datetime.now()
|
|
710
|
-
date_str = now.strftime("%Y-%m-%d")
|
|
711
|
-
time_period = now.strftime("%Y%m%d_%H%M")
|
|
712
|
-
|
|
713
|
-
insert_sql = f"""
|
|
714
|
-
INSERT INTO `{self.config.table_name}` (
|
|
715
|
-
`日期`, `统计时间`, `时间段`, `缓存命中数`, `缓存未命中数`, `缓存设置数`,
|
|
716
|
-
`缓存删除数`, `缓存错误数`, `命中率`, `总操作数`, `平均响应时间`,
|
|
717
|
-
`每秒操作数`, `唯一键数量`, `系统运行时间`, `热点键统计`,
|
|
718
|
-
`服务器主机`, `实例名称`
|
|
719
|
-
) VALUES (
|
|
720
|
-
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
721
|
-
)
|
|
722
|
-
"""
|
|
723
|
-
|
|
724
|
-
cursor.execute(insert_sql, (
|
|
725
|
-
date_str,
|
|
726
|
-
now,
|
|
727
|
-
time_period,
|
|
728
|
-
stats_data['hits'],
|
|
729
|
-
stats_data['misses'],
|
|
730
|
-
stats_data['sets'],
|
|
731
|
-
stats_data['deletes'],
|
|
732
|
-
stats_data['errors'],
|
|
733
|
-
stats_data['hit_rate'],
|
|
734
|
-
stats_data['total_operations'],
|
|
735
|
-
stats_data['avg_response_time'],
|
|
736
|
-
stats_data['ops_per_second'],
|
|
737
|
-
stats_data['unique_keys_count'],
|
|
738
|
-
stats_data['uptime_seconds'],
|
|
739
|
-
json.dumps(stats_data['hot_keys'], ensure_ascii=False, cls=MySQLDataEncoder),
|
|
740
|
-
socket.gethostname(),
|
|
741
|
-
self.instance_name
|
|
742
|
-
))
|
|
743
|
-
|
|
744
|
-
connection.commit()
|
|
745
|
-
|
|
746
|
-
self.logger.info("统计数据已提交到MySQL", {
|
|
747
|
-
'time_period': time_period,
|
|
748
|
-
'total_operations': stats_data['total_operations'],
|
|
749
|
-
'hit_rate': stats_data['hit_rate'],
|
|
750
|
-
'instance_name': self.instance_name
|
|
751
|
-
})
|
|
752
|
-
|
|
753
|
-
finally:
|
|
754
|
-
connection.close()
|
|
755
|
-
|
|
756
757
|
except Exception as e:
|
|
757
|
-
|
|
758
|
-
'
|
|
759
|
-
'
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
"""清理热点键统计 """
|
|
764
|
-
with self.hot_keys_lock:
|
|
765
|
-
current_count = len(self.hot_keys)
|
|
766
|
-
|
|
767
|
-
# 如果热点键数量超过最大限制,进行清理
|
|
768
|
-
if current_count > self.config.max_hot_keys:
|
|
769
|
-
# 保留访问次数最高的键
|
|
770
|
-
sorted_keys = sorted(self.hot_keys.items(), key=lambda x: x[1], reverse=True)
|
|
771
|
-
# 保留80%的热点键
|
|
772
|
-
keep_count = int(self.config.max_hot_keys * 0.8)
|
|
773
|
-
self.hot_keys = dict(sorted_keys[:keep_count])
|
|
774
|
-
|
|
775
|
-
self.logger.info("定期热点键清理", {
|
|
776
|
-
'清理前数量': current_count,
|
|
777
|
-
'清理后数量': len(self.hot_keys),
|
|
778
|
-
'保留比例': '80%'
|
|
779
|
-
})
|
|
758
|
+
return {
|
|
759
|
+
'status': 'unhealthy',
|
|
760
|
+
'error': str(e),
|
|
761
|
+
'redis_connected': False,
|
|
762
|
+
'system_state': self._state.value
|
|
763
|
+
}
|
|
780
764
|
|
|
781
|
-
def
|
|
782
|
-
"""
|
|
783
|
-
|
|
765
|
+
def _record_operation(self, operation: str, response_time: float = 0, namespace: str = "", key: str = ""):
|
|
766
|
+
"""记录操作统计 """
|
|
767
|
+
# 如果禁用统计功能,直接返回
|
|
768
|
+
if not self.config.enable_stats:
|
|
784
769
|
return
|
|
770
|
+
|
|
771
|
+
logger.debug("调用操作记录", {
|
|
772
|
+
'operation': operation,
|
|
773
|
+
'response_time_ms': round(response_time, 2),
|
|
774
|
+
'namespace': namespace,
|
|
775
|
+
'key': key[:50] + "..." if len(key) > 50 else key,
|
|
776
|
+
'instance_name': self.instance_name
|
|
777
|
+
})
|
|
785
778
|
|
|
786
|
-
try:
|
|
787
|
-
connection = self.mysql_pool.connection()
|
|
788
|
-
try:
|
|
789
|
-
with connection.cursor() as cursor:
|
|
790
|
-
cursor.execute(f"USE `{self.config.db_name}`")
|
|
791
|
-
|
|
792
|
-
# 计算过期时间点
|
|
793
|
-
from datetime import timedelta
|
|
794
|
-
expire_date = datetime.now() - timedelta(days=self.config.stats_retention)
|
|
795
|
-
|
|
796
|
-
# 删除过期数据
|
|
797
|
-
delete_sql = f"""
|
|
798
|
-
DELETE FROM `{self.config.table_name}`
|
|
799
|
-
WHERE `统计时间` < %s
|
|
800
|
-
"""
|
|
801
|
-
|
|
802
|
-
cursor.execute(delete_sql, (expire_date,))
|
|
803
|
-
deleted_rows = cursor.rowcount
|
|
804
|
-
connection.commit()
|
|
805
|
-
|
|
806
|
-
if deleted_rows > 0:
|
|
807
|
-
self.logger.info("清理过期MySQL统计数据", {
|
|
808
|
-
'deleted_rows': deleted_rows,
|
|
809
|
-
'expire_date': expire_date.strftime('%Y-%m-%d %H:%M:%S'),
|
|
810
|
-
'retention_days': self.config.stats_retention
|
|
811
|
-
})
|
|
812
|
-
|
|
813
|
-
finally:
|
|
814
|
-
connection.close()
|
|
815
|
-
|
|
816
|
-
except Exception as e:
|
|
817
|
-
self.logger.error(f"清理过期MySQL统计数据失败: {e}")
|
|
818
|
-
|
|
819
779
|
def shutdown(self):
|
|
820
780
|
"""关闭缓存系统"""
|
|
821
|
-
self.
|
|
822
|
-
'instance_name': self.instance_name,
|
|
823
|
-
'state': self._state.value
|
|
824
|
-
})
|
|
781
|
+
self._state = CacheSystemState.SHUTDOWN
|
|
825
782
|
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
self.logger.info("等待统计线程结束...")
|
|
830
|
-
self._stats_thread.join(timeout=10) # 最多等待10秒
|
|
831
|
-
|
|
832
|
-
if self._stats_thread.is_alive():
|
|
833
|
-
self.logger.warning("统计线程未能在超时时间内结束")
|
|
834
|
-
|
|
835
|
-
# 关闭线程池
|
|
836
|
-
if hasattr(self, '_executor'):
|
|
837
|
-
self.logger.info("关闭线程池...")
|
|
838
|
-
self._executor.shutdown(wait=True, timeout=5)
|
|
839
|
-
|
|
840
|
-
# 最后一次提交统计数据(如果MySQL可用)
|
|
841
|
-
if self.is_mysql_ready:
|
|
842
|
-
try:
|
|
843
|
-
self.logger.info("提交最终统计数据...")
|
|
844
|
-
stats_data = self.get_stats()
|
|
845
|
-
self._submit_stats_to_mysql(stats_data)
|
|
846
|
-
except Exception as e:
|
|
847
|
-
self.logger.error(f"关闭时提交统计数据失败: {e}")
|
|
783
|
+
if self.stats_collector:
|
|
784
|
+
# The stats_submitter is now a CacheStatsCollector, so we call its reset_stats
|
|
785
|
+
self.stats_collector.reset_stats()
|
|
848
786
|
|
|
849
|
-
|
|
850
|
-
'instance_name': self.instance_name
|
|
851
|
-
})
|
|
852
|
-
|
|
853
|
-
def __enter__(self):
|
|
854
|
-
"""上下文管理器支持"""
|
|
855
|
-
return self
|
|
856
|
-
|
|
857
|
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
858
|
-
"""上下文管理器退出时自动关闭"""
|
|
859
|
-
self.shutdown()
|
|
860
|
-
return False
|
|
787
|
+
logger.info("缓存系统已关闭", {'instance_name': self.instance_name})
|
|
861
788
|
|
|
862
789
|
|
|
863
790
|
class CacheManager:
|
|
864
|
-
"""缓存管理器
|
|
791
|
+
"""缓存管理器"""
|
|
865
792
|
|
|
866
793
|
_instance = None
|
|
867
|
-
_lock = threading.
|
|
794
|
+
_lock = threading.Lock()
|
|
868
795
|
|
|
869
796
|
def __new__(cls):
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
cls._instance
|
|
873
|
-
|
|
874
|
-
|
|
797
|
+
if cls._instance is None:
|
|
798
|
+
with cls._lock:
|
|
799
|
+
if cls._instance is None:
|
|
800
|
+
cls._instance = super().__new__(cls)
|
|
801
|
+
return cls._instance
|
|
875
802
|
|
|
876
803
|
def __init__(self):
|
|
877
|
-
if self
|
|
804
|
+
if not hasattr(self, '_initialized'):
|
|
805
|
+
self.cache_instance = None
|
|
806
|
+
self._initialized = True
|
|
807
|
+
|
|
808
|
+
def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default", **config):
|
|
809
|
+
"""初始化缓存系统"""
|
|
810
|
+
if self.cache_instance is not None:
|
|
811
|
+
logger.warning("缓存系统已初始化,跳过重复初始化", {
|
|
812
|
+
'existing_instance': self.cache_instance.instance_name,
|
|
813
|
+
'new_instance': instance_name
|
|
814
|
+
})
|
|
878
815
|
return
|
|
879
816
|
|
|
880
|
-
self.cache_instance =
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
def initialize(self, redis_client: redis.Redis, mysql_pool=None, instance_name: str = "default",
|
|
887
|
-
config: CacheConfig = None, db_name: str = None, table_name: str = None):
|
|
888
|
-
"""初始化缓存系统(非阻塞)"""
|
|
889
|
-
try:
|
|
890
|
-
# 立即创建缓存实例,内部会异步初始化MySQL和统计线程
|
|
891
|
-
self.cache_instance = SmartCacheSystem(
|
|
892
|
-
redis_client=redis_client,
|
|
893
|
-
mysql_pool=mysql_pool,
|
|
894
|
-
instance_name=instance_name,
|
|
895
|
-
config=config,
|
|
896
|
-
db_name=db_name,
|
|
897
|
-
table_name=table_name
|
|
898
|
-
)
|
|
899
|
-
self.initialization_error = None
|
|
900
|
-
self.logger.info("缓存管理器初始化成功(异步模式)", {
|
|
901
|
-
'instance_name': instance_name,
|
|
902
|
-
'mysql_enabled': mysql_pool is not None,
|
|
903
|
-
'db_name': self.cache_instance.config.db_name,
|
|
904
|
-
'table_name': self.cache_instance.config.table_name,
|
|
905
|
-
'note': 'MySQL初始化将在后台异步完成'
|
|
906
|
-
})
|
|
907
|
-
return self # 支持链式调用
|
|
908
|
-
|
|
909
|
-
except Exception as e:
|
|
910
|
-
self.initialization_error = str(e)
|
|
911
|
-
self.cache_instance = None
|
|
912
|
-
self.logger.error(f"缓存管理器初始化失败: {e}")
|
|
913
|
-
return self
|
|
817
|
+
self.cache_instance = SmartCacheSystem(
|
|
818
|
+
redis_client=redis_client,
|
|
819
|
+
mysql_pool=mysql_pool,
|
|
820
|
+
instance_name=instance_name,
|
|
821
|
+
**config
|
|
822
|
+
)
|
|
914
823
|
|
|
915
824
|
def get_cache(self) -> Optional[SmartCacheSystem]:
|
|
916
825
|
"""获取缓存实例"""
|
|
917
|
-
return self.cache_instance
|
|
826
|
+
return self.cache_instance
|
|
918
827
|
|
|
919
828
|
def is_available(self) -> bool:
|
|
920
829
|
"""检查缓存是否可用"""
|
|
921
|
-
return self.cache_instance is not None and self.
|
|
922
|
-
|
|
923
|
-
def enable(self):
|
|
924
|
-
"""启用缓存"""
|
|
925
|
-
self.enabled = True
|
|
926
|
-
self.logger.info("缓存系统已启用")
|
|
927
|
-
return self # 支持链式调用
|
|
928
|
-
|
|
929
|
-
def disable(self):
|
|
930
|
-
"""禁用缓存"""
|
|
931
|
-
self.enabled = False
|
|
932
|
-
self.logger.info("缓存系统已禁用")
|
|
933
|
-
return self # 支持链式调用
|
|
830
|
+
return self.cache_instance is not None and self.cache_instance.is_ready
|
|
934
831
|
|
|
935
832
|
def get_status(self) -> Dict[str, Any]:
|
|
936
833
|
"""获取缓存状态"""
|
|
834
|
+
if self.cache_instance is None:
|
|
835
|
+
return {
|
|
836
|
+
'enabled': False,
|
|
837
|
+
'available': False,
|
|
838
|
+
'stats_enabled': False,
|
|
839
|
+
'initialization_error': 'Cache not initialized'
|
|
840
|
+
}
|
|
841
|
+
|
|
937
842
|
return {
|
|
938
|
-
'enabled':
|
|
939
|
-
'available': self.cache_instance
|
|
940
|
-
'
|
|
941
|
-
'
|
|
843
|
+
'enabled': True,
|
|
844
|
+
'available': self.cache_instance.is_ready,
|
|
845
|
+
'mysql_ready': self.cache_instance.is_mysql_ready,
|
|
846
|
+
'stats_enabled': self.cache_instance.config.enable_stats,
|
|
847
|
+
'state': self.cache_instance._state.value,
|
|
848
|
+
'instance_name': self.cache_instance.instance_name
|
|
942
849
|
}
|
|
943
|
-
|
|
944
|
-
def shutdown(self):
|
|
945
|
-
"""关闭缓存系统"""
|
|
946
|
-
if self.cache_instance:
|
|
947
|
-
self.cache_instance.shutdown()
|
|
948
|
-
self.cache_instance = None
|
|
949
|
-
self.logger.info("缓存管理器已关闭")
|
|
950
850
|
|
|
951
851
|
|
|
952
|
-
#
|
|
852
|
+
# 全局缓存管理器实例
|
|
953
853
|
cache_manager = CacheManager()
|