mdbq 4.0.79__py3-none-any.whl → 4.0.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/__version__.py +1 -1
- mdbq/route/__init__.py +1 -0
- mdbq/route/analytics.py +618 -0
- mdbq/route/example.py +378 -0
- mdbq/route/monitor.py +691 -0
- mdbq/route/routes.py +576 -0
- mdbq/selenium/get_driver.py +5 -5
- {mdbq-4.0.79.dist-info → mdbq-4.0.81.dist-info}/METADATA +2 -2
- {mdbq-4.0.79.dist-info → mdbq-4.0.81.dist-info}/RECORD +11 -6
- {mdbq-4.0.79.dist-info → mdbq-4.0.81.dist-info}/WHEEL +1 -1
- {mdbq-4.0.79.dist-info → mdbq-4.0.81.dist-info}/top_level.txt +0 -0
mdbq/route/monitor.py
ADDED
@@ -0,0 +1,691 @@
|
|
1
|
+
"""
|
2
|
+
路由监控系统
|
3
|
+
专业的API接口访问监控、记录和统计系统
|
4
|
+
|
5
|
+
主要功能:
|
6
|
+
1. 监控所有路由接口的访问请求
|
7
|
+
2. 记录详细的请求信息(IP、设备、请求头、请求体等)
|
8
|
+
3. 提供统计分析功能
|
9
|
+
4. 异常处理和数据清理
|
10
|
+
|
11
|
+
"""
|
12
|
+
|
13
|
+
import os
|
14
|
+
import json
|
15
|
+
import time
|
16
|
+
import uuid
|
17
|
+
import pymysql
|
18
|
+
import hashlib
|
19
|
+
import functools
|
20
|
+
from datetime import datetime, timedelta
|
21
|
+
from typing import Dict, Any, Optional, List
|
22
|
+
from urllib.parse import urlparse, parse_qs
|
23
|
+
from dbutils.pooled_db import PooledDB
|
24
|
+
from mdbq.myconf import myconf
|
25
|
+
from flask import request, g
|
26
|
+
import re
|
27
|
+
import ipaddress
|
28
|
+
|
29
|
+
parser = myconf.ConfigParser()
|
30
|
+
host, port, username, password = parser.get_section_values(
|
31
|
+
file_path=os.path.join(os.path.expanduser("~"), 'spd.txt'),
|
32
|
+
section='mysql',
|
33
|
+
keys=['host', 'port', 'username', 'password'],
|
34
|
+
)
|
35
|
+
|
36
|
+
|
37
|
+
class RouteMonitor:
|
38
|
+
"""路由监控核心类"""
|
39
|
+
|
40
|
+
def __init__(self, pool=None):
|
41
|
+
"""初始化监控系统"""
|
42
|
+
if pool is not None:
|
43
|
+
self.pool = pool
|
44
|
+
else:
|
45
|
+
self.init_database_pool()
|
46
|
+
self.init_database_tables()
|
47
|
+
|
48
|
+
def init_database_pool(self):
|
49
|
+
"""初始化数据库连接池"""
|
50
|
+
self.pool = PooledDB(
|
51
|
+
creator=pymysql,
|
52
|
+
maxconnections=3, # 最大连接数
|
53
|
+
mincached=1, # 初始化空闲连接数
|
54
|
+
maxcached=3, # 空闲连接最大缓存数
|
55
|
+
blocking=True,
|
56
|
+
host=host,
|
57
|
+
port=int(port),
|
58
|
+
user=username,
|
59
|
+
password=password,
|
60
|
+
ping=1,
|
61
|
+
charset='utf8mb4',
|
62
|
+
cursorclass=pymysql.cursors.DictCursor
|
63
|
+
)
|
64
|
+
|
65
|
+
def init_database_tables(self):
|
66
|
+
"""初始化数据库表结构"""
|
67
|
+
try:
|
68
|
+
connection = self.pool.connection()
|
69
|
+
try:
|
70
|
+
with connection.cursor() as cursor:
|
71
|
+
# 创建详细请求记录表 - 修复MySQL 8.4+兼容性
|
72
|
+
cursor.execute("""
|
73
|
+
CREATE TABLE IF NOT EXISTS `api_request_logs` (
|
74
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
75
|
+
`request_id` VARCHAR(64) NOT NULL COMMENT '请求唯一标识',
|
76
|
+
`timestamp` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '请求时间(精确到毫秒)',
|
77
|
+
`method` VARCHAR(10) NOT NULL COMMENT 'HTTP方法',
|
78
|
+
`endpoint` VARCHAR(500) NOT NULL COMMENT '请求端点',
|
79
|
+
`full_url` TEXT COMMENT '完整URL',
|
80
|
+
`client_ip` VARCHAR(45) NOT NULL COMMENT '客户端IP地址',
|
81
|
+
`real_ip` VARCHAR(45) COMMENT '真实IP地址',
|
82
|
+
`forwarded_ips` TEXT COMMENT '转发IP链',
|
83
|
+
`user_agent` TEXT COMMENT '用户代理',
|
84
|
+
`referer` VARCHAR(1000) COMMENT '来源页面',
|
85
|
+
`host` VARCHAR(255) COMMENT '请求主机',
|
86
|
+
`scheme` VARCHAR(10) COMMENT '协议类型',
|
87
|
+
`port` INT COMMENT '端口号',
|
88
|
+
`request_headers` JSON COMMENT '请求头信息',
|
89
|
+
`request_params` JSON COMMENT '请求参数',
|
90
|
+
`request_body` LONGTEXT COMMENT '请求体内容',
|
91
|
+
`request_size` INT DEFAULT 0 COMMENT '请求大小(字节)',
|
92
|
+
`response_status` INT COMMENT '响应状态码',
|
93
|
+
`response_size` INT COMMENT '响应大小(字节)',
|
94
|
+
`process_time` DECIMAL(10,3) COMMENT '处理时间(毫秒)',
|
95
|
+
`session_id` VARCHAR(128) COMMENT '会话ID',
|
96
|
+
`user_id` VARCHAR(64) COMMENT '用户ID',
|
97
|
+
`auth_token` VARCHAR(255) COMMENT '认证令牌(脱敏)',
|
98
|
+
`device_fingerprint` VARCHAR(128) COMMENT '设备指纹',
|
99
|
+
`device_info` JSON COMMENT '设备信息',
|
100
|
+
`geo_country` VARCHAR(50) COMMENT '地理位置-国家',
|
101
|
+
`geo_region` VARCHAR(100) COMMENT '地理位置-地区',
|
102
|
+
`geo_city` VARCHAR(100) COMMENT '地理位置-城市',
|
103
|
+
`is_bot` BOOLEAN DEFAULT FALSE COMMENT '是否为机器人',
|
104
|
+
`is_mobile` BOOLEAN DEFAULT FALSE COMMENT '是否为移动设备',
|
105
|
+
`browser_name` VARCHAR(50) COMMENT '浏览器名称',
|
106
|
+
`browser_version` VARCHAR(20) COMMENT '浏览器版本',
|
107
|
+
`os_name` VARCHAR(50) COMMENT '操作系统名称',
|
108
|
+
`os_version` VARCHAR(20) COMMENT '操作系统版本',
|
109
|
+
`error_message` TEXT COMMENT '错误信息',
|
110
|
+
`business_data` JSON COMMENT '业务数据',
|
111
|
+
`tags` JSON COMMENT '标签信息',
|
112
|
+
UNIQUE KEY `uk_request_id` (`request_id`),
|
113
|
+
INDEX `idx_timestamp` (`timestamp`),
|
114
|
+
INDEX `idx_endpoint` (`endpoint`(191)),
|
115
|
+
INDEX `idx_client_ip` (`client_ip`),
|
116
|
+
INDEX `idx_user_id` (`user_id`),
|
117
|
+
INDEX `idx_status` (`response_status`),
|
118
|
+
INDEX `idx_method_endpoint` (`method`, `endpoint`(191)),
|
119
|
+
INDEX `idx_timestamp_endpoint` (`timestamp`, `endpoint`(191))
|
120
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
121
|
+
COMMENT='API请求详细日志表';
|
122
|
+
""")
|
123
|
+
|
124
|
+
# 创建访问统计汇总表
|
125
|
+
cursor.execute("""
|
126
|
+
CREATE TABLE IF NOT EXISTS `api_access_statistics` (
|
127
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
128
|
+
`date` DATE NOT NULL COMMENT '统计日期',
|
129
|
+
`hour` TINYINT NOT NULL DEFAULT 0 COMMENT '小时(0-23)',
|
130
|
+
`endpoint` VARCHAR(500) NOT NULL COMMENT '端点',
|
131
|
+
`method` VARCHAR(10) NOT NULL COMMENT 'HTTP方法',
|
132
|
+
`total_requests` INT UNSIGNED DEFAULT 0 COMMENT '总请求数',
|
133
|
+
`success_requests` INT UNSIGNED DEFAULT 0 COMMENT '成功请求数',
|
134
|
+
`error_requests` INT UNSIGNED DEFAULT 0 COMMENT '错误请求数',
|
135
|
+
`unique_ips` INT UNSIGNED DEFAULT 0 COMMENT '唯一IP数',
|
136
|
+
`unique_users` INT UNSIGNED DEFAULT 0 COMMENT '唯一用户数',
|
137
|
+
`avg_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应时间(毫秒)',
|
138
|
+
`max_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '最大响应时间(毫秒)',
|
139
|
+
`min_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '最小响应时间(毫秒)',
|
140
|
+
`total_request_size` BIGINT UNSIGNED DEFAULT 0 COMMENT '总请求大小(字节)',
|
141
|
+
`total_response_size` BIGINT UNSIGNED DEFAULT 0 COMMENT '总响应大小(字节)',
|
142
|
+
`bot_requests` INT UNSIGNED DEFAULT 0 COMMENT '机器人请求数',
|
143
|
+
`mobile_requests` INT UNSIGNED DEFAULT 0 COMMENT '移动端请求数',
|
144
|
+
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
145
|
+
`updated_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
146
|
+
UNIQUE KEY `uk_date_hour_endpoint_method` (`date`, `hour`, `endpoint`(191), `method`),
|
147
|
+
INDEX `idx_date` (`date`),
|
148
|
+
INDEX `idx_endpoint` (`endpoint`(191)),
|
149
|
+
INDEX `idx_date_endpoint` (`date`, `endpoint`(191))
|
150
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
151
|
+
COMMENT='API访问统计汇总表';
|
152
|
+
""")
|
153
|
+
|
154
|
+
# 创建IP访问统计表
|
155
|
+
cursor.execute("""
|
156
|
+
CREATE TABLE IF NOT EXISTS `ip_access_statistics` (
|
157
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
158
|
+
`date` DATE NOT NULL COMMENT '统计日期',
|
159
|
+
`ip_address` VARCHAR(45) NOT NULL COMMENT 'IP地址',
|
160
|
+
`total_requests` INT UNSIGNED DEFAULT 0 COMMENT '总请求数',
|
161
|
+
`unique_endpoints` INT UNSIGNED DEFAULT 0 COMMENT '访问的唯一端点数',
|
162
|
+
`success_rate` DECIMAL(5,2) DEFAULT 0 COMMENT '成功率(%)',
|
163
|
+
`avg_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应时间(毫秒)',
|
164
|
+
`first_access` DATETIME COMMENT '首次访问时间',
|
165
|
+
`last_access` DATETIME COMMENT '最后访问时间',
|
166
|
+
`user_agent_hash` VARCHAR(64) COMMENT '用户代理哈希',
|
167
|
+
`is_suspicious` BOOLEAN DEFAULT FALSE COMMENT '是否可疑',
|
168
|
+
`risk_score` TINYINT UNSIGNED DEFAULT 0 COMMENT '风险评分(0-100)',
|
169
|
+
`geo_country` VARCHAR(50) COMMENT '地理位置-国家',
|
170
|
+
`geo_region` VARCHAR(100) COMMENT '地理位置-地区',
|
171
|
+
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
|
172
|
+
`updated_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
|
173
|
+
UNIQUE KEY `uk_date_ip` (`date`, `ip_address`),
|
174
|
+
INDEX `idx_date` (`date`),
|
175
|
+
INDEX `idx_ip` (`ip_address`),
|
176
|
+
INDEX `idx_suspicious` (`is_suspicious`),
|
177
|
+
INDEX `idx_risk_score` (`risk_score`)
|
178
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
179
|
+
COMMENT='IP访问统计表';
|
180
|
+
""")
|
181
|
+
|
182
|
+
# 创建系统性能统计表
|
183
|
+
cursor.execute("""
|
184
|
+
CREATE TABLE IF NOT EXISTS `system_performance_stats` (
|
185
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
186
|
+
`timestamp` DATETIME NOT NULL COMMENT '统计时间',
|
187
|
+
`total_requests_per_minute` INT UNSIGNED DEFAULT 0 COMMENT '每分钟总请求数',
|
188
|
+
`avg_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应时间(毫秒)',
|
189
|
+
`error_rate` DECIMAL(5,2) DEFAULT 0 COMMENT '错误率(%)',
|
190
|
+
`active_ips` INT UNSIGNED DEFAULT 0 COMMENT '活跃IP数',
|
191
|
+
`peak_concurrent_requests` INT UNSIGNED DEFAULT 0 COMMENT '峰值并发请求数',
|
192
|
+
`slowest_endpoint` VARCHAR(500) COMMENT '最慢端点',
|
193
|
+
`slowest_response_time` DECIMAL(10,3) DEFAULT 0 COMMENT '最慢响应时间(毫秒)',
|
194
|
+
`most_accessed_endpoint` VARCHAR(500) COMMENT '最热门端点',
|
195
|
+
`most_accessed_count` INT UNSIGNED DEFAULT 0 COMMENT '最热门端点访问次数',
|
196
|
+
INDEX `idx_timestamp` (`timestamp`)
|
197
|
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
198
|
+
COMMENT='系统性能统计表';
|
199
|
+
""")
|
200
|
+
|
201
|
+
connection.commit()
|
202
|
+
finally:
|
203
|
+
connection.close()
|
204
|
+
except Exception as e:
|
205
|
+
# 静默处理初始化错误,避免影响主应用
|
206
|
+
pass
|
207
|
+
|
208
|
+
def generate_request_id(self) -> str:
|
209
|
+
"""生成唯一的请求ID"""
|
210
|
+
timestamp = str(int(time.time() * 1000)) # 毫秒时间戳
|
211
|
+
random_part = uuid.uuid4().hex[:8]
|
212
|
+
return f"req_{timestamp}_{random_part}"
|
213
|
+
|
214
|
+
def extract_device_info(self, user_agent: str) -> Dict[str, Any]:
|
215
|
+
"""提取设备信息"""
|
216
|
+
device_info = {
|
217
|
+
'is_mobile': False,
|
218
|
+
'is_bot': False,
|
219
|
+
'browser_name': 'Unknown',
|
220
|
+
'browser_version': 'Unknown',
|
221
|
+
'os_name': 'Unknown',
|
222
|
+
'os_version': 'Unknown'
|
223
|
+
}
|
224
|
+
|
225
|
+
if not user_agent:
|
226
|
+
return device_info
|
227
|
+
|
228
|
+
user_agent_lower = user_agent.lower()
|
229
|
+
|
230
|
+
# 检测移动设备
|
231
|
+
mobile_keywords = ['mobile', 'android', 'iphone', 'ipad', 'ipod', 'windows phone']
|
232
|
+
device_info['is_mobile'] = any(keyword in user_agent_lower for keyword in mobile_keywords)
|
233
|
+
|
234
|
+
# 检测机器人
|
235
|
+
bot_keywords = ['bot', 'crawler', 'spider', 'scraper', 'curl', 'wget', 'python-requests']
|
236
|
+
device_info['is_bot'] = any(keyword in user_agent_lower for keyword in bot_keywords)
|
237
|
+
|
238
|
+
# 浏览器检测
|
239
|
+
browsers = [
|
240
|
+
('chrome', r'chrome/(\d+)'),
|
241
|
+
('firefox', r'firefox/(\d+)'),
|
242
|
+
('safari', r'safari/(\d+)'),
|
243
|
+
('edge', r'edge/(\d+)'),
|
244
|
+
('opera', r'opera/(\d+)')
|
245
|
+
]
|
246
|
+
|
247
|
+
for browser, pattern in browsers:
|
248
|
+
match = re.search(pattern, user_agent_lower)
|
249
|
+
if match:
|
250
|
+
device_info['browser_name'] = browser.title()
|
251
|
+
device_info['browser_version'] = match.group(1)
|
252
|
+
break
|
253
|
+
|
254
|
+
# 操作系统检测
|
255
|
+
os_patterns = [
|
256
|
+
('Windows', r'windows nt (\d+\.\d+)'),
|
257
|
+
('macOS', r'mac os x (\d+_\d+)'),
|
258
|
+
('Linux', r'linux'),
|
259
|
+
('Android', r'android (\d+)'),
|
260
|
+
('iOS', r'os (\d+_\d+)')
|
261
|
+
]
|
262
|
+
|
263
|
+
for os_name, pattern in os_patterns:
|
264
|
+
match = re.search(pattern, user_agent_lower)
|
265
|
+
if match:
|
266
|
+
device_info['os_name'] = os_name
|
267
|
+
if len(match.groups()) > 0:
|
268
|
+
device_info['os_version'] = match.group(1).replace('_', '.')
|
269
|
+
break
|
270
|
+
|
271
|
+
return device_info
|
272
|
+
|
273
|
+
def generate_device_fingerprint(self, request_data: Dict) -> str:
|
274
|
+
"""生成设备指纹"""
|
275
|
+
fingerprint_data = {
|
276
|
+
'user_agent': request_data.get('user_agent', ''),
|
277
|
+
'accept_language': request_data.get('request_headers', {}).get('Accept-Language', ''),
|
278
|
+
'accept_encoding': request_data.get('request_headers', {}).get('Accept-Encoding', ''),
|
279
|
+
'connection': request_data.get('request_headers', {}).get('Connection', ''),
|
280
|
+
}
|
281
|
+
|
282
|
+
fingerprint_str = json.dumps(fingerprint_data, sort_keys=True)
|
283
|
+
return hashlib.md5(fingerprint_str.encode()).hexdigest()
|
284
|
+
|
285
|
+
def sanitize_data(self, data: Any, max_length: int = 10000) -> Any:
|
286
|
+
"""数据清理和截断"""
|
287
|
+
if data is None:
|
288
|
+
return None
|
289
|
+
|
290
|
+
if isinstance(data, str):
|
291
|
+
# 移除敏感信息
|
292
|
+
sensitive_patterns = [
|
293
|
+
(r'password["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'password: [REDACTED]'),
|
294
|
+
(r'token["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'token: [REDACTED]'),
|
295
|
+
(r'key["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'key: [REDACTED]'),
|
296
|
+
]
|
297
|
+
|
298
|
+
sanitized = data
|
299
|
+
for pattern, replacement in sensitive_patterns:
|
300
|
+
sanitized = re.sub(pattern, replacement, sanitized, flags=re.IGNORECASE)
|
301
|
+
|
302
|
+
# 截断过长的内容
|
303
|
+
if len(sanitized) > max_length:
|
304
|
+
sanitized = sanitized[:max_length] + '...[TRUNCATED]'
|
305
|
+
|
306
|
+
return sanitized
|
307
|
+
|
308
|
+
elif isinstance(data, dict):
|
309
|
+
sanitized = {}
|
310
|
+
for key, value in data.items():
|
311
|
+
if key.lower() in ['password', 'token', 'key', 'secret']:
|
312
|
+
sanitized[key] = '[REDACTED]'
|
313
|
+
else:
|
314
|
+
sanitized[key] = self.sanitize_data(value, max_length)
|
315
|
+
return sanitized
|
316
|
+
|
317
|
+
elif isinstance(data, list):
|
318
|
+
return [self.sanitize_data(item, max_length) for item in data[:100]] # 限制列表长度
|
319
|
+
|
320
|
+
return data
|
321
|
+
|
322
|
+
def get_real_ip(self, request) -> tuple:
|
323
|
+
"""获取真实IP地址"""
|
324
|
+
# IP地址优先级顺序
|
325
|
+
ip_headers = [
|
326
|
+
'X-Forwarded-For',
|
327
|
+
'X-Real-IP',
|
328
|
+
'CF-Connecting-IP', # Cloudflare
|
329
|
+
'X-Client-IP',
|
330
|
+
'X-Forwarded',
|
331
|
+
'Forwarded-For',
|
332
|
+
'Forwarded'
|
333
|
+
]
|
334
|
+
|
335
|
+
forwarded_ips = []
|
336
|
+
real_ip = request.remote_addr
|
337
|
+
|
338
|
+
for header in ip_headers:
|
339
|
+
header_value = request.headers.get(header)
|
340
|
+
if header_value:
|
341
|
+
# 处理多个IP的情况(用逗号分隔)
|
342
|
+
ips = [ip.strip() for ip in header_value.split(',')]
|
343
|
+
forwarded_ips.extend(ips)
|
344
|
+
|
345
|
+
# 取第一个有效的IP作为真实IP
|
346
|
+
for ip in ips:
|
347
|
+
if self.is_valid_ip(ip) and not self.is_private_ip(ip):
|
348
|
+
real_ip = ip
|
349
|
+
break
|
350
|
+
|
351
|
+
if real_ip != request.remote_addr:
|
352
|
+
break
|
353
|
+
|
354
|
+
return real_ip, forwarded_ips
|
355
|
+
|
356
|
+
def is_valid_ip(self, ip: str) -> bool:
|
357
|
+
"""验证IP地址格式"""
|
358
|
+
try:
|
359
|
+
ipaddress.ip_address(ip)
|
360
|
+
return True
|
361
|
+
except ValueError:
|
362
|
+
return False
|
363
|
+
|
364
|
+
def is_private_ip(self, ip: str) -> bool:
|
365
|
+
"""检查是否为私有IP"""
|
366
|
+
try:
|
367
|
+
return ipaddress.ip_address(ip).is_private
|
368
|
+
except ValueError:
|
369
|
+
return False
|
370
|
+
|
371
|
+
def collect_request_data(self, request) -> Dict[str, Any]:
|
372
|
+
"""收集请求数据"""
|
373
|
+
start_time = getattr(g, 'request_start_time', time.time())
|
374
|
+
request_id = self.generate_request_id()
|
375
|
+
|
376
|
+
# 设置请求ID到全局变量中,供后续使用
|
377
|
+
g.request_id = request_id
|
378
|
+
|
379
|
+
# 获取真实IP
|
380
|
+
real_ip, forwarded_ips = self.get_real_ip(request)
|
381
|
+
|
382
|
+
# 获取请求头信息
|
383
|
+
headers = dict(request.headers)
|
384
|
+
sanitized_headers = self.sanitize_data(headers)
|
385
|
+
|
386
|
+
# 获取请求参数
|
387
|
+
request_params = {}
|
388
|
+
if request.args:
|
389
|
+
request_params.update(dict(request.args))
|
390
|
+
|
391
|
+
# 获取请求体
|
392
|
+
request_body = None
|
393
|
+
request_size = 0
|
394
|
+
|
395
|
+
try:
|
396
|
+
if request.method in ['POST', 'PUT', 'PATCH']:
|
397
|
+
if request.is_json:
|
398
|
+
request_body = request.get_json()
|
399
|
+
elif request.form:
|
400
|
+
request_body = dict(request.form)
|
401
|
+
else:
|
402
|
+
body_data = request.get_data()
|
403
|
+
if body_data:
|
404
|
+
try:
|
405
|
+
request_body = body_data.decode('utf-8')
|
406
|
+
except UnicodeDecodeError:
|
407
|
+
request_body = f"[BINARY_DATA:{len(body_data)}_bytes]"
|
408
|
+
|
409
|
+
if request_body:
|
410
|
+
request_size = len(str(request_body).encode('utf-8'))
|
411
|
+
except Exception:
|
412
|
+
request_body = "[ERROR_READING_BODY]"
|
413
|
+
|
414
|
+
# 清理敏感数据
|
415
|
+
sanitized_body = self.sanitize_data(request_body)
|
416
|
+
sanitized_params = self.sanitize_data(request_params)
|
417
|
+
|
418
|
+
# 设备信息提取
|
419
|
+
user_agent = request.headers.get('User-Agent', '')
|
420
|
+
device_info = self.extract_device_info(user_agent)
|
421
|
+
|
422
|
+
# URL解析
|
423
|
+
parsed_url = urlparse(request.url)
|
424
|
+
|
425
|
+
# 构建请求数据
|
426
|
+
request_data = {
|
427
|
+
'request_id': request_id,
|
428
|
+
'timestamp': datetime.now(),
|
429
|
+
'method': request.method,
|
430
|
+
'endpoint': request.endpoint or request.path,
|
431
|
+
'full_url': request.url,
|
432
|
+
'client_ip': request.remote_addr,
|
433
|
+
'real_ip': real_ip,
|
434
|
+
'forwarded_ips': json.dumps(forwarded_ips) if forwarded_ips else None,
|
435
|
+
'user_agent': user_agent,
|
436
|
+
'referer': request.headers.get('Referer'),
|
437
|
+
'host': request.headers.get('Host'),
|
438
|
+
'scheme': parsed_url.scheme,
|
439
|
+
'port': parsed_url.port,
|
440
|
+
'request_headers': json.dumps(sanitized_headers),
|
441
|
+
'request_params': json.dumps(sanitized_params) if sanitized_params else None,
|
442
|
+
'request_body': json.dumps(sanitized_body) if sanitized_body else None,
|
443
|
+
'request_size': request_size,
|
444
|
+
'session_id': request.cookies.get('session_id'),
|
445
|
+
'user_id': getattr(request, 'current_user', {}).get('id') if hasattr(request, 'current_user') else None,
|
446
|
+
'auth_token': self.mask_token(request.headers.get('Authorization')),
|
447
|
+
'device_fingerprint': self.generate_device_fingerprint({
|
448
|
+
'user_agent': user_agent,
|
449
|
+
'request_headers': sanitized_headers
|
450
|
+
}),
|
451
|
+
'device_info': json.dumps(device_info),
|
452
|
+
'is_bot': device_info['is_bot'],
|
453
|
+
'is_mobile': device_info['is_mobile'],
|
454
|
+
'browser_name': device_info['browser_name'],
|
455
|
+
'browser_version': device_info['browser_version'],
|
456
|
+
'os_name': device_info['os_name'],
|
457
|
+
'os_version': device_info['os_version'],
|
458
|
+
}
|
459
|
+
|
460
|
+
return request_data
|
461
|
+
|
462
|
+
def mask_token(self, token: str) -> str:
|
463
|
+
"""脱敏处理令牌"""
|
464
|
+
if not token:
|
465
|
+
return None
|
466
|
+
|
467
|
+
if len(token) <= 8:
|
468
|
+
return '*' * len(token)
|
469
|
+
|
470
|
+
return token[:4] + '*' * (len(token) - 8) + token[-4:]
|
471
|
+
|
472
|
+
def save_request_log(self, request_data: Dict[str, Any], response_data: Dict[str, Any] = None):
|
473
|
+
"""保存请求日志到数据库"""
|
474
|
+
try:
|
475
|
+
# 合并响应数据
|
476
|
+
if response_data:
|
477
|
+
request_data.update(response_data)
|
478
|
+
|
479
|
+
connection = self.pool.connection()
|
480
|
+
try:
|
481
|
+
with connection.cursor() as cursor:
|
482
|
+
# 插入请求日志
|
483
|
+
columns = ', '.join([f"`{key}`" for key in request_data.keys()])
|
484
|
+
placeholders = ', '.join(['%s'] * len(request_data))
|
485
|
+
|
486
|
+
sql = f"""
|
487
|
+
INSERT INTO `api_request_logs` ({columns})
|
488
|
+
VALUES ({placeholders})
|
489
|
+
"""
|
490
|
+
|
491
|
+
cursor.execute(sql, list(request_data.values()))
|
492
|
+
connection.commit()
|
493
|
+
finally:
|
494
|
+
connection.close()
|
495
|
+
|
496
|
+
except Exception as e:
|
497
|
+
# 静默处理错误,不影响主业务
|
498
|
+
pass
|
499
|
+
|
500
|
+
def update_statistics(self, request_data: Dict[str, Any]):
|
501
|
+
"""更新统计数据"""
|
502
|
+
try:
|
503
|
+
connection = self.pool.connection()
|
504
|
+
try:
|
505
|
+
with connection.cursor() as cursor:
|
506
|
+
now = datetime.now()
|
507
|
+
date = now.date()
|
508
|
+
hour = now.hour
|
509
|
+
|
510
|
+
# 更新API访问统计
|
511
|
+
cursor.execute("""
|
512
|
+
INSERT INTO `api_access_statistics`
|
513
|
+
(`date`, `hour`, `endpoint`, `method`, `total_requests`,
|
514
|
+
`success_requests`, `error_requests`, `avg_response_time`)
|
515
|
+
VALUES (%s, %s, %s, %s, 1, %s, %s, %s)
|
516
|
+
ON DUPLICATE KEY UPDATE
|
517
|
+
`total_requests` = `total_requests` + 1,
|
518
|
+
`success_requests` = `success_requests` + %s,
|
519
|
+
`error_requests` = `error_requests` + %s,
|
520
|
+
`avg_response_time` = (
|
521
|
+
(`avg_response_time` * (`total_requests` - 1) + %s) / `total_requests`
|
522
|
+
),
|
523
|
+
`updated_at` = CURRENT_TIMESTAMP
|
524
|
+
""", (
|
525
|
+
date, hour,
|
526
|
+
request_data.get('endpoint', ''),
|
527
|
+
request_data.get('method', ''),
|
528
|
+
1 if (request_data.get('response_status', 500) < 400) else 0,
|
529
|
+
1 if (request_data.get('response_status', 500) >= 400) else 0,
|
530
|
+
request_data.get('process_time', 0),
|
531
|
+
1 if (request_data.get('response_status', 500) < 400) else 0,
|
532
|
+
1 if (request_data.get('response_status', 500) >= 400) else 0,
|
533
|
+
request_data.get('process_time', 0)
|
534
|
+
))
|
535
|
+
|
536
|
+
# 更新IP访问统计
|
537
|
+
cursor.execute("""
|
538
|
+
INSERT INTO `ip_access_statistics`
|
539
|
+
(`date`, `ip_address`, `total_requests`, `first_access`, `last_access`,
|
540
|
+
`user_agent_hash`)
|
541
|
+
VALUES (%s, %s, 1, %s, %s, %s)
|
542
|
+
ON DUPLICATE KEY UPDATE
|
543
|
+
`total_requests` = `total_requests` + 1,
|
544
|
+
`last_access` = %s,
|
545
|
+
`updated_at` = CURRENT_TIMESTAMP
|
546
|
+
""", (
|
547
|
+
date,
|
548
|
+
request_data.get('real_ip', request_data.get('client_ip')),
|
549
|
+
now, now,
|
550
|
+
hashlib.md5((request_data.get('user_agent', '')).encode()).hexdigest(),
|
551
|
+
now
|
552
|
+
))
|
553
|
+
|
554
|
+
connection.commit()
|
555
|
+
finally:
|
556
|
+
connection.close()
|
557
|
+
|
558
|
+
except Exception as e:
|
559
|
+
# 静默处理错误
|
560
|
+
pass
|
561
|
+
|
562
|
+
def monitor_request(self, func):
|
563
|
+
"""请求监控装饰器"""
|
564
|
+
@functools.wraps(func)
|
565
|
+
def wrapper(*args, **kwargs):
|
566
|
+
# 记录开始时间
|
567
|
+
start_time = time.time()
|
568
|
+
g.request_start_time = start_time
|
569
|
+
|
570
|
+
# 收集请求数据
|
571
|
+
request_data = self.collect_request_data(request)
|
572
|
+
|
573
|
+
try:
|
574
|
+
# 执行原函数
|
575
|
+
response = func(*args, **kwargs)
|
576
|
+
|
577
|
+
# 记录响应信息
|
578
|
+
end_time = time.time()
|
579
|
+
process_time = round((end_time - start_time) * 1000, 3)
|
580
|
+
|
581
|
+
response_data = {
|
582
|
+
'response_status': getattr(response, 'status_code', 200) if hasattr(response, 'status_code') else 200,
|
583
|
+
'process_time': process_time,
|
584
|
+
'response_size': len(str(response.get_data() if hasattr(response, 'get_data') else ''))
|
585
|
+
}
|
586
|
+
|
587
|
+
# 保存日志
|
588
|
+
self.save_request_log(request_data, response_data)
|
589
|
+
|
590
|
+
# 更新统计
|
591
|
+
request_data.update(response_data)
|
592
|
+
self.update_statistics(request_data)
|
593
|
+
|
594
|
+
return response
|
595
|
+
|
596
|
+
except Exception as e:
|
597
|
+
# 记录错误信息
|
598
|
+
end_time = time.time()
|
599
|
+
process_time = round((end_time - start_time) * 1000, 3)
|
600
|
+
|
601
|
+
error_data = {
|
602
|
+
'response_status': 500,
|
603
|
+
'process_time': process_time,
|
604
|
+
'error_message': str(e),
|
605
|
+
'response_size': 0
|
606
|
+
}
|
607
|
+
|
608
|
+
# 保存错误日志
|
609
|
+
self.save_request_log(request_data, error_data)
|
610
|
+
|
611
|
+
# 更新统计
|
612
|
+
request_data.update(error_data)
|
613
|
+
self.update_statistics(request_data)
|
614
|
+
|
615
|
+
# 重新抛出异常
|
616
|
+
raise e
|
617
|
+
|
618
|
+
return wrapper
|
619
|
+
|
620
|
+
def get_statistics_summary(self, days: int = 7) -> Dict[str, Any]:
|
621
|
+
"""获取统计摘要"""
|
622
|
+
try:
|
623
|
+
connection = self.pool.connection()
|
624
|
+
try:
|
625
|
+
with connection.cursor() as cursor:
|
626
|
+
end_date = datetime.now().date()
|
627
|
+
start_date = end_date - timedelta(days=days)
|
628
|
+
|
629
|
+
# 总体统计
|
630
|
+
cursor.execute("""
|
631
|
+
SELECT
|
632
|
+
SUM(total_requests) as total_requests,
|
633
|
+
SUM(success_requests) as success_requests,
|
634
|
+
SUM(error_requests) as error_requests,
|
635
|
+
AVG(avg_response_time) as avg_response_time,
|
636
|
+
COUNT(DISTINCT endpoint) as unique_endpoints
|
637
|
+
FROM api_access_statistics
|
638
|
+
WHERE date BETWEEN %s AND %s
|
639
|
+
""", (start_date, end_date))
|
640
|
+
|
641
|
+
summary = cursor.fetchone() or {}
|
642
|
+
|
643
|
+
# 热门端点
|
644
|
+
cursor.execute("""
|
645
|
+
SELECT endpoint, SUM(total_requests) as requests
|
646
|
+
FROM api_access_statistics
|
647
|
+
WHERE date BETWEEN %s AND %s
|
648
|
+
GROUP BY endpoint
|
649
|
+
ORDER BY requests DESC
|
650
|
+
LIMIT 10
|
651
|
+
""", (start_date, end_date))
|
652
|
+
|
653
|
+
top_endpoints = cursor.fetchall()
|
654
|
+
|
655
|
+
# 活跃IP统计
|
656
|
+
cursor.execute("""
|
657
|
+
SELECT COUNT(DISTINCT ip_address) as unique_ips,
|
658
|
+
SUM(total_requests) as total_ip_requests
|
659
|
+
FROM ip_access_statistics
|
660
|
+
WHERE date BETWEEN %s AND %s
|
661
|
+
""", (start_date, end_date))
|
662
|
+
|
663
|
+
ip_stats = cursor.fetchone() or {}
|
664
|
+
|
665
|
+
return {
|
666
|
+
'period': f'{start_date} to {end_date}',
|
667
|
+
'summary': summary,
|
668
|
+
'top_endpoints': top_endpoints,
|
669
|
+
'ip_statistics': ip_stats
|
670
|
+
}
|
671
|
+
finally:
|
672
|
+
connection.close()
|
673
|
+
|
674
|
+
except Exception as e:
|
675
|
+
return {'error': str(e)}
|
676
|
+
|
677
|
+
|
678
|
+
# 全局监控实例
|
679
|
+
route_monitor = RouteMonitor()
|
680
|
+
|
681
|
+
# 导出监控装饰器
|
682
|
+
monitor_request = route_monitor.monitor_request
|
683
|
+
|
684
|
+
# 导出其他有用的函数
|
685
|
+
def get_request_id():
|
686
|
+
"""获取当前请求ID"""
|
687
|
+
return getattr(g, 'request_id', None)
|
688
|
+
|
689
|
+
def get_statistics_summary(days: int = 7):
|
690
|
+
"""获取统计摘要"""
|
691
|
+
return route_monitor.get_statistics_summary(days)
|