mdbq 4.2.10__py3-none-any.whl → 4.2.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- mdbq/__version__.py +1 -1
- mdbq/auth/crypto.py +41 -7
- mdbq/redis/redis_cache.py +5 -0
- mdbq/route/monitor.py +593 -554
- {mdbq-4.2.10.dist-info → mdbq-4.2.12.dist-info}/METADATA +1 -1
- {mdbq-4.2.10.dist-info → mdbq-4.2.12.dist-info}/RECORD +8 -8
- {mdbq-4.2.10.dist-info → mdbq-4.2.12.dist-info}/WHEEL +0 -0
- {mdbq-4.2.10.dist-info → mdbq-4.2.12.dist-info}/top_level.txt +0 -0
mdbq/route/monitor.py
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
API 监控装饰器模块
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
1. 监控所有路由接口的访问请求
|
|
6
|
-
2. 记录详细的请求信息(IP、设备、请求头、请求体等)
|
|
7
|
-
3. 提供统计分析功能
|
|
8
|
-
4. 异常处理和数据清理
|
|
4
|
+
高性能、轻量级的 API 访问监控系统,专注于核心监控指标的收集和分析。
|
|
9
5
|
|
|
6
|
+
核心特性:
|
|
7
|
+
1. 请求监控:记录接口访问的核心信息(耗时、状态、ip 等)
|
|
8
|
+
2. 统计分析:提供按时间维度的访问统计和性能分析
|
|
9
|
+
3. 高性能:精简字段设计,优化索引策略,最小化性能影响
|
|
10
|
+
4. 安全性:自动脱敏敏感信息,支持 ip 风险评估
|
|
11
|
+
5. 自动清理:支持历史数据自动归档和清理
|
|
10
12
|
"""
|
|
11
13
|
|
|
12
14
|
import os
|
|
@@ -14,53 +16,65 @@ import json
|
|
|
14
16
|
import time
|
|
15
17
|
import uuid
|
|
16
18
|
import pymysql
|
|
17
|
-
import hashlib
|
|
18
19
|
import functools
|
|
19
20
|
from datetime import datetime, timedelta
|
|
20
|
-
from typing import Dict, Any
|
|
21
|
-
from urllib.parse import urlparse
|
|
21
|
+
from typing import Dict, Any, Optional
|
|
22
22
|
from dbutils.pooled_db import PooledDB # type: ignore
|
|
23
|
-
from mdbq.myconf import myconf # type: ignore
|
|
24
|
-
# from mdbq.log import mylogger
|
|
25
23
|
from flask import request, g
|
|
26
|
-
import re
|
|
27
|
-
import ipaddress
|
|
28
24
|
|
|
29
|
-
parser = myconf.ConfigParser()
|
|
30
|
-
host, port, username, password = parser.get_section_values(
|
|
31
|
-
file_path=os.path.join(os.path.expanduser("~"), 'spd.txt'),
|
|
32
|
-
section='mysql',
|
|
33
|
-
keys=['host', 'port', 'username', 'password'],
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
# logger = mylogger.MyLogger(
|
|
37
|
-
# logging_mode='file',
|
|
38
|
-
# log_level='info',
|
|
39
|
-
# log_format='json',
|
|
40
|
-
# max_log_size=50,
|
|
41
|
-
# backup_count=5,
|
|
42
|
-
# enable_async=False, # 是否启用异步日志
|
|
43
|
-
# sample_rate=1, # 采样DEBUG/INFO日志
|
|
44
|
-
# sensitive_fields=[], # 敏感字段过滤
|
|
45
|
-
# enable_metrics=False, # 是否启用性能指标
|
|
46
|
-
# )
|
|
47
25
|
|
|
48
26
|
|
|
49
27
|
class RouteMonitor:
|
|
50
|
-
"""
|
|
28
|
+
"""
|
|
29
|
+
路由监控核心类
|
|
30
|
+
|
|
31
|
+
负责 API 请求的监控、日志记录和统计分析。
|
|
32
|
+
采用精简设计,专注于核心监控指标,最大程度降低对业务性能的影响。
|
|
33
|
+
|
|
34
|
+
Attributes:
|
|
35
|
+
database (str): 监控数据库名称,默认为 'api监控系统'
|
|
36
|
+
pool (PooledDB): 数据库连接池
|
|
37
|
+
|
|
38
|
+
核心方法:
|
|
39
|
+
- api_monitor: 装饰器,用于监控 API 接口
|
|
40
|
+
- get_statistics_summary: 获取统计摘要数据
|
|
41
|
+
- cleanup_old_data: 清理历史数据
|
|
42
|
+
"""
|
|
51
43
|
|
|
52
|
-
def __init__(self, database='
|
|
53
|
-
"""
|
|
44
|
+
def __init__(self, database: str = 'api监控系统', pool = None):
|
|
45
|
+
"""
|
|
46
|
+
初始化监控系统
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
database: 数据库名称,默认为 'api监控系统'
|
|
50
|
+
pool: 数据库连接池对象,如果不传则使用默认配置创建
|
|
51
|
+
"""
|
|
54
52
|
self.database = database
|
|
55
|
-
self.
|
|
53
|
+
self.pool = pool
|
|
54
|
+
if self.pool is None:
|
|
55
|
+
self.init_database_pool()
|
|
56
56
|
self.init_database_tables()
|
|
57
57
|
|
|
58
58
|
def init_database_pool(self):
|
|
59
|
-
"""
|
|
59
|
+
"""
|
|
60
|
+
初始化数据库连接池
|
|
61
|
+
|
|
62
|
+
配置说明:
|
|
63
|
+
- 最大连接数:2(监控系统不需要大量连接)
|
|
64
|
+
- 编码:utf8mb4(支持中文和 emoji)
|
|
65
|
+
- 自动重连:开启
|
|
66
|
+
"""
|
|
67
|
+
from mdbq.myconf import myconf # type: ignore
|
|
68
|
+
parser = myconf.ConfigParser()
|
|
69
|
+
host, port, username, password = parser.get_section_values(
|
|
70
|
+
file_path=os.path.join(os.path.expanduser("~"), 'spd.txt'),
|
|
71
|
+
section='mysql',
|
|
72
|
+
keys=['host', 'port', 'username', 'password'],
|
|
73
|
+
)
|
|
60
74
|
try:
|
|
61
75
|
self.pool = PooledDB(
|
|
62
76
|
creator=pymysql,
|
|
63
|
-
maxconnections=2,
|
|
77
|
+
maxconnections=2,
|
|
64
78
|
mincached=1,
|
|
65
79
|
maxcached=2,
|
|
66
80
|
blocking=True,
|
|
@@ -68,174 +82,140 @@ class RouteMonitor:
|
|
|
68
82
|
port=int(port),
|
|
69
83
|
user=username,
|
|
70
84
|
password=password,
|
|
71
|
-
ping=1,
|
|
85
|
+
ping=1, # 自动重连
|
|
72
86
|
charset='utf8mb4',
|
|
73
87
|
cursorclass=pymysql.cursors.DictCursor
|
|
74
88
|
)
|
|
75
89
|
|
|
76
|
-
#
|
|
90
|
+
# 创建数据库
|
|
77
91
|
connection = self.pool.connection()
|
|
78
92
|
try:
|
|
79
93
|
with connection.cursor() as cursor:
|
|
80
|
-
cursor.execute(
|
|
94
|
+
cursor.execute(
|
|
95
|
+
f"CREATE DATABASE IF NOT EXISTS `{self.database}` "
|
|
96
|
+
f"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
|
97
|
+
)
|
|
81
98
|
cursor.execute(f"USE `{self.database}`")
|
|
82
99
|
finally:
|
|
83
100
|
connection.close()
|
|
84
101
|
|
|
85
102
|
except Exception as e:
|
|
86
|
-
# logger.error("数据库连接池初始化失败", {
|
|
87
|
-
# "错误信息": str(e),
|
|
88
|
-
# "数据库": self.database
|
|
89
|
-
# })
|
|
90
103
|
raise
|
|
91
104
|
|
|
92
105
|
def ensure_database_context(self, cursor):
|
|
93
|
-
"""
|
|
106
|
+
"""
|
|
107
|
+
确保游标处于正确的数据库上下文中
|
|
108
|
+
|
|
109
|
+
Args:
|
|
110
|
+
cursor: 数据库游标对象
|
|
111
|
+
"""
|
|
94
112
|
try:
|
|
95
113
|
cursor.execute(f"USE `{self.database}`")
|
|
96
|
-
except Exception
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.database}` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci")
|
|
114
|
+
except Exception:
|
|
115
|
+
cursor.execute(
|
|
116
|
+
f"CREATE DATABASE IF NOT EXISTS `{self.database}` "
|
|
117
|
+
f"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
|
118
|
+
)
|
|
102
119
|
cursor.execute(f"USE `{self.database}`")
|
|
103
120
|
|
|
104
121
|
def init_database_tables(self):
|
|
105
|
-
"""
|
|
122
|
+
"""
|
|
123
|
+
初始化数据库表结构
|
|
124
|
+
|
|
125
|
+
创建三张核心表:
|
|
126
|
+
1. api_访问日志:记录每次 API 请求的详细信息
|
|
127
|
+
2. api_接口统计:按小时汇总的接口性能统计
|
|
128
|
+
3. api_ip记录:IP 维度的访问统计
|
|
129
|
+
"""
|
|
106
130
|
try:
|
|
107
131
|
connection = self.pool.connection()
|
|
108
132
|
try:
|
|
109
133
|
with connection.cursor() as cursor:
|
|
110
|
-
# 确保使用正确的数据库上下文
|
|
111
134
|
self.ensure_database_context(cursor)
|
|
112
135
|
|
|
113
|
-
#
|
|
114
|
-
|
|
115
|
-
CREATE TABLE IF NOT EXISTS `api_request_logs` (
|
|
116
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
|
117
|
-
`request_id` VARCHAR(128) NOT NULL COMMENT '请求唯一标识',
|
|
118
|
-
`timestamp` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '请求时间(精确到毫秒)',
|
|
119
|
-
`method` VARCHAR(10) NOT NULL COMMENT 'HTTP方法',
|
|
120
|
-
`endpoint` VARCHAR(500) NOT NULL COMMENT '请求端点',
|
|
121
|
-
`full_url` TEXT COMMENT '完整URL',
|
|
122
|
-
`client_ip` VARCHAR(45) NOT NULL COMMENT '客户端IP地址',
|
|
123
|
-
`real_ip` VARCHAR(45) COMMENT '真实IP地址',
|
|
124
|
-
`forwarded_ips` TEXT COMMENT '转发IP链',
|
|
125
|
-
`user_agent` TEXT COMMENT '用户代理',
|
|
126
|
-
`referer` VARCHAR(1000) COMMENT '来源页面',
|
|
127
|
-
`host` VARCHAR(255) COMMENT '请求主机',
|
|
128
|
-
`scheme` VARCHAR(10) COMMENT '协议类型',
|
|
129
|
-
`port` INT COMMENT '端口号',
|
|
130
|
-
`request_headers` JSON COMMENT '请求头信息',
|
|
131
|
-
`request_params` JSON COMMENT '请求参数',
|
|
132
|
-
`request_body` LONGTEXT COMMENT '请求体内容',
|
|
133
|
-
`request_size` INT DEFAULT 0 COMMENT '请求大小(字节)',
|
|
134
|
-
`response_status` INT COMMENT '响应状态码',
|
|
135
|
-
`response_size` INT COMMENT '响应大小(字节)',
|
|
136
|
-
`process_time` DECIMAL(10,3) COMMENT '处理时间(毫秒)',
|
|
137
|
-
`session_id` VARCHAR(128) COMMENT '会话ID',
|
|
138
|
-
`user_id` VARCHAR(64) COMMENT '用户ID',
|
|
139
|
-
`auth_token` TEXT COMMENT '认证令牌(脱敏)',
|
|
140
|
-
`device_fingerprint` VARCHAR(256) COMMENT '设备指纹',
|
|
141
|
-
`device_info` JSON COMMENT '设备信息',
|
|
142
|
-
`geo_country` VARCHAR(100) COMMENT '地理位置-国家',
|
|
143
|
-
`geo_region` VARCHAR(100) COMMENT '地理位置-地区',
|
|
144
|
-
`geo_city` VARCHAR(100) COMMENT '地理位置-城市',
|
|
145
|
-
`is_bot` BOOLEAN DEFAULT FALSE COMMENT '是否为机器人',
|
|
146
|
-
`is_mobile` BOOLEAN DEFAULT FALSE COMMENT '是否为移动设备',
|
|
147
|
-
`browser_name` VARCHAR(50) COMMENT '浏览器名称',
|
|
148
|
-
`browser_version` VARCHAR(20) COMMENT '浏览器版本',
|
|
149
|
-
`os_name` VARCHAR(50) COMMENT '操作系统名称',
|
|
150
|
-
`os_version` VARCHAR(20) COMMENT '操作系统版本',
|
|
151
|
-
`error_message` TEXT COMMENT '错误信息',
|
|
152
|
-
`business_data` JSON COMMENT '业务数据',
|
|
153
|
-
`tags` JSON COMMENT '标签信息',
|
|
154
|
-
UNIQUE KEY `uk_request_id` (`request_id`),
|
|
155
|
-
INDEX `idx_timestamp` (`timestamp`),
|
|
156
|
-
INDEX `idx_endpoint` (`endpoint`(191)),
|
|
157
|
-
INDEX `idx_client_ip` (`client_ip`),
|
|
158
|
-
INDEX `idx_user_id` (`user_id`),
|
|
159
|
-
INDEX `idx_status` (`response_status`),
|
|
160
|
-
INDEX `idx_method_endpoint` (`method`, `endpoint`(191)),
|
|
161
|
-
INDEX `idx_timestamp_endpoint` (`timestamp`, `endpoint`(191))
|
|
162
|
-
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
163
|
-
COMMENT='API请求详细日志表';
|
|
164
|
-
""")
|
|
165
|
-
# 创建访问统计汇总表
|
|
136
|
+
# ==================== 表 1:访问日志表 ====================
|
|
137
|
+
# 设计原则:只保留核心监控字段,移除冗余信息
|
|
166
138
|
cursor.execute("""
|
|
167
|
-
CREATE TABLE IF NOT EXISTS `
|
|
168
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
169
|
-
`
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
`
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
|
|
188
|
-
INDEX `
|
|
189
|
-
INDEX `
|
|
190
|
-
INDEX `idx_date_endpoint` (`date`, `endpoint`(191))
|
|
139
|
+
CREATE TABLE IF NOT EXISTS `api_访问日志` (
|
|
140
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
141
|
+
`请求id` VARCHAR(64) NOT NULL COMMENT '请求唯一标识(用于追踪)',
|
|
142
|
+
`请求时间` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '请求时间,精确到毫秒',
|
|
143
|
+
`请求方法` VARCHAR(10) NOT NULL COMMENT 'HTTP 方法(GET/POST/PUT/DELETE等)',
|
|
144
|
+
`接口路径` VARCHAR(500) NOT NULL COMMENT 'API 接口路径',
|
|
145
|
+
`客户端ip` VARCHAR(45) NOT NULL COMMENT '客户端 ip 地址(支持 IPv6)',
|
|
146
|
+
`响应状态码` SMALLINT COMMENT 'HTTP 响应状态码',
|
|
147
|
+
`响应耗时` DECIMAL(10,3) COMMENT '请求处理耗时(毫秒)',
|
|
148
|
+
`用户标识` VARCHAR(64) COMMENT '用户id或标识(如有)',
|
|
149
|
+
`用户代理` VARCHAR(500) COMMENT '浏览器 User-Agent(精简版)',
|
|
150
|
+
`请求参数` TEXT COMMENT '请求参数(JSON格式,可选记录)',
|
|
151
|
+
`错误信息` TEXT COMMENT '错误信息(仅失败请求记录)',
|
|
152
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
153
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
154
|
+
|
|
155
|
+
UNIQUE KEY `uk_请求id` (`请求id`),
|
|
156
|
+
INDEX `idx_请求时间` (`请求时间`),
|
|
157
|
+
INDEX `idx_接口路径` (`接口路径`(191)),
|
|
158
|
+
INDEX `idx_客户端ip` (`客户端ip`),
|
|
159
|
+
INDEX `idx_响应状态码` (`响应状态码`),
|
|
160
|
+
INDEX `idx_用户标识` (`用户标识`),
|
|
161
|
+
INDEX `idx_时间_接口` (`请求时间`, `接口路径`(191))
|
|
191
162
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
192
|
-
COMMENT='API
|
|
163
|
+
COMMENT='API 访问日志表 - 记录每次请求的核心信息'
|
|
164
|
+
ROW_FORMAT=COMPRESSED;
|
|
193
165
|
""")
|
|
194
166
|
|
|
195
|
-
#
|
|
167
|
+
# ==================== 表 2:接口统计表 ====================
|
|
168
|
+
# 设计原则:按小时维度汇总,用于性能分析和趋势监控
|
|
196
169
|
cursor.execute("""
|
|
197
|
-
CREATE TABLE IF NOT EXISTS `
|
|
198
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
INDEX `
|
|
216
|
-
INDEX `
|
|
217
|
-
INDEX `idx_suspicious` (`is_suspicious`),
|
|
218
|
-
INDEX `idx_risk_score` (`risk_score`)
|
|
170
|
+
CREATE TABLE IF NOT EXISTS `api_接口统计` (
|
|
171
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
172
|
+
`统计日期` DATE NOT NULL COMMENT '统计日期',
|
|
173
|
+
`统计小时` TINYINT NOT NULL COMMENT '统计小时(0-23)',
|
|
174
|
+
`接口路径` VARCHAR(500) NOT NULL COMMENT 'API 接口路径',
|
|
175
|
+
`请求方法` VARCHAR(10) NOT NULL COMMENT 'HTTP 请求方法',
|
|
176
|
+
`请求总数` INT UNSIGNED DEFAULT 0 COMMENT '总请求次数',
|
|
177
|
+
`成功次数` INT UNSIGNED DEFAULT 0 COMMENT '成功响应次数(状态码 < 400)',
|
|
178
|
+
`失败次数` INT UNSIGNED DEFAULT 0 COMMENT '失败响应次数(状态码 >= 400)',
|
|
179
|
+
`平均耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应耗时(毫秒)',
|
|
180
|
+
`最大耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '最大响应耗时(毫秒)',
|
|
181
|
+
`最小耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '最小响应耗时(毫秒)',
|
|
182
|
+
`独立ip数` INT UNSIGNED DEFAULT 0 COMMENT '访问的独立 ip 数量',
|
|
183
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
184
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
185
|
+
|
|
186
|
+
UNIQUE KEY `uk_日期_小时_接口_方法` (`统计日期`, `统计小时`, `接口路径`(191), `请求方法`),
|
|
187
|
+
INDEX `idx_统计日期` (`统计日期`),
|
|
188
|
+
INDEX `idx_接口路径` (`接口路径`(191)),
|
|
189
|
+
INDEX `idx_日期_接口` (`统计日期`, `接口路径`(191))
|
|
219
190
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
220
|
-
COMMENT='
|
|
191
|
+
COMMENT='API 接口统计表 - 按小时汇总的接口性能数据';
|
|
221
192
|
""")
|
|
222
|
-
|
|
193
|
+
|
|
194
|
+
# ==================== 表 3:IP 访问记录表 ====================
|
|
195
|
+
# 设计原则:按日期汇总 IP 访问情况,用于安全分析和流量监控
|
|
223
196
|
cursor.execute("""
|
|
224
|
-
CREATE TABLE IF NOT EXISTS `
|
|
225
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
226
|
-
|
|
227
|
-
`
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
197
|
+
CREATE TABLE IF NOT EXISTS `api_ip记录` (
|
|
198
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
199
|
+
`统计日期` DATE NOT NULL COMMENT '统计日期',
|
|
200
|
+
`客户端ip` VARCHAR(45) NOT NULL COMMENT '客户端 ip 地址',
|
|
201
|
+
`请求总数` INT UNSIGNED DEFAULT 0 COMMENT '该 ip 当日总请求数',
|
|
202
|
+
`成功次数` INT UNSIGNED DEFAULT 0 COMMENT '成功请求次数',
|
|
203
|
+
`失败次数` INT UNSIGNED DEFAULT 0 COMMENT '失败请求次数',
|
|
204
|
+
`平均耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应耗时(毫秒)',
|
|
205
|
+
`首次访问` DATETIME COMMENT '首次访问时间',
|
|
206
|
+
`最后访问` DATETIME COMMENT '最后访问时间',
|
|
207
|
+
`访问接口数` INT UNSIGNED DEFAULT 0 COMMENT '访问的不同接口数量',
|
|
208
|
+
`风险评分` TINYINT UNSIGNED DEFAULT 0 COMMENT '风险评分(0-100,用于识别异常流量)',
|
|
209
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
210
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
211
|
+
|
|
212
|
+
UNIQUE KEY `uk_日期_ip` (`统计日期`, `客户端ip`),
|
|
213
|
+
INDEX `idx_统计日期` (`统计日期`),
|
|
214
|
+
INDEX `idx_客户端ip` (`客户端ip`),
|
|
215
|
+
INDEX `idx_风险评分` (`风险评分`),
|
|
216
|
+
INDEX `idx_请求总数` (`请求总数`)
|
|
237
217
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
238
|
-
COMMENT='
|
|
218
|
+
COMMENT='API ip 访问记录表 - ip 维度的访问统计';
|
|
239
219
|
""")
|
|
240
220
|
connection.commit()
|
|
241
221
|
|
|
@@ -243,287 +223,158 @@ class RouteMonitor:
|
|
|
243
223
|
connection.close()
|
|
244
224
|
|
|
245
225
|
except Exception as e:
|
|
246
|
-
#
|
|
247
|
-
# "错误信息": str(e),
|
|
248
|
-
# "错误类型": type(e).__name__,
|
|
249
|
-
# "数据库": self.database,
|
|
250
|
-
# "影响": "监控系统可能无法正常工作"
|
|
251
|
-
# })
|
|
252
|
-
# 静默处理初始化错误,避免影响主应用
|
|
226
|
+
# 静默处理初始化错误,避免影响主应用启动
|
|
253
227
|
pass
|
|
254
228
|
|
|
255
|
-
|
|
256
|
-
"""生成唯一的请求ID"""
|
|
257
|
-
timestamp = str(int(time.time() * 1000)) # 毫秒时间戳
|
|
258
|
-
random_part = uuid.uuid4().hex[:8]
|
|
259
|
-
return f"req_{timestamp}_{random_part}"
|
|
229
|
+
# ==================== 辅助方法 ====================
|
|
260
230
|
|
|
261
|
-
def
|
|
262
|
-
"""
|
|
263
|
-
|
|
264
|
-
'is_mobile': False,
|
|
265
|
-
'is_bot': False,
|
|
266
|
-
'browser_name': 'Unknown',
|
|
267
|
-
'browser_version': 'Unknown',
|
|
268
|
-
'os_name': 'Unknown',
|
|
269
|
-
'os_version': 'Unknown'
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
if not user_agent:
|
|
273
|
-
return device_info
|
|
274
|
-
|
|
275
|
-
user_agent_lower = user_agent.lower()
|
|
276
|
-
|
|
277
|
-
# 检测移动设备
|
|
278
|
-
mobile_keywords = ['mobile', 'android', 'iphone', 'ipad', 'ipod', 'windows phone']
|
|
279
|
-
device_info['is_mobile'] = any(keyword in user_agent_lower for keyword in mobile_keywords)
|
|
280
|
-
|
|
281
|
-
# 检测机器人
|
|
282
|
-
bot_keywords = ['bot', 'crawler', 'spider', 'scraper', 'curl', 'wget', 'python-requests']
|
|
283
|
-
device_info['is_bot'] = any(keyword in user_agent_lower for keyword in bot_keywords)
|
|
284
|
-
|
|
285
|
-
# 浏览器检测
|
|
286
|
-
browsers = [
|
|
287
|
-
('chrome', r'chrome/(\d+)'),
|
|
288
|
-
('firefox', r'firefox/(\d+)'),
|
|
289
|
-
('safari', r'safari/(\d+)'),
|
|
290
|
-
('edge', r'edge/(\d+)'),
|
|
291
|
-
('opera', r'opera/(\d+)')
|
|
292
|
-
]
|
|
293
|
-
|
|
294
|
-
for browser, pattern in browsers:
|
|
295
|
-
match = re.search(pattern, user_agent_lower)
|
|
296
|
-
if match:
|
|
297
|
-
device_info['browser_name'] = browser.title()
|
|
298
|
-
device_info['browser_version'] = match.group(1)
|
|
299
|
-
break
|
|
300
|
-
|
|
301
|
-
# 操作系统检测
|
|
302
|
-
os_patterns = [
|
|
303
|
-
('Windows', r'windows nt (\d+\.\d+)'),
|
|
304
|
-
('macOS', r'mac os x (\d+_\d+)'),
|
|
305
|
-
('Linux', r'linux'),
|
|
306
|
-
('Android', r'android (\d+)'),
|
|
307
|
-
('iOS', r'os (\d+_\d+)')
|
|
308
|
-
]
|
|
309
|
-
|
|
310
|
-
for os_name, pattern in os_patterns:
|
|
311
|
-
match = re.search(pattern, user_agent_lower)
|
|
312
|
-
if match:
|
|
313
|
-
device_info['os_name'] = os_name
|
|
314
|
-
if len(match.groups()) > 0:
|
|
315
|
-
device_info['os_version'] = match.group(1).replace('_', '.')
|
|
316
|
-
break
|
|
231
|
+
def generate_request_id(self) -> str:
|
|
232
|
+
"""
|
|
233
|
+
生成唯一的请求 ID
|
|
317
234
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
def generate_device_fingerprint(self, request_data: Dict) -> str:
|
|
321
|
-
"""生成设备指纹"""
|
|
322
|
-
fingerprint_data = {
|
|
323
|
-
'user_agent': request_data.get('user_agent', ''),
|
|
324
|
-
'accept_language': request_data.get('request_headers', {}).get('Accept-Language', ''),
|
|
325
|
-
'accept_encoding': request_data.get('request_headers', {}).get('Accept-Encoding', ''),
|
|
326
|
-
'connection': request_data.get('request_headers', {}).get('Connection', ''),
|
|
327
|
-
}
|
|
235
|
+
格式:req_{时间戳}_{随机字符串}
|
|
236
|
+
示例:req_1697654321123_a1b2c3d4
|
|
328
237
|
|
|
329
|
-
|
|
330
|
-
|
|
238
|
+
Returns:
|
|
239
|
+
str: 请求唯一标识符
|
|
240
|
+
"""
|
|
241
|
+
timestamp = str(int(time.time() * 1000))
|
|
242
|
+
random_part = uuid.uuid4().hex[:8]
|
|
243
|
+
return f"req_{timestamp}_{random_part}"
|
|
331
244
|
|
|
332
|
-
def
|
|
333
|
-
"""
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
sanitized = data
|
|
346
|
-
for pattern, replacement in sensitive_patterns:
|
|
347
|
-
sanitized = re.sub(pattern, replacement, sanitized, flags=re.IGNORECASE)
|
|
245
|
+
def get_real_ip(self, request) -> str:
|
|
246
|
+
"""
|
|
247
|
+
获取真实客户端 IP 地址
|
|
248
|
+
|
|
249
|
+
优先级顺序:
|
|
250
|
+
1. X-Forwarded-For(代理服务器传递的原始IP)
|
|
251
|
+
2. X-Real-IP(Nginx 等反向代理设置)
|
|
252
|
+
3. CF-Connecting-IP(Cloudflare CDN)
|
|
253
|
+
4. request.remote_addr(直连IP)
|
|
254
|
+
|
|
255
|
+
Args:
|
|
256
|
+
request: Flask request 对象
|
|
348
257
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
return sanitized
|
|
354
|
-
|
|
355
|
-
elif isinstance(data, dict):
|
|
356
|
-
sanitized = {}
|
|
357
|
-
for key, value in data.items():
|
|
358
|
-
if key.lower() in ['password', 'token', 'key', 'secret']:
|
|
359
|
-
sanitized[key] = '[REDACTED]'
|
|
360
|
-
else:
|
|
361
|
-
sanitized[key] = self.sanitize_data(value, max_length)
|
|
362
|
-
return sanitized
|
|
363
|
-
|
|
364
|
-
elif isinstance(data, list):
|
|
365
|
-
return [self.sanitize_data(item, max_length) for item in data[:100]] # 限制列表长度
|
|
366
|
-
|
|
367
|
-
return data
|
|
368
|
-
|
|
369
|
-
def get_real_ip(self, request) -> tuple:
|
|
370
|
-
"""获取真实IP地址"""
|
|
371
|
-
# IP地址优先级顺序
|
|
258
|
+
Returns:
|
|
259
|
+
str: 客户端真实 IP 地址
|
|
260
|
+
"""
|
|
261
|
+
# IP 头优先级列表
|
|
372
262
|
ip_headers = [
|
|
373
263
|
'X-Forwarded-For',
|
|
374
264
|
'X-Real-IP',
|
|
375
|
-
'CF-Connecting-IP',
|
|
265
|
+
'CF-Connecting-IP',
|
|
376
266
|
'X-Client-IP',
|
|
377
|
-
'X-Forwarded',
|
|
378
|
-
'Forwarded-For',
|
|
379
|
-
'Forwarded'
|
|
380
267
|
]
|
|
381
268
|
|
|
382
|
-
|
|
383
|
-
real_ip = request.remote_addr
|
|
384
|
-
|
|
269
|
+
# 尝试从请求头获取 IP
|
|
385
270
|
for header in ip_headers:
|
|
386
271
|
header_value = request.headers.get(header)
|
|
387
272
|
if header_value:
|
|
388
|
-
#
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
# 取第一个有效的IP作为真实IP
|
|
393
|
-
for ip in ips:
|
|
394
|
-
if self.is_valid_ip(ip) and not self.is_private_ip(ip):
|
|
395
|
-
real_ip = ip
|
|
396
|
-
break
|
|
397
|
-
|
|
398
|
-
if real_ip != request.remote_addr:
|
|
399
|
-
break
|
|
273
|
+
# X-Forwarded-For 可能包含多个 IP,取第一个
|
|
274
|
+
ip = header_value.split(',')[0].strip()
|
|
275
|
+
if ip:
|
|
276
|
+
return ip
|
|
400
277
|
|
|
401
|
-
|
|
278
|
+
# 如果没有代理头,返回直连 IP
|
|
279
|
+
return request.remote_addr or 'unknown'
|
|
402
280
|
|
|
403
|
-
def
|
|
404
|
-
"""
|
|
281
|
+
def sanitize_params(self, params: Dict[str, Any]) -> Optional[str]:
|
|
282
|
+
"""
|
|
283
|
+
清理和脱敏请求参数
|
|
284
|
+
|
|
285
|
+
自动移除敏感字段(如 password、token 等)
|
|
286
|
+
|
|
287
|
+
Args:
|
|
288
|
+
params: 请求参数字典
|
|
289
|
+
|
|
290
|
+
Returns:
|
|
291
|
+
str: JSON 格式的参数字符串(已脱敏),或 None
|
|
292
|
+
"""
|
|
293
|
+
if not params:
|
|
294
|
+
return None
|
|
295
|
+
|
|
296
|
+
# 敏感字段列表
|
|
297
|
+
sensitive_keys = ['password', 'passwd', 'pwd', 'token', 'secret', 'key', 'api_key', 'apikey']
|
|
298
|
+
|
|
299
|
+
# 创建副本并脱敏
|
|
300
|
+
sanitized = {}
|
|
301
|
+
for key, value in params.items():
|
|
302
|
+
if any(sensitive in key.lower() for sensitive in sensitive_keys):
|
|
303
|
+
sanitized[key] = '***'
|
|
304
|
+
else:
|
|
305
|
+
# 截断过长的值
|
|
306
|
+
if isinstance(value, str) and len(value) > 500:
|
|
307
|
+
sanitized[key] = value[:500] + '...'
|
|
308
|
+
else:
|
|
309
|
+
sanitized[key] = value
|
|
310
|
+
|
|
405
311
|
try:
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
return False
|
|
312
|
+
return json.dumps(sanitized, ensure_ascii=False)
|
|
313
|
+
except Exception:
|
|
314
|
+
return None
|
|
410
315
|
|
|
411
|
-
|
|
412
|
-
"""检查是否为私有IP"""
|
|
413
|
-
try:
|
|
414
|
-
return ipaddress.ip_address(ip).is_private
|
|
415
|
-
except ValueError:
|
|
416
|
-
return False
|
|
316
|
+
# ==================== 核心数据收集 ====================
|
|
417
317
|
|
|
418
318
|
def collect_request_data(self, request) -> Dict[str, Any]:
|
|
419
|
-
"""
|
|
420
|
-
|
|
421
|
-
request_id = self.generate_request_id()
|
|
422
|
-
|
|
423
|
-
# 设置请求ID到全局变量中,供后续使用
|
|
424
|
-
g.request_id = request_id
|
|
425
|
-
# 获取真实IP
|
|
426
|
-
real_ip, forwarded_ips = self.get_real_ip(request)
|
|
427
|
-
|
|
428
|
-
# 获取请求头信息
|
|
429
|
-
headers = dict(request.headers)
|
|
430
|
-
sanitized_headers = self.sanitize_data(headers)
|
|
431
|
-
|
|
432
|
-
# 获取请求参数
|
|
433
|
-
request_params = {}
|
|
434
|
-
if request.args:
|
|
435
|
-
request_params.update(dict(request.args))
|
|
319
|
+
"""
|
|
320
|
+
收集请求核心数据
|
|
436
321
|
|
|
437
|
-
|
|
438
|
-
request_body = None
|
|
439
|
-
request_size = 0
|
|
322
|
+
仅收集必要的监控信息,避免过度记录造成性能和存储压力。
|
|
440
323
|
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
if body_data:
|
|
450
|
-
try:
|
|
451
|
-
request_body = body_data.decode('utf-8')
|
|
452
|
-
except UnicodeDecodeError:
|
|
453
|
-
request_body = f"[BINARY_DATA:{len(body_data)}_bytes]"
|
|
454
|
-
|
|
455
|
-
if request_body:
|
|
456
|
-
request_size = len(str(request_body).encode('utf-8'))
|
|
457
|
-
except Exception as e:
|
|
458
|
-
request_body = "[ERROR_READING_BODY]"
|
|
459
|
-
# logger.warning("读取请求体失败", {
|
|
460
|
-
# "请求ID": request_id,
|
|
461
|
-
# "错误": str(e)
|
|
462
|
-
# })
|
|
324
|
+
Args:
|
|
325
|
+
request: Flask request 对象
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
dict: 包含请求核心信息的字典
|
|
329
|
+
"""
|
|
330
|
+
request_id = self.generate_request_id()
|
|
331
|
+
g.request_id = request_id # 保存到全局变量,供后续使用
|
|
463
332
|
|
|
464
|
-
#
|
|
465
|
-
|
|
466
|
-
sanitized_params = self.sanitize_data(request_params)
|
|
333
|
+
# 获取客户端 IP
|
|
334
|
+
client_ip = self.get_real_ip(request)
|
|
467
335
|
|
|
468
|
-
#
|
|
336
|
+
# 获取 User-Agent(截断过长的)
|
|
469
337
|
user_agent = request.headers.get('User-Agent', '')
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
338
|
+
if len(user_agent) > 500:
|
|
339
|
+
user_agent = user_agent[:500]
|
|
340
|
+
|
|
341
|
+
# 获取用户标识(如果有)
|
|
342
|
+
user_id = None
|
|
343
|
+
if hasattr(g, 'current_user_id'):
|
|
344
|
+
user_id = str(g.current_user_id)
|
|
345
|
+
elif hasattr(g, 'user_id'):
|
|
346
|
+
user_id = str(g.user_id)
|
|
347
|
+
|
|
348
|
+
# 收集请求参数(可选,仅在需要时记录)
|
|
349
|
+
request_params = None
|
|
350
|
+
if request.args:
|
|
351
|
+
request_params = self.sanitize_params(dict(request.args))
|
|
473
352
|
|
|
474
|
-
#
|
|
353
|
+
# 构建请求数据字典
|
|
475
354
|
request_data = {
|
|
476
|
-
'
|
|
477
|
-
'
|
|
478
|
-
'
|
|
479
|
-
'
|
|
480
|
-
'
|
|
481
|
-
'
|
|
482
|
-
'
|
|
483
|
-
'
|
|
484
|
-
'user_agent': user_agent,
|
|
485
|
-
'referer': request.headers.get('Referer'),
|
|
486
|
-
'host': request.headers.get('Host'),
|
|
487
|
-
'scheme': parsed_url.scheme,
|
|
488
|
-
'port': parsed_url.port,
|
|
489
|
-
'request_headers': json.dumps(sanitized_headers),
|
|
490
|
-
'request_params': json.dumps(sanitized_params) if sanitized_params else None,
|
|
491
|
-
'request_body': json.dumps(sanitized_body) if sanitized_body else None,
|
|
492
|
-
'request_size': request_size,
|
|
493
|
-
'session_id': request.cookies.get('session_id'),
|
|
494
|
-
'user_id': getattr(request, 'current_user', {}).get('id') if hasattr(request, 'current_user') else None,
|
|
495
|
-
'auth_token': self.mask_token(request.headers.get('Authorization')),
|
|
496
|
-
'device_fingerprint': self.generate_device_fingerprint({
|
|
497
|
-
'user_agent': user_agent,
|
|
498
|
-
'request_headers': sanitized_headers
|
|
499
|
-
}),
|
|
500
|
-
'device_info': json.dumps(device_info),
|
|
501
|
-
'is_bot': device_info['is_bot'],
|
|
502
|
-
'is_mobile': device_info['is_mobile'],
|
|
503
|
-
'browser_name': device_info['browser_name'],
|
|
504
|
-
'browser_version': device_info['browser_version'],
|
|
505
|
-
'os_name': device_info['os_name'],
|
|
506
|
-
'os_version': device_info['os_version'],
|
|
355
|
+
'请求id': request_id,
|
|
356
|
+
'请求时间': datetime.now(),
|
|
357
|
+
'请求方法': request.method,
|
|
358
|
+
'接口路径': request.endpoint or request.path,
|
|
359
|
+
'客户端ip': client_ip,
|
|
360
|
+
'用户标识': user_id,
|
|
361
|
+
'用户代理': user_agent,
|
|
362
|
+
'请求参数': request_params,
|
|
507
363
|
}
|
|
508
364
|
|
|
509
365
|
return request_data
|
|
510
366
|
|
|
511
|
-
|
|
512
|
-
"""脱敏处理令牌"""
|
|
513
|
-
if not token:
|
|
514
|
-
return None
|
|
515
|
-
|
|
516
|
-
if len(token) <= 8:
|
|
517
|
-
return '*' * len(token)
|
|
518
|
-
|
|
519
|
-
if len(token) > 255:
|
|
520
|
-
return token[:10] + '*' * (min(len(token), 10)) + token[-10:]
|
|
521
|
-
|
|
522
|
-
return token[:4] + '*' * (len(token) - 8) + token[-4:]
|
|
367
|
+
# ==================== 数据持久化 ====================
|
|
523
368
|
|
|
524
369
|
def save_request_log(self, request_data: Dict[str, Any], response_data: Dict[str, Any] = None):
|
|
525
|
-
"""
|
|
526
|
-
|
|
370
|
+
"""
|
|
371
|
+
保存请求日志到数据库
|
|
372
|
+
|
|
373
|
+
Args:
|
|
374
|
+
request_data: 请求数据字典
|
|
375
|
+
response_data: 响应数据字典(可选)
|
|
376
|
+
"""
|
|
377
|
+
request_id = request_data.get('请求id', 'unknown')
|
|
527
378
|
|
|
528
379
|
try:
|
|
529
380
|
# 合并响应数据
|
|
@@ -533,92 +384,119 @@ class RouteMonitor:
|
|
|
533
384
|
connection = self.pool.connection()
|
|
534
385
|
try:
|
|
535
386
|
with connection.cursor() as cursor:
|
|
536
|
-
# 确保使用正确的数据库上下文
|
|
537
387
|
self.ensure_database_context(cursor)
|
|
538
388
|
|
|
539
389
|
# 插入请求日志
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
390
|
+
sql = """
|
|
391
|
+
INSERT INTO `api_访问日志` (
|
|
392
|
+
`请求id`, `请求时间`, `请求方法`, `接口路径`, `客户端ip`,
|
|
393
|
+
`响应状态码`, `响应耗时`, `用户标识`, `用户代理`, `请求参数`, `错误信息`
|
|
394
|
+
) VALUES (
|
|
395
|
+
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
396
|
+
)
|
|
546
397
|
"""
|
|
547
398
|
|
|
548
|
-
cursor.execute(sql,
|
|
399
|
+
cursor.execute(sql, (
|
|
400
|
+
request_data.get('请求id'),
|
|
401
|
+
request_data.get('请求时间'),
|
|
402
|
+
request_data.get('请求方法'),
|
|
403
|
+
request_data.get('接口路径'),
|
|
404
|
+
request_data.get('客户端ip'),
|
|
405
|
+
request_data.get('响应状态码'),
|
|
406
|
+
request_data.get('响应耗时'),
|
|
407
|
+
request_data.get('用户标识'),
|
|
408
|
+
request_data.get('用户代理'),
|
|
409
|
+
request_data.get('请求参数'),
|
|
410
|
+
request_data.get('错误信息'),
|
|
411
|
+
))
|
|
412
|
+
|
|
549
413
|
connection.commit()
|
|
550
414
|
|
|
551
415
|
finally:
|
|
552
416
|
connection.close()
|
|
553
417
|
|
|
554
418
|
except Exception as e:
|
|
555
|
-
#
|
|
556
|
-
# "请求ID": request_id,
|
|
557
|
-
# "错误信息": str(e),
|
|
558
|
-
# "错误类型": type(e).__name__,
|
|
559
|
-
# "影响": "日志丢失,但不影响主业务"
|
|
560
|
-
# })
|
|
561
|
-
# 静默处理错误,不影响主业务
|
|
419
|
+
# 静默处理错误,避免影响主业务
|
|
562
420
|
pass
|
|
563
421
|
|
|
564
422
|
def update_statistics(self, request_data: Dict[str, Any]):
|
|
565
|
-
"""
|
|
566
|
-
|
|
567
|
-
# endpoint = request_data.get('endpoint', '')
|
|
568
|
-
# status_code = request_data.get('response_status', 500)
|
|
423
|
+
"""
|
|
424
|
+
更新统计数据
|
|
569
425
|
|
|
426
|
+
包括:
|
|
427
|
+
1. 接口统计:按小时汇总接口性能数据
|
|
428
|
+
2. IP 统计:按日期汇总 IP 访问数据
|
|
429
|
+
|
|
430
|
+
Args:
|
|
431
|
+
request_data: 包含请求和响应信息的字典
|
|
432
|
+
"""
|
|
570
433
|
try:
|
|
571
434
|
connection = self.pool.connection()
|
|
572
435
|
try:
|
|
573
436
|
with connection.cursor() as cursor:
|
|
574
|
-
# 确保使用正确的数据库上下文
|
|
575
437
|
self.ensure_database_context(cursor)
|
|
576
438
|
|
|
577
439
|
now = datetime.now()
|
|
578
440
|
date = now.date()
|
|
579
441
|
hour = now.hour
|
|
580
442
|
|
|
581
|
-
#
|
|
443
|
+
# 判断是否成功(状态码 < 400)
|
|
444
|
+
status_code = request_data.get('响应状态码', 500)
|
|
445
|
+
is_success = 1 if status_code < 400 else 0
|
|
446
|
+
is_error = 1 if status_code >= 400 else 0
|
|
447
|
+
response_time = request_data.get('响应耗时', 0)
|
|
448
|
+
|
|
449
|
+
# 1. 更新接口统计表
|
|
582
450
|
cursor.execute("""
|
|
583
|
-
INSERT INTO `
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
451
|
+
INSERT INTO `api_接口统计` (
|
|
452
|
+
`统计日期`, `统计小时`, `接口路径`, `请求方法`,
|
|
453
|
+
`请求总数`, `成功次数`, `失败次数`,
|
|
454
|
+
`平均耗时`, `最大耗时`, `最小耗时`
|
|
455
|
+
) VALUES (
|
|
456
|
+
%s, %s, %s, %s, 1, %s, %s, %s, %s, %s
|
|
457
|
+
) ON DUPLICATE KEY UPDATE
|
|
458
|
+
`请求总数` = `请求总数` + 1,
|
|
459
|
+
`成功次数` = `成功次数` + %s,
|
|
460
|
+
`失败次数` = `失败次数` + %s,
|
|
461
|
+
`平均耗时` = (
|
|
462
|
+
(`平均耗时` * (`请求总数` - 1) + %s) / `请求总数`
|
|
463
|
+
),
|
|
464
|
+
`最大耗时` = GREATEST(`最大耗时`, %s),
|
|
465
|
+
`最小耗时` = LEAST(`最小耗时`, %s)
|
|
595
466
|
""", (
|
|
596
467
|
date, hour,
|
|
597
|
-
request_data.get('
|
|
598
|
-
request_data.get('
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
468
|
+
request_data.get('接口路径', ''),
|
|
469
|
+
request_data.get('请求方法', ''),
|
|
470
|
+
is_success, is_error,
|
|
471
|
+
response_time, response_time, response_time,
|
|
472
|
+
is_success, is_error,
|
|
473
|
+
response_time,
|
|
474
|
+
response_time,
|
|
475
|
+
response_time
|
|
605
476
|
))
|
|
606
477
|
|
|
607
|
-
# 更新IP
|
|
478
|
+
# 2. 更新 IP 统计表
|
|
608
479
|
cursor.execute("""
|
|
609
|
-
INSERT INTO `
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
VALUES (
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
480
|
+
INSERT INTO `api_ip记录` (
|
|
481
|
+
`统计日期`, `客户端ip`, `请求总数`, `成功次数`, `失败次数`,
|
|
482
|
+
`平均耗时`, `首次访问`, `最后访问`
|
|
483
|
+
) VALUES (
|
|
484
|
+
%s, %s, 1, %s, %s, %s, %s, %s
|
|
485
|
+
) ON DUPLICATE KEY UPDATE
|
|
486
|
+
`请求总数` = `请求总数` + 1,
|
|
487
|
+
`成功次数` = `成功次数` + %s,
|
|
488
|
+
`失败次数` = `失败次数` + %s,
|
|
489
|
+
`平均耗时` = (
|
|
490
|
+
(`平均耗时` * (`请求总数` - 1) + %s) / `请求总数`
|
|
491
|
+
),
|
|
492
|
+
`最后访问` = %s
|
|
617
493
|
""", (
|
|
618
494
|
date,
|
|
619
|
-
request_data.get('
|
|
620
|
-
|
|
621
|
-
|
|
495
|
+
request_data.get('客户端ip', ''),
|
|
496
|
+
is_success, is_error,
|
|
497
|
+
response_time, now, now,
|
|
498
|
+
is_success, is_error,
|
|
499
|
+
response_time,
|
|
622
500
|
now
|
|
623
501
|
))
|
|
624
502
|
|
|
@@ -628,17 +506,37 @@ class RouteMonitor:
|
|
|
628
506
|
connection.close()
|
|
629
507
|
|
|
630
508
|
except Exception as e:
|
|
631
|
-
# logger.error("更新统计数据失败", {
|
|
632
|
-
# "请求ID": request_id,
|
|
633
|
-
# "错误信息": str(e),
|
|
634
|
-
# "错误类型": type(e).__name__,
|
|
635
|
-
# "影响": "统计数据缺失,但不影响主业务"
|
|
636
|
-
# })
|
|
637
509
|
# 静默处理错误
|
|
638
510
|
pass
|
|
511
|
+
|
|
512
|
+
# ==================== 核心装饰器 ====================
|
|
639
513
|
|
|
640
|
-
def
|
|
641
|
-
"""
|
|
514
|
+
def api_monitor(self, func):
|
|
515
|
+
"""
|
|
516
|
+
API 监控装饰器
|
|
517
|
+
|
|
518
|
+
使用方法:
|
|
519
|
+
```python
|
|
520
|
+
from mdbq.route.monitor import api_monitor
|
|
521
|
+
|
|
522
|
+
@app.route('/api/users')
|
|
523
|
+
@api_monitor
|
|
524
|
+
def get_users():
|
|
525
|
+
return {'users': [...]}
|
|
526
|
+
```
|
|
527
|
+
|
|
528
|
+
功能:
|
|
529
|
+
1. 自动记录请求的核心信息(IP、耗时、状态等)
|
|
530
|
+
2. 实时更新统计数据
|
|
531
|
+
3. 异常情况也会被记录
|
|
532
|
+
4. 不影响主业务逻辑的执行
|
|
533
|
+
|
|
534
|
+
Args:
|
|
535
|
+
func: 被装饰的函数
|
|
536
|
+
|
|
537
|
+
Returns:
|
|
538
|
+
装饰后的函数
|
|
539
|
+
"""
|
|
642
540
|
@functools.wraps(func)
|
|
643
541
|
def wrapper(*args, **kwargs):
|
|
644
542
|
# 记录开始时间
|
|
@@ -647,37 +545,30 @@ class RouteMonitor:
|
|
|
647
545
|
|
|
648
546
|
# 收集请求数据
|
|
649
547
|
request_data = self.collect_request_data(request)
|
|
650
|
-
request_id = request_data.get('
|
|
548
|
+
request_id = request_data.get('请求id', 'unknown')
|
|
651
549
|
|
|
652
550
|
try:
|
|
653
551
|
# 执行原函数
|
|
654
552
|
response = func(*args, **kwargs)
|
|
655
553
|
|
|
656
|
-
#
|
|
554
|
+
# 计算响应时间
|
|
657
555
|
end_time = time.time()
|
|
658
|
-
process_time = round((end_time - start_time) * 1000, 3)
|
|
659
|
-
|
|
660
|
-
response_status = getattr(response, 'status_code', 200) if hasattr(response, 'status_code') else 200
|
|
556
|
+
process_time = round((end_time - start_time) * 1000, 3) # 毫秒
|
|
661
557
|
|
|
662
|
-
#
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
response_size = len(str(response.get_data()))
|
|
670
|
-
else:
|
|
671
|
-
response_size = 0
|
|
672
|
-
except (RuntimeError, Exception):
|
|
673
|
-
# 如果获取数据失败(如直通模式),使用默认值
|
|
674
|
-
response_size = -1 # 标记为无法获取大小
|
|
558
|
+
# 获取响应状态码
|
|
559
|
+
response_status = 200
|
|
560
|
+
if hasattr(response, 'status_code'):
|
|
561
|
+
response_status = response.status_code
|
|
562
|
+
elif isinstance(response, tuple) and len(response) > 1:
|
|
563
|
+
# 处理 (data, status_code) 格式的返回
|
|
564
|
+
response_status = response[1]
|
|
675
565
|
|
|
566
|
+
# 更新响应数据
|
|
676
567
|
response_data = {
|
|
677
|
-
'
|
|
678
|
-
'
|
|
679
|
-
'response_size': response_size
|
|
568
|
+
'响应状态码': response_status,
|
|
569
|
+
'响应耗时': process_time,
|
|
680
570
|
}
|
|
571
|
+
|
|
681
572
|
# 保存日志
|
|
682
573
|
self.save_request_log(request_data, response_data)
|
|
683
574
|
|
|
@@ -692,22 +583,13 @@ class RouteMonitor:
|
|
|
692
583
|
end_time = time.time()
|
|
693
584
|
process_time = round((end_time - start_time) * 1000, 3)
|
|
694
585
|
|
|
586
|
+
# 构建错误数据
|
|
695
587
|
error_data = {
|
|
696
|
-
'
|
|
697
|
-
'
|
|
698
|
-
'
|
|
699
|
-
'response_size': 0
|
|
588
|
+
'响应状态码': 500,
|
|
589
|
+
'响应耗时': process_time,
|
|
590
|
+
'错误信息': f"{type(e).__name__}: {str(e)}"
|
|
700
591
|
}
|
|
701
592
|
|
|
702
|
-
# logger.error("请求处理异常", {
|
|
703
|
-
# "请求ID": request_id,
|
|
704
|
-
# "函数名": func.__name__,
|
|
705
|
-
# "错误信息": str(e),
|
|
706
|
-
# "错误类型": type(e).__name__,
|
|
707
|
-
# "处理时间": f"{process_time}ms",
|
|
708
|
-
# "结果": "异常"
|
|
709
|
-
# })
|
|
710
|
-
|
|
711
593
|
# 保存错误日志
|
|
712
594
|
self.save_request_log(request_data, error_data)
|
|
713
595
|
|
|
@@ -715,63 +597,101 @@ class RouteMonitor:
|
|
|
715
597
|
request_data.update(error_data)
|
|
716
598
|
self.update_statistics(request_data)
|
|
717
599
|
|
|
718
|
-
#
|
|
719
|
-
raise
|
|
600
|
+
# 重新抛出异常,不影响原有错误处理逻辑
|
|
601
|
+
raise
|
|
720
602
|
|
|
721
603
|
return wrapper
|
|
722
604
|
|
|
605
|
+
|
|
606
|
+
# ==================== 数据查询与分析 ====================
|
|
607
|
+
|
|
723
608
|
def get_statistics_summary(self, days: int = 7) -> Dict[str, Any]:
|
|
724
|
-
"""
|
|
609
|
+
"""
|
|
610
|
+
获取统计摘要
|
|
611
|
+
|
|
612
|
+
提供指定天数内的 API 访问统计概览。
|
|
613
|
+
|
|
614
|
+
Args:
|
|
615
|
+
days: 统计天数,默认 7 天
|
|
616
|
+
|
|
617
|
+
Returns:
|
|
618
|
+
dict: 包含以下内容的统计摘要:
|
|
619
|
+
- 统计周期
|
|
620
|
+
- 总体统计(总请求数、成功率、平均耗时等)
|
|
621
|
+
- 热门接口 TOP 10
|
|
622
|
+
- IP 统计
|
|
623
|
+
"""
|
|
725
624
|
try:
|
|
726
625
|
connection = self.pool.connection()
|
|
727
626
|
try:
|
|
728
627
|
with connection.cursor() as cursor:
|
|
729
|
-
# 确保使用正确的数据库上下文
|
|
730
628
|
self.ensure_database_context(cursor)
|
|
731
629
|
|
|
732
630
|
end_date = datetime.now().date()
|
|
733
631
|
start_date = end_date - timedelta(days=days)
|
|
734
632
|
|
|
735
|
-
# 总体统计
|
|
633
|
+
# 1. 总体统计
|
|
736
634
|
cursor.execute("""
|
|
737
635
|
SELECT
|
|
738
|
-
SUM(
|
|
739
|
-
SUM(
|
|
740
|
-
SUM(
|
|
741
|
-
AVG(
|
|
742
|
-
COUNT(DISTINCT
|
|
743
|
-
FROM
|
|
744
|
-
WHERE
|
|
636
|
+
SUM(请求总数) as 总请求数,
|
|
637
|
+
SUM(成功次数) as 成功次数,
|
|
638
|
+
SUM(失败次数) as 失败次数,
|
|
639
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时,
|
|
640
|
+
COUNT(DISTINCT 接口路径) as 接口数量
|
|
641
|
+
FROM api_接口统计
|
|
642
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
745
643
|
""", (start_date, end_date))
|
|
746
644
|
|
|
747
645
|
summary = cursor.fetchone() or {}
|
|
748
|
-
|
|
646
|
+
|
|
647
|
+
# 2. 热门接口 TOP 10
|
|
749
648
|
cursor.execute("""
|
|
750
|
-
SELECT
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
649
|
+
SELECT
|
|
650
|
+
接口路径,
|
|
651
|
+
SUM(请求总数) as 请求次数,
|
|
652
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时
|
|
653
|
+
FROM api_接口统计
|
|
654
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
655
|
+
GROUP BY 接口路径
|
|
656
|
+
ORDER BY 请求次数 DESC
|
|
755
657
|
LIMIT 10
|
|
756
658
|
""", (start_date, end_date))
|
|
757
659
|
|
|
758
660
|
top_endpoints = cursor.fetchall()
|
|
759
661
|
|
|
760
|
-
#
|
|
662
|
+
# 3. IP 统计
|
|
761
663
|
cursor.execute("""
|
|
762
|
-
SELECT
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
664
|
+
SELECT
|
|
665
|
+
COUNT(DISTINCT 客户端ip) as 独立ip数,
|
|
666
|
+
SUM(请求总数) as ip总请求数
|
|
667
|
+
FROM api_ip记录
|
|
668
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
766
669
|
""", (start_date, end_date))
|
|
767
670
|
|
|
768
671
|
ip_stats = cursor.fetchone() or {}
|
|
769
672
|
|
|
673
|
+
# 4. 性能最慢的接口 TOP 5
|
|
674
|
+
cursor.execute("""
|
|
675
|
+
SELECT
|
|
676
|
+
接口路径,
|
|
677
|
+
ROUND(MAX(最大耗时), 2) as 最大耗时,
|
|
678
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时
|
|
679
|
+
FROM api_接口统计
|
|
680
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
681
|
+
GROUP BY 接口路径
|
|
682
|
+
ORDER BY 最大耗时 DESC
|
|
683
|
+
LIMIT 5
|
|
684
|
+
""", (start_date, end_date))
|
|
685
|
+
|
|
686
|
+
slow_endpoints = cursor.fetchall()
|
|
687
|
+
|
|
688
|
+
# 构建返回结果
|
|
770
689
|
result = {
|
|
771
|
-
'
|
|
772
|
-
'
|
|
773
|
-
'
|
|
774
|
-
'
|
|
690
|
+
'统计周期': f'{start_date} 至 {end_date}',
|
|
691
|
+
'总体统计': summary,
|
|
692
|
+
'热门接口': top_endpoints,
|
|
693
|
+
'ip统计': ip_stats,
|
|
694
|
+
'慢接口': slow_endpoints
|
|
775
695
|
}
|
|
776
696
|
|
|
777
697
|
return result
|
|
@@ -780,26 +700,145 @@ class RouteMonitor:
|
|
|
780
700
|
connection.close()
|
|
781
701
|
|
|
782
702
|
except Exception as e:
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
703
|
+
return {'错误': str(e)}
|
|
704
|
+
|
|
705
|
+
def cleanup_old_data(self, days_to_keep: int = 30) -> Dict[str, int]:
|
|
706
|
+
"""
|
|
707
|
+
清理历史数据
|
|
708
|
+
|
|
709
|
+
删除指定天数之前的访问日志,保留统计数据。
|
|
710
|
+
建议定期执行(如每天凌晨)以控制数据库大小。
|
|
711
|
+
|
|
712
|
+
Args:
|
|
713
|
+
days_to_keep: 保留最近多少天的数据,默认 30 天
|
|
714
|
+
|
|
715
|
+
Returns:
|
|
716
|
+
dict: 清理结果,包含删除的记录数
|
|
717
|
+
"""
|
|
718
|
+
try:
|
|
719
|
+
connection = self.pool.connection()
|
|
720
|
+
try:
|
|
721
|
+
with connection.cursor() as cursor:
|
|
722
|
+
self.ensure_database_context(cursor)
|
|
723
|
+
|
|
724
|
+
cutoff_date = datetime.now().date() - timedelta(days=days_to_keep)
|
|
725
|
+
|
|
726
|
+
# 1. 清理访问日志表
|
|
727
|
+
cursor.execute("""
|
|
728
|
+
DELETE FROM api_访问日志
|
|
729
|
+
WHERE 请求时间 < %s
|
|
730
|
+
""", (cutoff_date,))
|
|
731
|
+
|
|
732
|
+
deleted_logs = cursor.rowcount
|
|
733
|
+
|
|
734
|
+
# 2. 清理 ip 记录表(可选,通常保留更久)
|
|
735
|
+
cursor.execute("""
|
|
736
|
+
DELETE FROM api_ip记录
|
|
737
|
+
WHERE 统计日期 < %s
|
|
738
|
+
""", (cutoff_date,))
|
|
739
|
+
|
|
740
|
+
deleted_ip_records = cursor.rowcount
|
|
741
|
+
|
|
742
|
+
connection.commit()
|
|
743
|
+
|
|
744
|
+
result = {
|
|
745
|
+
'删除日志数': deleted_logs,
|
|
746
|
+
'删除ip记录数': deleted_ip_records,
|
|
747
|
+
'清理日期': str(cutoff_date)
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
return result
|
|
751
|
+
|
|
752
|
+
finally:
|
|
753
|
+
connection.close()
|
|
754
|
+
|
|
755
|
+
except Exception as e:
|
|
756
|
+
return {'错误': str(e)}
|
|
757
|
+
|
|
790
758
|
|
|
759
|
+
# ==================== 全局实例与导出 ====================
|
|
791
760
|
|
|
792
|
-
#
|
|
761
|
+
# 创建全局监控实例
|
|
793
762
|
route_monitor = RouteMonitor()
|
|
794
763
|
|
|
795
|
-
#
|
|
796
|
-
|
|
764
|
+
# 导出核心装饰器(推荐使用此方式)
|
|
765
|
+
api_monitor = route_monitor.api_monitor
|
|
797
766
|
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
767
|
+
|
|
768
|
+
# ==================== 便捷工具函数 ====================
|
|
769
|
+
|
|
770
|
+
def get_request_id() -> Optional[str]:
|
|
771
|
+
"""
|
|
772
|
+
获取当前请求的唯一 ID
|
|
773
|
+
|
|
774
|
+
可在被 @api_monitor 装饰的函数内调用,用于日志关联和问题追踪。
|
|
775
|
+
|
|
776
|
+
Returns:
|
|
777
|
+
str: 请求 ID,如果不在请求上下文中则返回 None
|
|
778
|
+
|
|
779
|
+
示例:
|
|
780
|
+
```python
|
|
781
|
+
@api_monitor
|
|
782
|
+
def my_api():
|
|
783
|
+
req_id = get_request_id()
|
|
784
|
+
logger.info(f"处理请求: {req_id}")
|
|
785
|
+
return {'status': 'ok'}
|
|
786
|
+
```
|
|
787
|
+
"""
|
|
801
788
|
return getattr(g, 'request_id', None)
|
|
802
789
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
790
|
+
|
|
791
|
+
def get_statistics_summary(days: int = 7) -> Dict[str, Any]:
|
|
792
|
+
"""
|
|
793
|
+
获取统计摘要数据
|
|
794
|
+
|
|
795
|
+
Args:
|
|
796
|
+
days: 统计天数,默认 7 天
|
|
797
|
+
|
|
798
|
+
Returns:
|
|
799
|
+
dict: 统计摘要数据
|
|
800
|
+
|
|
801
|
+
示例:
|
|
802
|
+
```python
|
|
803
|
+
# 获取最近 7 天的统计
|
|
804
|
+
stats = get_statistics_summary(7)
|
|
805
|
+
print(stats['总体统计'])
|
|
806
|
+
|
|
807
|
+
# 获取最近 30 天的统计
|
|
808
|
+
stats = get_statistics_summary(30)
|
|
809
|
+
```
|
|
810
|
+
"""
|
|
811
|
+
return route_monitor.get_statistics_summary(days)
|
|
812
|
+
|
|
813
|
+
|
|
814
|
+
def cleanup_old_data(days_to_keep: int = 30) -> Dict[str, int]:
|
|
815
|
+
"""
|
|
816
|
+
清理历史数据
|
|
817
|
+
|
|
818
|
+
删除指定天数之前的详细日志,保留统计数据。
|
|
819
|
+
建议通过定时任务定期执行。
|
|
820
|
+
|
|
821
|
+
Args:
|
|
822
|
+
days_to_keep: 保留最近多少天的数据,默认 30 天
|
|
823
|
+
|
|
824
|
+
Returns:
|
|
825
|
+
dict: 清理结果统计
|
|
826
|
+
|
|
827
|
+
示例:
|
|
828
|
+
```python
|
|
829
|
+
# 保留最近 30 天的数据,删除更早的
|
|
830
|
+
result = cleanup_old_data(30)
|
|
831
|
+
print(f"清理完成: {result}")
|
|
832
|
+
```
|
|
833
|
+
"""
|
|
834
|
+
return route_monitor.cleanup_old_data(days_to_keep)
|
|
835
|
+
|
|
836
|
+
|
|
837
|
+
# ==================== 模块导出列表 ====================
|
|
838
|
+
__all__ = [
|
|
839
|
+
'RouteMonitor',
|
|
840
|
+
'api_monitor',
|
|
841
|
+
'get_request_id',
|
|
842
|
+
'get_statistics_summary',
|
|
843
|
+
'cleanup_old_data',
|
|
844
|
+
]
|