mdbq 4.2.11__py3-none-any.whl → 4.2.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- mdbq/__version__.py +1 -1
- mdbq/route/monitor.py +753 -558
- mdbq/route/monitor_worker.py +152 -0
- {mdbq-4.2.11.dist-info → mdbq-4.2.13.dist-info}/METADATA +1 -1
- {mdbq-4.2.11.dist-info → mdbq-4.2.13.dist-info}/RECORD +7 -6
- {mdbq-4.2.11.dist-info → mdbq-4.2.13.dist-info}/WHEEL +0 -0
- {mdbq-4.2.11.dist-info → mdbq-4.2.13.dist-info}/top_level.txt +0 -0
mdbq/route/monitor.py
CHANGED
|
@@ -1,66 +1,111 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
2
|
+
API 监控装饰器模块
|
|
3
3
|
|
|
4
|
-
|
|
5
|
-
1. 监控所有路由接口的访问请求
|
|
6
|
-
2. 记录详细的请求信息(IP、设备、请求头、请求体等)
|
|
7
|
-
3. 提供统计分析功能
|
|
8
|
-
4. 异常处理和数据清理
|
|
4
|
+
高性能、轻量级的 API 访问监控系统,专注于核心监控指标的收集和分析。
|
|
9
5
|
|
|
6
|
+
核心特性:
|
|
7
|
+
1. 请求监控:记录接口访问的核心信息(耗时、状态、ip 等)
|
|
8
|
+
2. 统计分析:提供按时间维度的访问统计和性能分析
|
|
9
|
+
3. 高性能:精简字段设计,优化索引策略,最小化性能影响
|
|
10
|
+
4. 安全性:自动脱敏敏感信息,支持 ip 风险评估
|
|
11
|
+
5. 自动清理:支持历史数据自动归档和清理
|
|
10
12
|
"""
|
|
11
13
|
|
|
12
14
|
import os
|
|
13
15
|
import json
|
|
14
16
|
import time
|
|
15
17
|
import uuid
|
|
18
|
+
import threading
|
|
16
19
|
import pymysql
|
|
17
|
-
import hashlib
|
|
18
20
|
import functools
|
|
21
|
+
import hashlib
|
|
19
22
|
from datetime import datetime, timedelta
|
|
20
|
-
from typing import Dict, Any
|
|
21
|
-
from urllib.parse import urlparse
|
|
23
|
+
from typing import Dict, Any, Optional
|
|
22
24
|
from dbutils.pooled_db import PooledDB # type: ignore
|
|
23
|
-
from mdbq.myconf import myconf # type: ignore
|
|
24
|
-
# from mdbq.log import mylogger
|
|
25
25
|
from flask import request, g
|
|
26
|
-
import re
|
|
27
|
-
import ipaddress
|
|
28
|
-
|
|
29
|
-
parser = myconf.ConfigParser()
|
|
30
|
-
host, port, username, password = parser.get_section_values(
|
|
31
|
-
file_path=os.path.join(os.path.expanduser("~"), 'spd.txt'),
|
|
32
|
-
section='mysql',
|
|
33
|
-
keys=['host', 'port', 'username', 'password'],
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
# logger = mylogger.MyLogger(
|
|
37
|
-
# logging_mode='file',
|
|
38
|
-
# log_level='info',
|
|
39
|
-
# log_format='json',
|
|
40
|
-
# max_log_size=50,
|
|
41
|
-
# backup_count=5,
|
|
42
|
-
# enable_async=False, # 是否启用异步日志
|
|
43
|
-
# sample_rate=1, # 采样DEBUG/INFO日志
|
|
44
|
-
# sensitive_fields=[], # 敏感字段过滤
|
|
45
|
-
# enable_metrics=False, # 是否启用性能指标
|
|
46
|
-
# )
|
|
47
26
|
|
|
48
27
|
|
|
49
28
|
class RouteMonitor:
|
|
50
|
-
"""
|
|
29
|
+
"""
|
|
30
|
+
路由监控核心类
|
|
31
|
+
|
|
32
|
+
负责 API 请求的监控、日志记录和统计分析。
|
|
51
33
|
|
|
52
|
-
|
|
53
|
-
|
|
34
|
+
Attributes:
|
|
35
|
+
database (str): 监控数据库名称,默认为 'api监控系统'
|
|
36
|
+
pool (PooledDB): 数据库连接池
|
|
37
|
+
|
|
38
|
+
核心方法:
|
|
39
|
+
- api_monitor: 装饰器,用于监控 API 接口
|
|
40
|
+
- get_statistics_summary: 获取统计摘要数据
|
|
41
|
+
- cleanup_old_data: 清理历史数据
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
def __init__(self, database: str = 'api监控系统', pool = None, redis_client = None, enable_async: bool = True):
|
|
45
|
+
"""
|
|
46
|
+
初始化监控系统
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
database: 数据库名称,默认为 'api监控系统'
|
|
50
|
+
pool: 数据库连接池对象,如果不传则使用默认配置创建
|
|
51
|
+
redis_client: Redis 客户端,用于异步队列(可选,如果不传则同步写入)
|
|
52
|
+
enable_async: 是否启用异步模式(需要 redis_client),默认 True
|
|
53
|
+
"""
|
|
54
54
|
self.database = database
|
|
55
|
-
self.
|
|
55
|
+
self.pool = pool
|
|
56
|
+
if self.pool is None:
|
|
57
|
+
self.init_database_pool()
|
|
56
58
|
self.init_database_tables()
|
|
59
|
+
|
|
60
|
+
# Redis 异步队列配置
|
|
61
|
+
self.redis_client = redis_client
|
|
62
|
+
self.enable_async = enable_async and self.redis_client is not None
|
|
63
|
+
# 队列名ASCII化,避免中文带来的跨系统兼容问题
|
|
64
|
+
try:
|
|
65
|
+
db_hash = hashlib.md5(self.database.encode('utf-8')).hexdigest()[:8]
|
|
66
|
+
self.queue_name = f"api_monitor:{db_hash}:tasks"
|
|
67
|
+
except Exception:
|
|
68
|
+
self.queue_name = "api_monitor:tasks"
|
|
69
|
+
|
|
70
|
+
# 线程锁(用于保护统计数据)
|
|
71
|
+
import threading
|
|
72
|
+
self._stats_lock = threading.Lock()
|
|
73
|
+
|
|
74
|
+
# 统计信息
|
|
75
|
+
self._stats = {
|
|
76
|
+
'total_requests': 0,
|
|
77
|
+
'queued_tasks': 0,
|
|
78
|
+
'sync_writes': 0,
|
|
79
|
+
'queue_failures': 0,
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if self.enable_async:
|
|
83
|
+
# 使用 Redis 队列模式(适合 uwsgi 多进程)
|
|
84
|
+
pass # 队列消费者需要单独进程运行
|
|
85
|
+
else:
|
|
86
|
+
# 降级为同步模式
|
|
87
|
+
pass
|
|
57
88
|
|
|
58
89
|
def init_database_pool(self):
|
|
59
|
-
"""
|
|
90
|
+
"""
|
|
91
|
+
初始化数据库连接池
|
|
92
|
+
|
|
93
|
+
配置说明:
|
|
94
|
+
- 最大连接数:2(监控系统不需要大量连接)
|
|
95
|
+
- 编码:utf8mb4(支持中文和 emoji)
|
|
96
|
+
- 自动重连:开启
|
|
97
|
+
"""
|
|
98
|
+
from mdbq.myconf import myconf # type: ignore
|
|
99
|
+
parser = myconf.ConfigParser()
|
|
100
|
+
host, port, username, password = parser.get_section_values(
|
|
101
|
+
file_path=os.path.join(os.path.expanduser("~"), 'spd.txt'),
|
|
102
|
+
section='mysql',
|
|
103
|
+
keys=['host', 'port', 'username', 'password'],
|
|
104
|
+
)
|
|
60
105
|
try:
|
|
61
106
|
self.pool = PooledDB(
|
|
62
107
|
creator=pymysql,
|
|
63
|
-
maxconnections=2,
|
|
108
|
+
maxconnections=2,
|
|
64
109
|
mincached=1,
|
|
65
110
|
maxcached=2,
|
|
66
111
|
blocking=True,
|
|
@@ -68,174 +113,141 @@ class RouteMonitor:
|
|
|
68
113
|
port=int(port),
|
|
69
114
|
user=username,
|
|
70
115
|
password=password,
|
|
71
|
-
ping=1,
|
|
116
|
+
ping=1, # 自动重连
|
|
72
117
|
charset='utf8mb4',
|
|
73
118
|
cursorclass=pymysql.cursors.DictCursor
|
|
74
119
|
)
|
|
75
120
|
|
|
76
|
-
#
|
|
121
|
+
# 创建数据库
|
|
77
122
|
connection = self.pool.connection()
|
|
78
123
|
try:
|
|
79
124
|
with connection.cursor() as cursor:
|
|
80
|
-
cursor.execute(
|
|
125
|
+
cursor.execute(
|
|
126
|
+
f"CREATE DATABASE IF NOT EXISTS `{self.database}` "
|
|
127
|
+
f"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
|
128
|
+
)
|
|
81
129
|
cursor.execute(f"USE `{self.database}`")
|
|
82
130
|
finally:
|
|
83
131
|
connection.close()
|
|
84
132
|
|
|
85
133
|
except Exception as e:
|
|
86
|
-
#
|
|
87
|
-
# "错误信息": str(e),
|
|
88
|
-
# "数据库": self.database
|
|
89
|
-
# })
|
|
134
|
+
# 保持原有行为:抛出异常由上层处理
|
|
90
135
|
raise
|
|
91
136
|
|
|
92
137
|
def ensure_database_context(self, cursor):
|
|
93
|
-
"""
|
|
138
|
+
"""
|
|
139
|
+
确保游标处于正确的数据库上下文中
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
cursor: 数据库游标对象
|
|
143
|
+
"""
|
|
94
144
|
try:
|
|
95
145
|
cursor.execute(f"USE `{self.database}`")
|
|
96
|
-
except Exception
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.database}` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci")
|
|
146
|
+
except Exception:
|
|
147
|
+
cursor.execute(
|
|
148
|
+
f"CREATE DATABASE IF NOT EXISTS `{self.database}` "
|
|
149
|
+
f"DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
|
|
150
|
+
)
|
|
102
151
|
cursor.execute(f"USE `{self.database}`")
|
|
103
152
|
|
|
104
153
|
def init_database_tables(self):
|
|
105
|
-
"""
|
|
154
|
+
"""
|
|
155
|
+
初始化数据库表结构
|
|
156
|
+
|
|
157
|
+
创建三张核心表:
|
|
158
|
+
1. api_访问日志:记录每次 API 请求的详细信息
|
|
159
|
+
2. api_接口统计:按小时汇总的接口性能统计
|
|
160
|
+
3. api_ip记录:IP 维度的访问统计
|
|
161
|
+
"""
|
|
106
162
|
try:
|
|
107
163
|
connection = self.pool.connection()
|
|
108
164
|
try:
|
|
109
165
|
with connection.cursor() as cursor:
|
|
110
|
-
# 确保使用正确的数据库上下文
|
|
111
166
|
self.ensure_database_context(cursor)
|
|
112
167
|
|
|
113
|
-
#
|
|
114
|
-
|
|
115
|
-
CREATE TABLE IF NOT EXISTS `api_request_logs` (
|
|
116
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键ID',
|
|
117
|
-
`request_id` VARCHAR(128) NOT NULL COMMENT '请求唯一标识',
|
|
118
|
-
`timestamp` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '请求时间(精确到毫秒)',
|
|
119
|
-
`method` VARCHAR(10) NOT NULL COMMENT 'HTTP方法',
|
|
120
|
-
`endpoint` VARCHAR(500) NOT NULL COMMENT '请求端点',
|
|
121
|
-
`full_url` TEXT COMMENT '完整URL',
|
|
122
|
-
`client_ip` VARCHAR(45) NOT NULL COMMENT '客户端IP地址',
|
|
123
|
-
`real_ip` VARCHAR(45) COMMENT '真实IP地址',
|
|
124
|
-
`forwarded_ips` TEXT COMMENT '转发IP链',
|
|
125
|
-
`user_agent` TEXT COMMENT '用户代理',
|
|
126
|
-
`referer` VARCHAR(1000) COMMENT '来源页面',
|
|
127
|
-
`host` VARCHAR(255) COMMENT '请求主机',
|
|
128
|
-
`scheme` VARCHAR(10) COMMENT '协议类型',
|
|
129
|
-
`port` INT COMMENT '端口号',
|
|
130
|
-
`request_headers` JSON COMMENT '请求头信息',
|
|
131
|
-
`request_params` JSON COMMENT '请求参数',
|
|
132
|
-
`request_body` LONGTEXT COMMENT '请求体内容',
|
|
133
|
-
`request_size` INT DEFAULT 0 COMMENT '请求大小(字节)',
|
|
134
|
-
`response_status` INT COMMENT '响应状态码',
|
|
135
|
-
`response_size` INT COMMENT '响应大小(字节)',
|
|
136
|
-
`process_time` DECIMAL(10,3) COMMENT '处理时间(毫秒)',
|
|
137
|
-
`session_id` VARCHAR(128) COMMENT '会话ID',
|
|
138
|
-
`user_id` VARCHAR(64) COMMENT '用户ID',
|
|
139
|
-
`auth_token` TEXT COMMENT '认证令牌(脱敏)',
|
|
140
|
-
`device_fingerprint` VARCHAR(256) COMMENT '设备指纹',
|
|
141
|
-
`device_info` JSON COMMENT '设备信息',
|
|
142
|
-
`geo_country` VARCHAR(100) COMMENT '地理位置-国家',
|
|
143
|
-
`geo_region` VARCHAR(100) COMMENT '地理位置-地区',
|
|
144
|
-
`geo_city` VARCHAR(100) COMMENT '地理位置-城市',
|
|
145
|
-
`is_bot` BOOLEAN DEFAULT FALSE COMMENT '是否为机器人',
|
|
146
|
-
`is_mobile` BOOLEAN DEFAULT FALSE COMMENT '是否为移动设备',
|
|
147
|
-
`browser_name` VARCHAR(50) COMMENT '浏览器名称',
|
|
148
|
-
`browser_version` VARCHAR(20) COMMENT '浏览器版本',
|
|
149
|
-
`os_name` VARCHAR(50) COMMENT '操作系统名称',
|
|
150
|
-
`os_version` VARCHAR(20) COMMENT '操作系统版本',
|
|
151
|
-
`error_message` TEXT COMMENT '错误信息',
|
|
152
|
-
`business_data` JSON COMMENT '业务数据',
|
|
153
|
-
`tags` JSON COMMENT '标签信息',
|
|
154
|
-
UNIQUE KEY `uk_request_id` (`request_id`),
|
|
155
|
-
INDEX `idx_timestamp` (`timestamp`),
|
|
156
|
-
INDEX `idx_endpoint` (`endpoint`(191)),
|
|
157
|
-
INDEX `idx_client_ip` (`client_ip`),
|
|
158
|
-
INDEX `idx_user_id` (`user_id`),
|
|
159
|
-
INDEX `idx_status` (`response_status`),
|
|
160
|
-
INDEX `idx_method_endpoint` (`method`, `endpoint`(191)),
|
|
161
|
-
INDEX `idx_timestamp_endpoint` (`timestamp`, `endpoint`(191))
|
|
162
|
-
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
163
|
-
COMMENT='API请求详细日志表';
|
|
164
|
-
""")
|
|
165
|
-
# 创建访问统计汇总表
|
|
168
|
+
# ==================== 表 1:访问日志表 ====================
|
|
169
|
+
# 设计原则:只保留核心监控字段,移除冗余信息
|
|
166
170
|
cursor.execute("""
|
|
167
|
-
CREATE TABLE IF NOT EXISTS `
|
|
168
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
169
|
-
`
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
`
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
`
|
|
184
|
-
`
|
|
185
|
-
`
|
|
186
|
-
`
|
|
187
|
-
|
|
188
|
-
INDEX `
|
|
189
|
-
INDEX `
|
|
190
|
-
INDEX `idx_date_endpoint` (`date`, `endpoint`(191))
|
|
171
|
+
CREATE TABLE IF NOT EXISTS `api_访问日志` (
|
|
172
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
173
|
+
`请求id` VARCHAR(64) NOT NULL COMMENT '请求唯一标识(用于追踪)',
|
|
174
|
+
`请求时间` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '请求时间,精确到毫秒',
|
|
175
|
+
`请求方法` VARCHAR(10) NOT NULL COMMENT 'HTTP 方法(GET/POST/PUT/DELETE等)',
|
|
176
|
+
`接口路径` VARCHAR(500) NOT NULL COMMENT 'API 接口路径',
|
|
177
|
+
`客户端ip` VARCHAR(45) NOT NULL COMMENT '客户端 ip 地址(支持 IPv6)',
|
|
178
|
+
`响应状态码` SMALLINT COMMENT 'HTTP 响应状态码',
|
|
179
|
+
`响应耗时` DECIMAL(10,3) COMMENT '请求处理耗时(毫秒)',
|
|
180
|
+
`用户标识` VARCHAR(64) COMMENT '用户id或标识(如有)',
|
|
181
|
+
`用户代理` VARCHAR(500) COMMENT '浏览器 User-Agent(精简版)',
|
|
182
|
+
`请求参数` TEXT COMMENT '请求参数(JSON格式,可选记录)',
|
|
183
|
+
`错误信息` TEXT COMMENT '错误信息(仅失败请求记录)',
|
|
184
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
185
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
186
|
+
|
|
187
|
+
UNIQUE KEY `uk_请求id` (`请求id`),
|
|
188
|
+
INDEX `idx_请求时间` (`请求时间`),
|
|
189
|
+
INDEX `idx_接口路径` (`接口路径`(191)),
|
|
190
|
+
INDEX `idx_客户端ip` (`客户端ip`),
|
|
191
|
+
INDEX `idx_响应状态码` (`响应状态码`),
|
|
192
|
+
INDEX `idx_用户标识` (`用户标识`),
|
|
193
|
+
INDEX `idx_时间_接口` (`请求时间`, `接口路径`(191))
|
|
191
194
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
192
|
-
COMMENT='API
|
|
195
|
+
COMMENT='API 访问日志表 - 记录每次请求的核心信息'
|
|
196
|
+
ROW_FORMAT=COMPRESSED;
|
|
193
197
|
""")
|
|
194
198
|
|
|
195
|
-
#
|
|
199
|
+
# ==================== 表 2:接口统计表 ====================
|
|
200
|
+
# 设计原则:按小时维度汇总,用于性能分析和趋势监控
|
|
196
201
|
cursor.execute("""
|
|
197
|
-
CREATE TABLE IF NOT EXISTS `
|
|
198
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
INDEX `
|
|
216
|
-
INDEX `
|
|
217
|
-
INDEX `idx_suspicious` (`is_suspicious`),
|
|
218
|
-
INDEX `idx_risk_score` (`risk_score`)
|
|
202
|
+
CREATE TABLE IF NOT EXISTS `api_接口统计` (
|
|
203
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
204
|
+
`统计日期` DATE NOT NULL COMMENT '统计日期',
|
|
205
|
+
`统计小时` TINYINT NOT NULL COMMENT '统计小时(0-23)',
|
|
206
|
+
`接口路径` VARCHAR(500) NOT NULL COMMENT 'API 接口路径',
|
|
207
|
+
`请求方法` VARCHAR(10) NOT NULL COMMENT 'HTTP 请求方法',
|
|
208
|
+
`请求总数` INT UNSIGNED DEFAULT 0 COMMENT '总请求次数',
|
|
209
|
+
`成功次数` INT UNSIGNED DEFAULT 0 COMMENT '成功响应次数(状态码 < 400)',
|
|
210
|
+
`失败次数` INT UNSIGNED DEFAULT 0 COMMENT '失败响应次数(状态码 >= 400)',
|
|
211
|
+
`平均耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应耗时(毫秒)',
|
|
212
|
+
`最大耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '最大响应耗时(毫秒)',
|
|
213
|
+
`最小耗时` DECIMAL(10,3) DEFAULT NULL COMMENT '最小响应耗时(毫秒)',
|
|
214
|
+
`独立ip数` INT UNSIGNED DEFAULT 0 COMMENT '访问的独立 ip 数量',
|
|
215
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
216
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
217
|
+
|
|
218
|
+
UNIQUE KEY `uk_日期_小时_接口_方法` (`统计日期`, `统计小时`, `接口路径`(191), `请求方法`),
|
|
219
|
+
INDEX `idx_统计日期` (`统计日期`),
|
|
220
|
+
INDEX `idx_接口路径` (`接口路径`(191)),
|
|
221
|
+
INDEX `idx_日期_接口` (`统计日期`, `接口路径`(191))
|
|
219
222
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
220
|
-
COMMENT='
|
|
223
|
+
COMMENT='API 接口统计表 - 按小时汇总的接口性能数据';
|
|
221
224
|
""")
|
|
222
|
-
|
|
225
|
+
|
|
226
|
+
# ==================== 表 3:IP 访问记录表 ====================
|
|
227
|
+
# 设计原则:按日期汇总 IP 访问情况,用于安全分析和流量监控
|
|
223
228
|
cursor.execute("""
|
|
224
|
-
CREATE TABLE IF NOT EXISTS `
|
|
225
|
-
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '
|
|
226
|
-
|
|
227
|
-
`
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
229
|
+
CREATE TABLE IF NOT EXISTS `api_ip记录` (
|
|
230
|
+
`id` BIGINT AUTO_INCREMENT PRIMARY KEY COMMENT '主键,自增id',
|
|
231
|
+
`统计日期` DATE NOT NULL COMMENT '统计日期',
|
|
232
|
+
`客户端ip` VARCHAR(45) NOT NULL COMMENT '客户端 ip 地址',
|
|
233
|
+
`请求总数` INT UNSIGNED DEFAULT 0 COMMENT '该 ip 当日总请求数',
|
|
234
|
+
`成功次数` INT UNSIGNED DEFAULT 0 COMMENT '成功请求次数',
|
|
235
|
+
`失败次数` INT UNSIGNED DEFAULT 0 COMMENT '失败请求次数',
|
|
236
|
+
`平均耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应耗时(毫秒)',
|
|
237
|
+
`首次访问` DATETIME COMMENT '首次访问时间',
|
|
238
|
+
`最后访问` DATETIME COMMENT '最后访问时间',
|
|
239
|
+
`访问接口数` INT UNSIGNED DEFAULT 0 COMMENT '访问的不同接口数量',
|
|
240
|
+
`风险评分` TINYINT UNSIGNED DEFAULT 0 COMMENT '风险评分(0-100,用于识别异常流量)',
|
|
241
|
+
`创建时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间',
|
|
242
|
+
`更新时间` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间',
|
|
243
|
+
|
|
244
|
+
UNIQUE KEY `uk_日期_ip` (`统计日期`, `客户端ip`),
|
|
245
|
+
INDEX `idx_统计日期` (`统计日期`),
|
|
246
|
+
INDEX `idx_客户端ip` (`客户端ip`),
|
|
247
|
+
INDEX `idx_风险评分` (`风险评分`),
|
|
248
|
+
INDEX `idx_请求总数` (`请求总数`)
|
|
237
249
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
238
|
-
COMMENT='
|
|
250
|
+
COMMENT='API ip 访问记录表 - ip 维度的访问统计';
|
|
239
251
|
""")
|
|
240
252
|
connection.commit()
|
|
241
253
|
|
|
@@ -243,287 +255,258 @@ class RouteMonitor:
|
|
|
243
255
|
connection.close()
|
|
244
256
|
|
|
245
257
|
except Exception as e:
|
|
246
|
-
#
|
|
247
|
-
# "错误信息": str(e),
|
|
248
|
-
# "错误类型": type(e).__name__,
|
|
249
|
-
# "数据库": self.database,
|
|
250
|
-
# "影响": "监控系统可能无法正常工作"
|
|
251
|
-
# })
|
|
252
|
-
# 静默处理初始化错误,避免影响主应用
|
|
258
|
+
# 保持静默降级行为,不影响主应用启动
|
|
253
259
|
pass
|
|
254
260
|
|
|
255
|
-
|
|
256
|
-
"""生成唯一的请求ID"""
|
|
257
|
-
timestamp = str(int(time.time() * 1000)) # 毫秒时间戳
|
|
258
|
-
random_part = uuid.uuid4().hex[:8]
|
|
259
|
-
return f"req_{timestamp}_{random_part}"
|
|
261
|
+
# ==================== Redis 队列方法 ====================
|
|
260
262
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
'
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
if not user_agent:
|
|
273
|
-
return device_info
|
|
274
|
-
|
|
275
|
-
user_agent_lower = user_agent.lower()
|
|
276
|
-
|
|
277
|
-
# 检测移动设备
|
|
278
|
-
mobile_keywords = ['mobile', 'android', 'iphone', 'ipad', 'ipod', 'windows phone']
|
|
279
|
-
device_info['is_mobile'] = any(keyword in user_agent_lower for keyword in mobile_keywords)
|
|
280
|
-
|
|
281
|
-
# 检测机器人
|
|
282
|
-
bot_keywords = ['bot', 'crawler', 'spider', 'scraper', 'curl', 'wget', 'python-requests']
|
|
283
|
-
device_info['is_bot'] = any(keyword in user_agent_lower for keyword in bot_keywords)
|
|
284
|
-
|
|
285
|
-
# 浏览器检测
|
|
286
|
-
browsers = [
|
|
287
|
-
('chrome', r'chrome/(\d+)'),
|
|
288
|
-
('firefox', r'firefox/(\d+)'),
|
|
289
|
-
('safari', r'safari/(\d+)'),
|
|
290
|
-
('edge', r'edge/(\d+)'),
|
|
291
|
-
('opera', r'opera/(\d+)')
|
|
292
|
-
]
|
|
293
|
-
|
|
294
|
-
for browser, pattern in browsers:
|
|
295
|
-
match = re.search(pattern, user_agent_lower)
|
|
296
|
-
if match:
|
|
297
|
-
device_info['browser_name'] = browser.title()
|
|
298
|
-
device_info['browser_version'] = match.group(1)
|
|
299
|
-
break
|
|
300
|
-
|
|
301
|
-
# 操作系统检测
|
|
302
|
-
os_patterns = [
|
|
303
|
-
('Windows', r'windows nt (\d+\.\d+)'),
|
|
304
|
-
('macOS', r'mac os x (\d+_\d+)'),
|
|
305
|
-
('Linux', r'linux'),
|
|
306
|
-
('Android', r'android (\d+)'),
|
|
307
|
-
('iOS', r'os (\d+_\d+)')
|
|
308
|
-
]
|
|
309
|
-
|
|
310
|
-
for os_name, pattern in os_patterns:
|
|
311
|
-
match = re.search(pattern, user_agent_lower)
|
|
312
|
-
if match:
|
|
313
|
-
device_info['os_name'] = os_name
|
|
314
|
-
if len(match.groups()) > 0:
|
|
315
|
-
device_info['os_version'] = match.group(1).replace('_', '.')
|
|
316
|
-
break
|
|
317
|
-
|
|
318
|
-
return device_info
|
|
263
|
+
@staticmethod
|
|
264
|
+
def _datetime_converter(obj):
|
|
265
|
+
"""
|
|
266
|
+
JSON 序列化时的 datetime 转换器
|
|
267
|
+
|
|
268
|
+
将 datetime 对象转换为特殊格式的字典,便于反序列化时还原
|
|
269
|
+
"""
|
|
270
|
+
if isinstance(obj, datetime):
|
|
271
|
+
return {'__datetime__': True, 'value': obj.isoformat()}
|
|
272
|
+
return str(obj)
|
|
319
273
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
return hashlib.md5(fingerprint_str.encode()).hexdigest()
|
|
274
|
+
@staticmethod
|
|
275
|
+
def _datetime_decoder(dct):
|
|
276
|
+
"""
|
|
277
|
+
JSON 反序列化时的 datetime 解码器
|
|
278
|
+
|
|
279
|
+
将特殊格式的字典还原为 datetime 对象
|
|
280
|
+
"""
|
|
281
|
+
if isinstance(dct, dict) and '__datetime__' in dct:
|
|
282
|
+
return datetime.fromisoformat(dct['value'])
|
|
283
|
+
return dct
|
|
331
284
|
|
|
332
|
-
def
|
|
333
|
-
"""
|
|
334
|
-
|
|
335
|
-
return None
|
|
285
|
+
def _push_to_queue(self, task_data: Dict[str, Any]) -> bool:
|
|
286
|
+
"""
|
|
287
|
+
将监控任务推入 Redis 队列(非阻塞,极快)
|
|
336
288
|
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
sensitive_patterns = [
|
|
340
|
-
(r'password["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'password: [REDACTED]'),
|
|
341
|
-
(r'token["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'token: [REDACTED]'),
|
|
342
|
-
(r'key["\']?\s*[:=]\s*["\']?[^"\'&\s]+', 'key: [REDACTED]'),
|
|
343
|
-
]
|
|
289
|
+
Args:
|
|
290
|
+
task_data: 任务数据字典
|
|
344
291
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
292
|
+
Returns:
|
|
293
|
+
bool: 是否成功推入队列
|
|
294
|
+
"""
|
|
295
|
+
if not self.enable_async:
|
|
296
|
+
return False
|
|
297
|
+
|
|
298
|
+
try:
|
|
299
|
+
# 将任务数据序列化为 JSON(使用 datetime 转换器)
|
|
300
|
+
task_json = json.dumps(task_data, default=self._datetime_converter, ensure_ascii=False)
|
|
348
301
|
|
|
349
|
-
#
|
|
350
|
-
|
|
351
|
-
sanitized = sanitized[:max_length] + '...[TRUNCATED]'
|
|
302
|
+
# 推入 Redis 列表(LPUSH,从左侧推入)
|
|
303
|
+
self.redis_client.lpush(self.queue_name, task_json)
|
|
352
304
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
305
|
+
with self._stats_lock:
|
|
306
|
+
self._stats['queued_tasks'] += 1
|
|
307
|
+
return True
|
|
308
|
+
|
|
309
|
+
except Exception as e:
|
|
310
|
+
# 静默处理错误
|
|
311
|
+
with self._stats_lock:
|
|
312
|
+
self._stats['queue_failures'] += 1
|
|
313
|
+
return False
|
|
314
|
+
|
|
315
|
+
def get_monitor_stats(self) -> Dict[str, Any]:
|
|
316
|
+
"""
|
|
317
|
+
获取监控系统统计信息
|
|
318
|
+
|
|
319
|
+
Returns:
|
|
320
|
+
dict: 统计信息字典
|
|
321
|
+
"""
|
|
322
|
+
with self._stats_lock:
|
|
323
|
+
stats = self._stats.copy()
|
|
324
|
+
|
|
325
|
+
# 添加队列信息
|
|
326
|
+
if self.enable_async:
|
|
327
|
+
try:
|
|
328
|
+
queue_length = self.redis_client.llen(self.queue_name)
|
|
329
|
+
stats['queue_length'] = queue_length
|
|
330
|
+
stats['mode'] = 'async_redis'
|
|
331
|
+
except:
|
|
332
|
+
stats['queue_length'] = -1
|
|
333
|
+
stats['mode'] = 'async_redis_error'
|
|
334
|
+
else:
|
|
335
|
+
stats['queue_length'] = 0
|
|
336
|
+
stats['mode'] = 'sync'
|
|
337
|
+
|
|
338
|
+
return stats
|
|
339
|
+
|
|
340
|
+
# ==================== 辅助方法 ====================
|
|
341
|
+
|
|
342
|
+
def generate_request_id(self) -> str:
|
|
343
|
+
"""
|
|
344
|
+
生成唯一的请求 ID
|
|
363
345
|
|
|
364
|
-
|
|
365
|
-
|
|
346
|
+
格式:req_{时间戳}_{随机字符串}
|
|
347
|
+
示例:req_1697654321123_a1b2c3d4
|
|
366
348
|
|
|
367
|
-
|
|
349
|
+
Returns:
|
|
350
|
+
str: 请求唯一标识符
|
|
351
|
+
"""
|
|
352
|
+
timestamp = str(int(time.time() * 1000))
|
|
353
|
+
random_part = uuid.uuid4().hex[:8]
|
|
354
|
+
return f"req_{timestamp}_{random_part}"
|
|
368
355
|
|
|
369
|
-
def get_real_ip(self, request) ->
|
|
370
|
-
"""
|
|
371
|
-
|
|
356
|
+
def get_real_ip(self, request) -> str:
|
|
357
|
+
"""
|
|
358
|
+
获取真实客户端 IP 地址
|
|
359
|
+
|
|
360
|
+
优先级顺序:
|
|
361
|
+
1. X-Forwarded-For(代理服务器传递的原始IP)
|
|
362
|
+
2. X-Real-IP(Nginx 等反向代理设置)
|
|
363
|
+
3. CF-Connecting-IP(Cloudflare CDN)
|
|
364
|
+
4. request.remote_addr(直连IP)
|
|
365
|
+
|
|
366
|
+
Args:
|
|
367
|
+
request: Flask request 对象
|
|
368
|
+
|
|
369
|
+
Returns:
|
|
370
|
+
str: 客户端真实 IP 地址
|
|
371
|
+
"""
|
|
372
|
+
# IP 头优先级列表
|
|
372
373
|
ip_headers = [
|
|
373
374
|
'X-Forwarded-For',
|
|
374
375
|
'X-Real-IP',
|
|
375
|
-
'CF-Connecting-IP',
|
|
376
|
+
'CF-Connecting-IP',
|
|
376
377
|
'X-Client-IP',
|
|
377
|
-
'X-Forwarded',
|
|
378
|
-
'Forwarded-For',
|
|
379
|
-
'Forwarded'
|
|
380
378
|
]
|
|
381
379
|
|
|
382
|
-
|
|
383
|
-
real_ip = request.remote_addr
|
|
384
|
-
|
|
380
|
+
# 尝试从请求头获取 IP
|
|
385
381
|
for header in ip_headers:
|
|
386
382
|
header_value = request.headers.get(header)
|
|
387
383
|
if header_value:
|
|
388
|
-
#
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
# 取第一个有效的IP作为真实IP
|
|
393
|
-
for ip in ips:
|
|
394
|
-
if self.is_valid_ip(ip) and not self.is_private_ip(ip):
|
|
395
|
-
real_ip = ip
|
|
396
|
-
break
|
|
397
|
-
|
|
398
|
-
if real_ip != request.remote_addr:
|
|
399
|
-
break
|
|
384
|
+
# X-Forwarded-For 可能包含多个 IP,取第一个
|
|
385
|
+
ip = header_value.split(',')[0].strip()
|
|
386
|
+
if ip:
|
|
387
|
+
return ip
|
|
400
388
|
|
|
401
|
-
|
|
389
|
+
# 如果没有代理头,返回直连 IP
|
|
390
|
+
return request.remote_addr or 'unknown'
|
|
402
391
|
|
|
403
|
-
def
|
|
404
|
-
"""
|
|
392
|
+
def sanitize_params(self, params: Dict[str, Any]) -> Optional[str]:
|
|
393
|
+
"""
|
|
394
|
+
清理和脱敏请求参数
|
|
395
|
+
|
|
396
|
+
自动移除敏感字段(如 password、token 等)
|
|
397
|
+
|
|
398
|
+
Args:
|
|
399
|
+
params: 请求参数字典
|
|
400
|
+
|
|
401
|
+
Returns:
|
|
402
|
+
str: JSON 格式的参数字符串(已脱敏),或 None
|
|
403
|
+
"""
|
|
404
|
+
if not params:
|
|
405
|
+
return None
|
|
406
|
+
|
|
407
|
+
# 敏感字段列表
|
|
408
|
+
sensitive_keys = ['password', 'passwd', 'pwd', 'token', 'secret', 'key', 'api_key', 'apikey']
|
|
409
|
+
|
|
410
|
+
# 创建副本并脱敏
|
|
411
|
+
sanitized = {}
|
|
412
|
+
for key, value in params.items():
|
|
413
|
+
if any(sensitive in key.lower() for sensitive in sensitive_keys):
|
|
414
|
+
sanitized[key] = '***'
|
|
415
|
+
else:
|
|
416
|
+
# 截断过长的值
|
|
417
|
+
if isinstance(value, str) and len(value) > 500:
|
|
418
|
+
sanitized[key] = value[:500] + '...'
|
|
419
|
+
else:
|
|
420
|
+
sanitized[key] = value
|
|
421
|
+
|
|
405
422
|
try:
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
return False
|
|
423
|
+
return json.dumps(sanitized, ensure_ascii=False)
|
|
424
|
+
except Exception:
|
|
425
|
+
return None
|
|
410
426
|
|
|
411
|
-
|
|
412
|
-
"""检查是否为私有IP"""
|
|
413
|
-
try:
|
|
414
|
-
return ipaddress.ip_address(ip).is_private
|
|
415
|
-
except ValueError:
|
|
416
|
-
return False
|
|
427
|
+
# ==================== 核心数据收集 ====================
|
|
417
428
|
|
|
418
429
|
def collect_request_data(self, request) -> Dict[str, Any]:
|
|
419
|
-
"""
|
|
420
|
-
|
|
430
|
+
"""
|
|
431
|
+
收集请求核心数据
|
|
432
|
+
|
|
433
|
+
仅收集必要的监控信息,避免过度记录造成性能和存储压力。
|
|
434
|
+
|
|
435
|
+
Args:
|
|
436
|
+
request: Flask request 对象
|
|
437
|
+
|
|
438
|
+
Returns:
|
|
439
|
+
dict: 包含请求核心信息的字典
|
|
440
|
+
"""
|
|
421
441
|
request_id = self.generate_request_id()
|
|
442
|
+
g.request_id = request_id # 保存到全局变量,供后续使用
|
|
422
443
|
|
|
423
|
-
#
|
|
424
|
-
|
|
425
|
-
# 获取真实IP
|
|
426
|
-
real_ip, forwarded_ips = self.get_real_ip(request)
|
|
444
|
+
# 获取客户端 IP
|
|
445
|
+
client_ip = self.get_real_ip(request)
|
|
427
446
|
|
|
428
|
-
#
|
|
429
|
-
|
|
430
|
-
|
|
447
|
+
# 获取 User-Agent(截断过长的)
|
|
448
|
+
user_agent = request.headers.get('User-Agent', '')
|
|
449
|
+
if len(user_agent) > 500:
|
|
450
|
+
user_agent = user_agent[:500]
|
|
431
451
|
|
|
432
|
-
#
|
|
433
|
-
|
|
434
|
-
if
|
|
435
|
-
|
|
452
|
+
# 获取用户标识(如果有)
|
|
453
|
+
user_id = None
|
|
454
|
+
if hasattr(g, 'current_user_id'):
|
|
455
|
+
user_id = str(g.current_user_id)
|
|
456
|
+
elif hasattr(g, 'user_id'):
|
|
457
|
+
user_id = str(g.user_id)
|
|
436
458
|
|
|
437
|
-
#
|
|
438
|
-
|
|
439
|
-
|
|
459
|
+
# 收集请求参数(GET 参数 + POST 数据)
|
|
460
|
+
request_params = None
|
|
461
|
+
params_dict = {}
|
|
440
462
|
|
|
441
|
-
|
|
442
|
-
|
|
463
|
+
# 1. 收集 URL 参数(GET)
|
|
464
|
+
if request.args:
|
|
465
|
+
params_dict.update(dict(request.args))
|
|
466
|
+
|
|
467
|
+
# 2. 收集 POST/PUT/PATCH 请求体数据
|
|
468
|
+
if request.method in ['POST', 'PUT', 'PATCH']:
|
|
469
|
+
try:
|
|
443
470
|
if request.is_json:
|
|
444
|
-
|
|
471
|
+
# JSON 数据
|
|
472
|
+
json_data = request.get_json(silent=True)
|
|
473
|
+
if json_data:
|
|
474
|
+
params_dict.update(json_data)
|
|
445
475
|
elif request.form:
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
try:
|
|
451
|
-
request_body = body_data.decode('utf-8')
|
|
452
|
-
except UnicodeDecodeError:
|
|
453
|
-
request_body = f"[BINARY_DATA:{len(body_data)}_bytes]"
|
|
454
|
-
|
|
455
|
-
if request_body:
|
|
456
|
-
request_size = len(str(request_body).encode('utf-8'))
|
|
457
|
-
except Exception as e:
|
|
458
|
-
request_body = "[ERROR_READING_BODY]"
|
|
459
|
-
# logger.warning("读取请求体失败", {
|
|
460
|
-
# "请求ID": request_id,
|
|
461
|
-
# "错误": str(e)
|
|
462
|
-
# })
|
|
476
|
+
# 表单数据
|
|
477
|
+
params_dict.update(dict(request.form))
|
|
478
|
+
except Exception:
|
|
479
|
+
pass
|
|
463
480
|
|
|
464
|
-
#
|
|
465
|
-
|
|
466
|
-
|
|
481
|
+
# 3. 脱敏处理
|
|
482
|
+
if params_dict:
|
|
483
|
+
request_params = self.sanitize_params(params_dict)
|
|
467
484
|
|
|
468
|
-
#
|
|
469
|
-
user_agent = request.headers.get('User-Agent', '')
|
|
470
|
-
device_info = self.extract_device_info(user_agent)
|
|
471
|
-
# URL解析
|
|
472
|
-
parsed_url = urlparse(request.url)
|
|
473
|
-
|
|
474
|
-
# 构建请求数据
|
|
485
|
+
# 构建请求数据字典
|
|
475
486
|
request_data = {
|
|
476
|
-
'
|
|
477
|
-
'
|
|
478
|
-
'
|
|
479
|
-
'
|
|
480
|
-
'
|
|
481
|
-
'
|
|
482
|
-
'
|
|
483
|
-
'
|
|
484
|
-
'user_agent': user_agent,
|
|
485
|
-
'referer': request.headers.get('Referer'),
|
|
486
|
-
'host': request.headers.get('Host'),
|
|
487
|
-
'scheme': parsed_url.scheme,
|
|
488
|
-
'port': parsed_url.port,
|
|
489
|
-
'request_headers': json.dumps(sanitized_headers),
|
|
490
|
-
'request_params': json.dumps(sanitized_params) if sanitized_params else None,
|
|
491
|
-
'request_body': json.dumps(sanitized_body) if sanitized_body else None,
|
|
492
|
-
'request_size': request_size,
|
|
493
|
-
'session_id': request.cookies.get('session_id'),
|
|
494
|
-
'user_id': getattr(request, 'current_user', {}).get('id') if hasattr(request, 'current_user') else None,
|
|
495
|
-
'auth_token': self.mask_token(request.headers.get('Authorization')),
|
|
496
|
-
'device_fingerprint': self.generate_device_fingerprint({
|
|
497
|
-
'user_agent': user_agent,
|
|
498
|
-
'request_headers': sanitized_headers
|
|
499
|
-
}),
|
|
500
|
-
'device_info': json.dumps(device_info),
|
|
501
|
-
'is_bot': device_info['is_bot'],
|
|
502
|
-
'is_mobile': device_info['is_mobile'],
|
|
503
|
-
'browser_name': device_info['browser_name'],
|
|
504
|
-
'browser_version': device_info['browser_version'],
|
|
505
|
-
'os_name': device_info['os_name'],
|
|
506
|
-
'os_version': device_info['os_version'],
|
|
487
|
+
'请求id': request_id,
|
|
488
|
+
'请求时间': datetime.now(),
|
|
489
|
+
'请求方法': request.method,
|
|
490
|
+
'接口路径': request.endpoint or request.path,
|
|
491
|
+
'客户端ip': client_ip,
|
|
492
|
+
'用户标识': user_id,
|
|
493
|
+
'用户代理': user_agent,
|
|
494
|
+
'请求参数': request_params,
|
|
507
495
|
}
|
|
508
496
|
|
|
509
497
|
return request_data
|
|
510
498
|
|
|
511
|
-
|
|
512
|
-
"""脱敏处理令牌"""
|
|
513
|
-
if not token:
|
|
514
|
-
return None
|
|
515
|
-
|
|
516
|
-
if len(token) <= 8:
|
|
517
|
-
return '*' * len(token)
|
|
518
|
-
|
|
519
|
-
if len(token) > 255:
|
|
520
|
-
return token[:10] + '*' * (min(len(token), 10)) + token[-10:]
|
|
521
|
-
|
|
522
|
-
return token[:4] + '*' * (len(token) - 8) + token[-4:]
|
|
499
|
+
# ==================== 数据持久化 ====================
|
|
523
500
|
|
|
524
501
|
def save_request_log(self, request_data: Dict[str, Any], response_data: Dict[str, Any] = None):
|
|
525
|
-
"""
|
|
526
|
-
|
|
502
|
+
"""
|
|
503
|
+
保存请求日志到数据库
|
|
504
|
+
|
|
505
|
+
Args:
|
|
506
|
+
request_data: 请求数据字典
|
|
507
|
+
response_data: 响应数据字典(可选)
|
|
508
|
+
"""
|
|
509
|
+
request_id = request_data.get('请求id', 'unknown')
|
|
527
510
|
|
|
528
511
|
try:
|
|
529
512
|
# 合并响应数据
|
|
@@ -533,92 +516,127 @@ class RouteMonitor:
|
|
|
533
516
|
connection = self.pool.connection()
|
|
534
517
|
try:
|
|
535
518
|
with connection.cursor() as cursor:
|
|
536
|
-
# 确保使用正确的数据库上下文
|
|
537
519
|
self.ensure_database_context(cursor)
|
|
538
520
|
|
|
539
521
|
# 插入请求日志
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
522
|
+
sql = """
|
|
523
|
+
INSERT INTO `api_访问日志` (
|
|
524
|
+
`请求id`, `请求时间`, `请求方法`, `接口路径`, `客户端ip`,
|
|
525
|
+
`响应状态码`, `响应耗时`, `用户标识`, `用户代理`, `请求参数`, `错误信息`
|
|
526
|
+
) VALUES (
|
|
527
|
+
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
528
|
+
)
|
|
546
529
|
"""
|
|
547
530
|
|
|
548
|
-
cursor.execute(sql,
|
|
531
|
+
cursor.execute(sql, (
|
|
532
|
+
request_data.get('请求id'),
|
|
533
|
+
request_data.get('请求时间'),
|
|
534
|
+
request_data.get('请求方法'),
|
|
535
|
+
request_data.get('接口路径'),
|
|
536
|
+
request_data.get('客户端ip'),
|
|
537
|
+
request_data.get('响应状态码'),
|
|
538
|
+
request_data.get('响应耗时'),
|
|
539
|
+
request_data.get('用户标识'),
|
|
540
|
+
request_data.get('用户代理'),
|
|
541
|
+
request_data.get('请求参数'),
|
|
542
|
+
request_data.get('错误信息'),
|
|
543
|
+
))
|
|
544
|
+
|
|
549
545
|
connection.commit()
|
|
550
546
|
|
|
551
547
|
finally:
|
|
552
548
|
connection.close()
|
|
553
549
|
|
|
554
550
|
except Exception as e:
|
|
555
|
-
#
|
|
556
|
-
# "请求ID": request_id,
|
|
557
|
-
# "错误信息": str(e),
|
|
558
|
-
# "错误类型": type(e).__name__,
|
|
559
|
-
# "影响": "日志丢失,但不影响主业务"
|
|
560
|
-
# })
|
|
561
|
-
# 静默处理错误,不影响主业务
|
|
551
|
+
# 静默处理错误,避免影响主业务
|
|
562
552
|
pass
|
|
563
553
|
|
|
564
554
|
def update_statistics(self, request_data: Dict[str, Any]):
|
|
565
|
-
"""
|
|
566
|
-
|
|
567
|
-
# endpoint = request_data.get('endpoint', '')
|
|
568
|
-
# status_code = request_data.get('response_status', 500)
|
|
555
|
+
"""
|
|
556
|
+
更新统计数据
|
|
569
557
|
|
|
558
|
+
包括:
|
|
559
|
+
1. 接口统计:按小时汇总接口性能数据
|
|
560
|
+
2. IP 统计:按日期汇总 IP 访问数据
|
|
561
|
+
|
|
562
|
+
Args:
|
|
563
|
+
request_data: 包含请求和响应信息的字典
|
|
564
|
+
"""
|
|
570
565
|
try:
|
|
571
566
|
connection = self.pool.connection()
|
|
572
567
|
try:
|
|
573
568
|
with connection.cursor() as cursor:
|
|
574
|
-
# 确保使用正确的数据库上下文
|
|
575
569
|
self.ensure_database_context(cursor)
|
|
576
570
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
571
|
+
# 使用请求时间而不是当前时间,避免统计时间不一致
|
|
572
|
+
request_time = request_data.get('请求时间', datetime.now())
|
|
573
|
+
date = request_time.date()
|
|
574
|
+
hour = request_time.hour
|
|
575
|
+
now = datetime.now() # 用于IP统计的最后访问时间
|
|
576
|
+
|
|
577
|
+
# 判断是否成功(状态码 < 400)
|
|
578
|
+
status_code = request_data.get('响应状态码', 500)
|
|
579
|
+
is_success = 1 if status_code < 400 else 0
|
|
580
|
+
is_error = 1 if status_code >= 400 else 0
|
|
581
|
+
response_time = request_data.get('响应耗时', 0)
|
|
580
582
|
|
|
581
|
-
#
|
|
583
|
+
# 1. 更新接口统计表
|
|
582
584
|
cursor.execute("""
|
|
583
|
-
INSERT INTO `
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
585
|
+
INSERT INTO `api_接口统计` (
|
|
586
|
+
`统计日期`, `统计小时`, `接口路径`, `请求方法`,
|
|
587
|
+
`请求总数`, `成功次数`, `失败次数`,
|
|
588
|
+
`平均耗时`, `最大耗时`, `最小耗时`
|
|
589
|
+
) VALUES (
|
|
590
|
+
%s, %s, %s, %s, 1, %s, %s, %s, %s, %s
|
|
591
|
+
) ON DUPLICATE KEY UPDATE
|
|
592
|
+
`请求总数` = `请求总数` + 1,
|
|
593
|
+
`成功次数` = `成功次数` + %s,
|
|
594
|
+
`失败次数` = `失败次数` + %s,
|
|
595
|
+
`平均耗时` = (
|
|
596
|
+
(`平均耗时` * `请求总数` + %s) / (`请求总数` + 1)
|
|
593
597
|
),
|
|
594
|
-
|
|
598
|
+
`最大耗时` = GREATEST(`最大耗时`, %s),
|
|
599
|
+
`最小耗时` = (
|
|
600
|
+
CASE
|
|
601
|
+
WHEN `最小耗时` IS NULL OR `最小耗时` = 0 THEN %s
|
|
602
|
+
ELSE LEAST(`最小耗时`, %s)
|
|
603
|
+
END
|
|
604
|
+
)
|
|
595
605
|
""", (
|
|
596
606
|
date, hour,
|
|
597
|
-
request_data.get('
|
|
598
|
-
request_data.get('
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
607
|
+
request_data.get('接口路径', ''),
|
|
608
|
+
request_data.get('请求方法', ''),
|
|
609
|
+
is_success, is_error,
|
|
610
|
+
response_time, response_time, response_time,
|
|
611
|
+
is_success, is_error,
|
|
612
|
+
response_time,
|
|
613
|
+
response_time,
|
|
614
|
+
response_time,
|
|
615
|
+
response_time
|
|
605
616
|
))
|
|
606
617
|
|
|
607
|
-
# 更新IP
|
|
618
|
+
# 2. 更新 IP 统计表
|
|
608
619
|
cursor.execute("""
|
|
609
|
-
INSERT INTO `
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
VALUES (
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
620
|
+
INSERT INTO `api_ip记录` (
|
|
621
|
+
`统计日期`, `客户端ip`, `请求总数`, `成功次数`, `失败次数`,
|
|
622
|
+
`平均耗时`, `首次访问`, `最后访问`
|
|
623
|
+
) VALUES (
|
|
624
|
+
%s, %s, 1, %s, %s, %s, %s, %s
|
|
625
|
+
) ON DUPLICATE KEY UPDATE
|
|
626
|
+
`请求总数` = `请求总数` + 1,
|
|
627
|
+
`成功次数` = `成功次数` + %s,
|
|
628
|
+
`失败次数` = `失败次数` + %s,
|
|
629
|
+
`平均耗时` = (
|
|
630
|
+
(`平均耗时` * `请求总数` + %s) / (`请求总数` + 1)
|
|
631
|
+
),
|
|
632
|
+
`最后访问` = %s
|
|
617
633
|
""", (
|
|
618
634
|
date,
|
|
619
|
-
request_data.get('
|
|
620
|
-
|
|
621
|
-
|
|
635
|
+
request_data.get('客户端ip', ''),
|
|
636
|
+
is_success, is_error,
|
|
637
|
+
response_time, now, now,
|
|
638
|
+
is_success, is_error,
|
|
639
|
+
response_time,
|
|
622
640
|
now
|
|
623
641
|
))
|
|
624
642
|
|
|
@@ -628,17 +646,37 @@ class RouteMonitor:
|
|
|
628
646
|
connection.close()
|
|
629
647
|
|
|
630
648
|
except Exception as e:
|
|
631
|
-
# logger.error("更新统计数据失败", {
|
|
632
|
-
# "请求ID": request_id,
|
|
633
|
-
# "错误信息": str(e),
|
|
634
|
-
# "错误类型": type(e).__name__,
|
|
635
|
-
# "影响": "统计数据缺失,但不影响主业务"
|
|
636
|
-
# })
|
|
637
649
|
# 静默处理错误
|
|
638
650
|
pass
|
|
651
|
+
|
|
652
|
+
# ==================== 核心装饰器 ====================
|
|
639
653
|
|
|
640
|
-
def
|
|
641
|
-
"""
|
|
654
|
+
def api_monitor(self, func):
|
|
655
|
+
"""
|
|
656
|
+
API 监控装饰器
|
|
657
|
+
|
|
658
|
+
使用方法:
|
|
659
|
+
```python
|
|
660
|
+
from mdbq.route.monitor import api_monitor
|
|
661
|
+
|
|
662
|
+
@app.route('/api/users')
|
|
663
|
+
@api_monitor
|
|
664
|
+
def get_users():
|
|
665
|
+
return {'users': [...]}
|
|
666
|
+
```
|
|
667
|
+
|
|
668
|
+
功能:
|
|
669
|
+
1. 自动记录请求的核心信息(IP、耗时、状态等)
|
|
670
|
+
2. 实时更新统计数据
|
|
671
|
+
3. 异常情况也会被记录
|
|
672
|
+
4. 不影响主业务逻辑的执行
|
|
673
|
+
|
|
674
|
+
Args:
|
|
675
|
+
func: 被装饰的函数
|
|
676
|
+
|
|
677
|
+
Returns:
|
|
678
|
+
装饰后的函数
|
|
679
|
+
"""
|
|
642
680
|
@functools.wraps(func)
|
|
643
681
|
def wrapper(*args, **kwargs):
|
|
644
682
|
# 记录开始时间
|
|
@@ -647,43 +685,56 @@ class RouteMonitor:
|
|
|
647
685
|
|
|
648
686
|
# 收集请求数据
|
|
649
687
|
request_data = self.collect_request_data(request)
|
|
650
|
-
request_id = request_data.get('
|
|
688
|
+
request_id = request_data.get('请求id', 'unknown')
|
|
689
|
+
|
|
690
|
+
# 统计总请求数(线程安全)
|
|
691
|
+
with self._stats_lock:
|
|
692
|
+
self._stats['total_requests'] += 1
|
|
651
693
|
|
|
652
694
|
try:
|
|
653
695
|
# 执行原函数
|
|
654
696
|
response = func(*args, **kwargs)
|
|
655
697
|
|
|
656
|
-
#
|
|
698
|
+
# 计算响应时间
|
|
657
699
|
end_time = time.time()
|
|
658
|
-
process_time = round((end_time - start_time) * 1000, 3)
|
|
700
|
+
process_time = round((end_time - start_time) * 1000, 3) # 毫秒
|
|
659
701
|
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
# 如果获取数据失败(如直通模式),使用默认值
|
|
674
|
-
response_size = -1 # 标记为无法获取大小
|
|
702
|
+
# 获取响应状态码
|
|
703
|
+
response_status = 200
|
|
704
|
+
if hasattr(response, 'status_code'):
|
|
705
|
+
response_status = response.status_code
|
|
706
|
+
elif isinstance(response, tuple) and len(response) > 1:
|
|
707
|
+
# 处理 (data, status_code) 格式的返回
|
|
708
|
+
response_status = response[1]
|
|
709
|
+
# 健壮化:支持 '200 OK' 等字符串状态
|
|
710
|
+
if isinstance(response_status, str):
|
|
711
|
+
try:
|
|
712
|
+
response_status = int(str(response_status).split()[0])
|
|
713
|
+
except Exception:
|
|
714
|
+
response_status = 200
|
|
675
715
|
|
|
716
|
+
# 更新响应数据
|
|
676
717
|
response_data = {
|
|
677
|
-
'
|
|
678
|
-
'
|
|
679
|
-
'response_size': response_size
|
|
718
|
+
'响应状态码': response_status,
|
|
719
|
+
'响应耗时': process_time,
|
|
680
720
|
}
|
|
681
|
-
# 保存日志
|
|
682
|
-
self.save_request_log(request_data, response_data)
|
|
683
721
|
|
|
684
|
-
#
|
|
722
|
+
# 合并数据
|
|
685
723
|
request_data.update(response_data)
|
|
686
|
-
|
|
724
|
+
|
|
725
|
+
# 【改进】优先使用 Redis 队列(非阻塞)
|
|
726
|
+
queued = self._push_to_queue({
|
|
727
|
+
'type': 'request_log',
|
|
728
|
+
'data': request_data,
|
|
729
|
+
'timestamp': time.time()
|
|
730
|
+
})
|
|
731
|
+
|
|
732
|
+
# 如果队列失败,降级为同步写入
|
|
733
|
+
if not queued:
|
|
734
|
+
with self._stats_lock:
|
|
735
|
+
self._stats['sync_writes'] += 1
|
|
736
|
+
self.save_request_log(request_data, None)
|
|
737
|
+
self.update_statistics(request_data)
|
|
687
738
|
|
|
688
739
|
return response
|
|
689
740
|
|
|
@@ -692,86 +743,125 @@ class RouteMonitor:
|
|
|
692
743
|
end_time = time.time()
|
|
693
744
|
process_time = round((end_time - start_time) * 1000, 3)
|
|
694
745
|
|
|
746
|
+
# 构建错误数据
|
|
695
747
|
error_data = {
|
|
696
|
-
'
|
|
697
|
-
'
|
|
698
|
-
'
|
|
699
|
-
'response_size': 0
|
|
748
|
+
'响应状态码': 500,
|
|
749
|
+
'响应耗时': process_time,
|
|
750
|
+
'错误信息': f"{type(e).__name__}: {str(e)}"
|
|
700
751
|
}
|
|
701
752
|
|
|
702
|
-
#
|
|
703
|
-
|
|
704
|
-
# "函数名": func.__name__,
|
|
705
|
-
# "错误信息": str(e),
|
|
706
|
-
# "错误类型": type(e).__name__,
|
|
707
|
-
# "处理时间": f"{process_time}ms",
|
|
708
|
-
# "结果": "异常"
|
|
709
|
-
# })
|
|
753
|
+
# 合并数据
|
|
754
|
+
request_data.update(error_data)
|
|
710
755
|
|
|
711
|
-
#
|
|
712
|
-
self.
|
|
756
|
+
# 【改进】优先使用 Redis 队列(非阻塞)
|
|
757
|
+
queued = self._push_to_queue({
|
|
758
|
+
'type': 'request_log',
|
|
759
|
+
'data': request_data,
|
|
760
|
+
'timestamp': time.time()
|
|
761
|
+
})
|
|
713
762
|
|
|
714
|
-
#
|
|
715
|
-
|
|
716
|
-
|
|
763
|
+
# 如果队列失败,降级为同步写入
|
|
764
|
+
if not queued:
|
|
765
|
+
with self._stats_lock:
|
|
766
|
+
self._stats['sync_writes'] += 1
|
|
767
|
+
self.save_request_log(request_data, None)
|
|
768
|
+
self.update_statistics(request_data)
|
|
717
769
|
|
|
718
|
-
#
|
|
719
|
-
raise
|
|
770
|
+
# 重新抛出异常,不影响原有错误处理逻辑
|
|
771
|
+
raise
|
|
720
772
|
|
|
721
773
|
return wrapper
|
|
722
774
|
|
|
775
|
+
|
|
776
|
+
# ==================== 数据查询与分析 ====================
|
|
777
|
+
|
|
723
778
|
def get_statistics_summary(self, days: int = 7) -> Dict[str, Any]:
|
|
724
|
-
"""
|
|
779
|
+
"""
|
|
780
|
+
获取统计摘要
|
|
781
|
+
|
|
782
|
+
提供指定天数内的 API 访问统计概览。
|
|
783
|
+
|
|
784
|
+
Args:
|
|
785
|
+
days: 统计天数,默认 7 天
|
|
786
|
+
|
|
787
|
+
Returns:
|
|
788
|
+
dict: 包含以下内容的统计摘要:
|
|
789
|
+
- 统计周期
|
|
790
|
+
- 总体统计(总请求数、成功率、平均耗时等)
|
|
791
|
+
- 热门接口 TOP 10
|
|
792
|
+
- IP 统计
|
|
793
|
+
"""
|
|
725
794
|
try:
|
|
726
795
|
connection = self.pool.connection()
|
|
727
796
|
try:
|
|
728
797
|
with connection.cursor() as cursor:
|
|
729
|
-
# 确保使用正确的数据库上下文
|
|
730
798
|
self.ensure_database_context(cursor)
|
|
731
799
|
|
|
732
800
|
end_date = datetime.now().date()
|
|
733
801
|
start_date = end_date - timedelta(days=days)
|
|
734
802
|
|
|
735
|
-
# 总体统计
|
|
803
|
+
# 1. 总体统计
|
|
736
804
|
cursor.execute("""
|
|
737
805
|
SELECT
|
|
738
|
-
SUM(
|
|
739
|
-
SUM(
|
|
740
|
-
SUM(
|
|
741
|
-
AVG(
|
|
742
|
-
COUNT(DISTINCT
|
|
743
|
-
FROM
|
|
744
|
-
WHERE
|
|
806
|
+
SUM(请求总数) as 总请求数,
|
|
807
|
+
SUM(成功次数) as 成功次数,
|
|
808
|
+
SUM(失败次数) as 失败次数,
|
|
809
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时,
|
|
810
|
+
COUNT(DISTINCT 接口路径) as 接口数量
|
|
811
|
+
FROM api_接口统计
|
|
812
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
745
813
|
""", (start_date, end_date))
|
|
746
814
|
|
|
747
815
|
summary = cursor.fetchone() or {}
|
|
748
|
-
|
|
816
|
+
|
|
817
|
+
# 2. 热门接口 TOP 10
|
|
749
818
|
cursor.execute("""
|
|
750
|
-
SELECT
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
819
|
+
SELECT
|
|
820
|
+
接口路径,
|
|
821
|
+
SUM(请求总数) as 请求次数,
|
|
822
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时
|
|
823
|
+
FROM api_接口统计
|
|
824
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
825
|
+
GROUP BY 接口路径
|
|
826
|
+
ORDER BY 请求次数 DESC
|
|
755
827
|
LIMIT 10
|
|
756
828
|
""", (start_date, end_date))
|
|
757
829
|
|
|
758
830
|
top_endpoints = cursor.fetchall()
|
|
759
831
|
|
|
760
|
-
#
|
|
832
|
+
# 3. IP 统计
|
|
761
833
|
cursor.execute("""
|
|
762
|
-
SELECT
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
834
|
+
SELECT
|
|
835
|
+
COUNT(DISTINCT 客户端ip) as 独立ip数,
|
|
836
|
+
SUM(请求总数) as ip总请求数
|
|
837
|
+
FROM api_ip记录
|
|
838
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
766
839
|
""", (start_date, end_date))
|
|
767
840
|
|
|
768
841
|
ip_stats = cursor.fetchone() or {}
|
|
769
842
|
|
|
843
|
+
# 4. 性能最慢的接口 TOP 5
|
|
844
|
+
cursor.execute("""
|
|
845
|
+
SELECT
|
|
846
|
+
接口路径,
|
|
847
|
+
ROUND(MAX(最大耗时), 2) as 最大耗时,
|
|
848
|
+
ROUND(AVG(平均耗时), 2) as 平均耗时
|
|
849
|
+
FROM api_接口统计
|
|
850
|
+
WHERE 统计日期 BETWEEN %s AND %s
|
|
851
|
+
GROUP BY 接口路径
|
|
852
|
+
ORDER BY 最大耗时 DESC
|
|
853
|
+
LIMIT 5
|
|
854
|
+
""", (start_date, end_date))
|
|
855
|
+
|
|
856
|
+
slow_endpoints = cursor.fetchall()
|
|
857
|
+
|
|
858
|
+
# 构建返回结果
|
|
770
859
|
result = {
|
|
771
|
-
'
|
|
772
|
-
'
|
|
773
|
-
'
|
|
774
|
-
'
|
|
860
|
+
'统计周期': f'{start_date} 至 {end_date}',
|
|
861
|
+
'总体统计': summary,
|
|
862
|
+
'热门接口': top_endpoints,
|
|
863
|
+
'ip统计': ip_stats,
|
|
864
|
+
'慢接口': slow_endpoints
|
|
775
865
|
}
|
|
776
866
|
|
|
777
867
|
return result
|
|
@@ -780,26 +870,131 @@ class RouteMonitor:
|
|
|
780
870
|
connection.close()
|
|
781
871
|
|
|
782
872
|
except Exception as e:
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
873
|
+
return {'错误': str(e)}
|
|
874
|
+
|
|
875
|
+
def cleanup_old_data(self, days_to_keep: int = 30) -> Dict[str, int]:
|
|
876
|
+
"""
|
|
877
|
+
清理历史数据
|
|
878
|
+
|
|
879
|
+
删除指定天数之前的访问日志,保留统计数据。
|
|
880
|
+
建议定期执行(如每天凌晨)以控制数据库大小。
|
|
881
|
+
|
|
882
|
+
Args:
|
|
883
|
+
days_to_keep: 保留最近多少天的数据,默认 30 天
|
|
884
|
+
|
|
885
|
+
Returns:
|
|
886
|
+
dict: 清理结果,包含删除的记录数
|
|
887
|
+
"""
|
|
888
|
+
try:
|
|
889
|
+
connection = self.pool.connection()
|
|
890
|
+
try:
|
|
891
|
+
with connection.cursor() as cursor:
|
|
892
|
+
self.ensure_database_context(cursor)
|
|
893
|
+
|
|
894
|
+
cutoff_date = datetime.now().date() - timedelta(days=days_to_keep)
|
|
895
|
+
|
|
896
|
+
# 1. 清理访问日志表
|
|
897
|
+
cursor.execute("""
|
|
898
|
+
DELETE FROM api_访问日志
|
|
899
|
+
WHERE 请求时间 < %s
|
|
900
|
+
""", (cutoff_date,))
|
|
901
|
+
|
|
902
|
+
deleted_logs = cursor.rowcount
|
|
903
|
+
|
|
904
|
+
# 2. 清理 ip 记录表(可选,通常保留更久)
|
|
905
|
+
cursor.execute("""
|
|
906
|
+
DELETE FROM api_ip记录
|
|
907
|
+
WHERE 统计日期 < %s
|
|
908
|
+
""", (cutoff_date,))
|
|
909
|
+
|
|
910
|
+
deleted_ip_records = cursor.rowcount
|
|
911
|
+
|
|
912
|
+
connection.commit()
|
|
913
|
+
|
|
914
|
+
result = {
|
|
915
|
+
'删除日志数': deleted_logs,
|
|
916
|
+
'删除ip记录数': deleted_ip_records,
|
|
917
|
+
'清理日期': str(cutoff_date)
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
return result
|
|
921
|
+
|
|
922
|
+
finally:
|
|
923
|
+
connection.close()
|
|
924
|
+
|
|
925
|
+
except Exception as e:
|
|
926
|
+
return {'错误': str(e)}
|
|
927
|
+
|
|
928
|
+
def consume_queue_tasks(self, batch_size: int = 100, timeout: float = 1.0):
|
|
929
|
+
"""
|
|
930
|
+
从 Redis 队列中消费任务并写入数据库(用于单独的消费者进程)
|
|
931
|
+
|
|
932
|
+
这个方法应该在单独的进程中循环调用,不要在 uwsgi worker 中调用!
|
|
933
|
+
|
|
934
|
+
Args:
|
|
935
|
+
batch_size: 每次处理的最大任务数
|
|
936
|
+
timeout: 从队列获取任务的超时时间(秒)
|
|
937
|
+
|
|
938
|
+
Returns:
|
|
939
|
+
int: 处理的任务数量
|
|
940
|
+
"""
|
|
941
|
+
if not self.enable_async:
|
|
942
|
+
return 0
|
|
943
|
+
|
|
944
|
+
processed = 0
|
|
945
|
+
|
|
946
|
+
try:
|
|
947
|
+
# 批量从队列中取出任务(BRPOP,从右侧阻塞弹出)
|
|
948
|
+
for _ in range(batch_size):
|
|
949
|
+
result = self.redis_client.brpop(self.queue_name, timeout=timeout)
|
|
950
|
+
if not result:
|
|
951
|
+
break # 队列为空
|
|
952
|
+
|
|
953
|
+
_, task_json = result
|
|
954
|
+
|
|
955
|
+
try:
|
|
956
|
+
# (Redis 可能返回 bytes)
|
|
957
|
+
if isinstance(task_json, bytes):
|
|
958
|
+
task_json = task_json.decode('utf-8')
|
|
959
|
+
|
|
960
|
+
# 解析任务数据,还原 datetime 对象(使用 datetime 解码器)
|
|
961
|
+
task = json.loads(task_json, object_hook=self._datetime_decoder)
|
|
962
|
+
task_type = task.get('type')
|
|
963
|
+
task_data = task.get('data', {})
|
|
964
|
+
|
|
965
|
+
# 处理不同类型的任务
|
|
966
|
+
if task_type == 'request_log':
|
|
967
|
+
# 写入日志和统计
|
|
968
|
+
self.save_request_log(task_data, None)
|
|
969
|
+
self.update_statistics(task_data)
|
|
970
|
+
processed += 1
|
|
971
|
+
|
|
972
|
+
except Exception as e:
|
|
973
|
+
# 单个任务失败不影响其他任务
|
|
974
|
+
pass
|
|
975
|
+
|
|
976
|
+
return processed
|
|
977
|
+
|
|
978
|
+
except Exception as e:
|
|
979
|
+
# 静默处理错误
|
|
980
|
+
return processed
|
|
981
|
+
|
|
790
982
|
|
|
983
|
+
# # ==================== 全局实例与导出 ====================
|
|
791
984
|
|
|
792
|
-
#
|
|
793
|
-
route_monitor = RouteMonitor()
|
|
985
|
+
# # 创建全局监控实例
|
|
986
|
+
# route_monitor = RouteMonitor()
|
|
794
987
|
|
|
795
|
-
#
|
|
796
|
-
|
|
988
|
+
# # 导出核心装饰器(推荐使用此方式)
|
|
989
|
+
# api_monitor = route_monitor.api_monitor
|
|
797
990
|
|
|
798
|
-
# 导出其他有用的函数
|
|
799
|
-
def get_request_id():
|
|
800
|
-
"""获取当前请求ID"""
|
|
801
|
-
return getattr(g, 'request_id', None)
|
|
802
991
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
992
|
+
# ==================== 模块导出列表 ====================
|
|
993
|
+
__all__ = [
|
|
994
|
+
'RouteMonitor',
|
|
995
|
+
'api_monitor',
|
|
996
|
+
'get_request_id',
|
|
997
|
+
'get_statistics_summary',
|
|
998
|
+
'cleanup_old_data',
|
|
999
|
+
'get_async_stats',
|
|
1000
|
+
]
|