mdbq 4.0.81__tar.gz → 4.0.83__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mdbq-4.0.81 → mdbq-4.0.83}/PKG-INFO +1 -1
- mdbq-4.0.83/mdbq/__version__.py +1 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/route/monitor.py +269 -27
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq.egg-info/PKG-INFO +1 -1
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq.egg-info/SOURCES.txt +0 -1
- mdbq-4.0.81/mdbq/__version__.py +0 -1
- mdbq-4.0.81/mdbq/route/example.py +0 -378
- {mdbq-4.0.81 → mdbq-4.0.83}/README.txt +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/log/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/log/mylogger.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/myconf/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/myconf/myconf.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/deduplicator.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/mysql.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/s_query.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/unique_.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/mysql/uploader.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/download_sku_picture.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/error_handler.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/otk.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/pov_city.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/other/ua_sj.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/pbix/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/pbix/pbix_refresh.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/pbix/refresh_all.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/redis/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/redis/getredis.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/route/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/route/analytics.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/route/routes.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/selenium/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/selenium/get_driver.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq/spider/__init__.py +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq.egg-info/dependency_links.txt +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/mdbq.egg-info/top_level.txt +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/setup.cfg +0 -0
- {mdbq-4.0.81 → mdbq-4.0.83}/setup.py +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
VERSION = '4.0.83'
|
@@ -22,6 +22,7 @@ from typing import Dict, Any, Optional, List
|
|
22
22
|
from urllib.parse import urlparse, parse_qs
|
23
23
|
from dbutils.pooled_db import PooledDB
|
24
24
|
from mdbq.myconf import myconf
|
25
|
+
from mdbq.log import mylogger
|
25
26
|
from flask import request, g
|
26
27
|
import re
|
27
28
|
import ipaddress
|
@@ -33,41 +34,89 @@ host, port, username, password = parser.get_section_values(
|
|
33
34
|
keys=['host', 'port', 'username', 'password'],
|
34
35
|
)
|
35
36
|
|
37
|
+
logger = mylogger.MyLogger(
|
38
|
+
logging_mode='file',
|
39
|
+
log_level='debug',
|
40
|
+
log_format='json',
|
41
|
+
max_log_size=50,
|
42
|
+
backup_count=5,
|
43
|
+
enable_async=False, # 是否启用异步日志
|
44
|
+
sample_rate=1, # 采样DEBUG/INFO日志
|
45
|
+
sensitive_fields=[], # 敏感字段过滤
|
46
|
+
enable_metrics=False, # 是否启用性能指标
|
47
|
+
)
|
48
|
+
|
36
49
|
|
37
50
|
class RouteMonitor:
|
38
51
|
"""路由监控核心类"""
|
39
52
|
|
40
|
-
def __init__(self,
|
53
|
+
def __init__(self, database='api_monitor_logs'):
|
41
54
|
"""初始化监控系统"""
|
42
|
-
|
43
|
-
|
44
|
-
else:
|
45
|
-
self.init_database_pool()
|
55
|
+
self.database = database
|
56
|
+
self.init_database_pool()
|
46
57
|
self.init_database_tables()
|
47
|
-
|
58
|
+
|
48
59
|
def init_database_pool(self):
|
49
60
|
"""初始化数据库连接池"""
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
61
|
+
try:
|
62
|
+
self.pool = PooledDB(
|
63
|
+
creator=pymysql,
|
64
|
+
maxconnections=2, # 监控系统连接数较小
|
65
|
+
mincached=1,
|
66
|
+
maxcached=2,
|
67
|
+
blocking=True,
|
68
|
+
host=host,
|
69
|
+
port=int(port),
|
70
|
+
user=username,
|
71
|
+
password=password,
|
72
|
+
ping=1,
|
73
|
+
charset='utf8mb4',
|
74
|
+
cursorclass=pymysql.cursors.DictCursor
|
75
|
+
)
|
76
|
+
|
77
|
+
# 创建数据库并切换
|
78
|
+
connection = self.pool.connection()
|
79
|
+
try:
|
80
|
+
with connection.cursor() as cursor:
|
81
|
+
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.database}` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci")
|
82
|
+
cursor.execute(f"USE `{self.database}`")
|
83
|
+
finally:
|
84
|
+
connection.close()
|
85
|
+
|
86
|
+
except Exception as e:
|
87
|
+
logger.error("❌ 数据库连接池初始化失败", {
|
88
|
+
"错误信息": str(e),
|
89
|
+
"数据库": self.database
|
90
|
+
})
|
91
|
+
raise
|
92
|
+
|
93
|
+
def ensure_database_context(self, cursor):
|
94
|
+
"""确保当前游标处于正确的数据库上下文中"""
|
95
|
+
try:
|
96
|
+
cursor.execute(f"USE `{self.database}`")
|
97
|
+
except Exception as e:
|
98
|
+
logger.warning("切换数据库上下文失败,尝试重新创建", {
|
99
|
+
"数据库": self.database,
|
100
|
+
"错误": str(e)
|
101
|
+
})
|
102
|
+
cursor.execute(f"CREATE DATABASE IF NOT EXISTS `{self.database}` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci")
|
103
|
+
cursor.execute(f"USE `{self.database}`")
|
64
104
|
|
65
105
|
def init_database_tables(self):
|
66
106
|
"""初始化数据库表结构"""
|
67
107
|
try:
|
108
|
+
logger.debug("🗄️ 开始创建/验证数据库表结构", {
|
109
|
+
"操作": "表结构初始化",
|
110
|
+
"预期表数": 4,
|
111
|
+
"数据库": self.database
|
112
|
+
})
|
113
|
+
|
68
114
|
connection = self.pool.connection()
|
69
115
|
try:
|
70
116
|
with connection.cursor() as cursor:
|
117
|
+
# 确保使用正确的数据库上下文
|
118
|
+
self.ensure_database_context(cursor)
|
119
|
+
|
71
120
|
# 创建详细请求记录表 - 修复MySQL 8.4+兼容性
|
72
121
|
cursor.execute("""
|
73
122
|
CREATE TABLE IF NOT EXISTS `api_request_logs` (
|
@@ -120,7 +169,6 @@ class RouteMonitor:
|
|
120
169
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
121
170
|
COMMENT='API请求详细日志表';
|
122
171
|
""")
|
123
|
-
|
124
172
|
# 创建访问统计汇总表
|
125
173
|
cursor.execute("""
|
126
174
|
CREATE TABLE IF NOT EXISTS `api_access_statistics` (
|
@@ -178,7 +226,6 @@ class RouteMonitor:
|
|
178
226
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
179
227
|
COMMENT='IP访问统计表';
|
180
228
|
""")
|
181
|
-
|
182
229
|
# 创建系统性能统计表
|
183
230
|
cursor.execute("""
|
184
231
|
CREATE TABLE IF NOT EXISTS `system_performance_stats` (
|
@@ -197,11 +244,24 @@ class RouteMonitor:
|
|
197
244
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
198
245
|
COMMENT='系统性能统计表';
|
199
246
|
""")
|
200
|
-
|
201
247
|
connection.commit()
|
248
|
+
logger.debug("🎯 所有数据库表结构初始化完成", {
|
249
|
+
"创建表数": 4,
|
250
|
+
"操作状态": "成功",
|
251
|
+
"数据库": self.database,
|
252
|
+
"数据库引擎": "InnoDB"
|
253
|
+
})
|
254
|
+
|
202
255
|
finally:
|
203
256
|
connection.close()
|
257
|
+
|
204
258
|
except Exception as e:
|
259
|
+
logger.error("❌ 数据库表结构初始化失败", {
|
260
|
+
"错误信息": str(e),
|
261
|
+
"错误类型": type(e).__name__,
|
262
|
+
"数据库": self.database,
|
263
|
+
"影响": "监控系统可能无法正常工作"
|
264
|
+
})
|
205
265
|
# 静默处理初始化错误,避免影响主应用
|
206
266
|
pass
|
207
267
|
|
@@ -376,9 +436,24 @@ class RouteMonitor:
|
|
376
436
|
# 设置请求ID到全局变量中,供后续使用
|
377
437
|
g.request_id = request_id
|
378
438
|
|
439
|
+
logger.debug("📋 开始收集请求数据", {
|
440
|
+
"请求ID": request_id,
|
441
|
+
"请求方法": request.method,
|
442
|
+
"端点": request.endpoint or request.path,
|
443
|
+
"客户端IP": request.remote_addr
|
444
|
+
})
|
445
|
+
|
379
446
|
# 获取真实IP
|
380
447
|
real_ip, forwarded_ips = self.get_real_ip(request)
|
381
448
|
|
449
|
+
if real_ip != request.remote_addr:
|
450
|
+
logger.debug("🔍 检测到IP转发", {
|
451
|
+
"请求ID": request_id,
|
452
|
+
"原始IP": request.remote_addr,
|
453
|
+
"真实IP": real_ip,
|
454
|
+
"转发链长度": len(forwarded_ips) if forwarded_ips else 0
|
455
|
+
})
|
456
|
+
|
382
457
|
# 获取请求头信息
|
383
458
|
headers = dict(request.headers)
|
384
459
|
sanitized_headers = self.sanitize_data(headers)
|
@@ -405,11 +480,25 @@ class RouteMonitor:
|
|
405
480
|
request_body = body_data.decode('utf-8')
|
406
481
|
except UnicodeDecodeError:
|
407
482
|
request_body = f"[BINARY_DATA:{len(body_data)}_bytes]"
|
483
|
+
logger.debug("📁 检测到二进制数据", {
|
484
|
+
"请求ID": request_id,
|
485
|
+
"数据大小": len(body_data),
|
486
|
+
"处理方式": "标记为二进制"
|
487
|
+
})
|
408
488
|
|
409
489
|
if request_body:
|
410
490
|
request_size = len(str(request_body).encode('utf-8'))
|
411
|
-
|
491
|
+
logger.debug("📊 请求体信息", {
|
492
|
+
"请求ID": request_id,
|
493
|
+
"数据类型": "JSON" if request.is_json else "表单" if request.form else "文本",
|
494
|
+
"大小": f"{request_size} bytes"
|
495
|
+
})
|
496
|
+
except Exception as e:
|
412
497
|
request_body = "[ERROR_READING_BODY]"
|
498
|
+
logger.warning("⚠️ 读取请求体失败", {
|
499
|
+
"请求ID": request_id,
|
500
|
+
"错误": str(e)
|
501
|
+
})
|
413
502
|
|
414
503
|
# 清理敏感数据
|
415
504
|
sanitized_body = self.sanitize_data(request_body)
|
@@ -419,6 +508,20 @@ class RouteMonitor:
|
|
419
508
|
user_agent = request.headers.get('User-Agent', '')
|
420
509
|
device_info = self.extract_device_info(user_agent)
|
421
510
|
|
511
|
+
if device_info['is_bot']:
|
512
|
+
logger.debug("🤖 检测到机器人请求", {
|
513
|
+
"请求ID": request_id,
|
514
|
+
"用户代理": user_agent[:100] + "..." if len(user_agent) > 100 else user_agent,
|
515
|
+
"IP": real_ip
|
516
|
+
})
|
517
|
+
|
518
|
+
if device_info['is_mobile']:
|
519
|
+
logger.debug("📱 检测到移动设备请求", {
|
520
|
+
"请求ID": request_id,
|
521
|
+
"操作系统": device_info['os_name'],
|
522
|
+
"浏览器": device_info['browser_name']
|
523
|
+
})
|
524
|
+
|
422
525
|
# URL解析
|
423
526
|
parsed_url = urlparse(request.url)
|
424
527
|
|
@@ -457,6 +560,14 @@ class RouteMonitor:
|
|
457
560
|
'os_version': device_info['os_version'],
|
458
561
|
}
|
459
562
|
|
563
|
+
logger.debug("✅ 请求数据收集完成", {
|
564
|
+
"请求ID": request_id,
|
565
|
+
"数据字段数": len(request_data),
|
566
|
+
"请求大小": f"{request_size} bytes",
|
567
|
+
"设备类型": "移动" if device_info['is_mobile'] else "桌面",
|
568
|
+
"是否机器人": device_info['is_bot']
|
569
|
+
})
|
570
|
+
|
460
571
|
return request_data
|
461
572
|
|
462
573
|
def mask_token(self, token: str) -> str:
|
@@ -471,14 +582,26 @@ class RouteMonitor:
|
|
471
582
|
|
472
583
|
def save_request_log(self, request_data: Dict[str, Any], response_data: Dict[str, Any] = None):
|
473
584
|
"""保存请求日志到数据库"""
|
585
|
+
request_id = request_data.get('request_id', 'unknown')
|
586
|
+
|
474
587
|
try:
|
475
588
|
# 合并响应数据
|
476
589
|
if response_data:
|
477
590
|
request_data.update(response_data)
|
478
591
|
|
592
|
+
logger.debug("💾 保存请求日志到数据库", {
|
593
|
+
"请求ID": request_id,
|
594
|
+
"端点": request_data.get('endpoint', ''),
|
595
|
+
"状态码": request_data.get('response_status', '未知'),
|
596
|
+
"处理时间": f"{request_data.get('process_time', 0)}ms"
|
597
|
+
})
|
598
|
+
|
479
599
|
connection = self.pool.connection()
|
480
600
|
try:
|
481
601
|
with connection.cursor() as cursor:
|
602
|
+
# 确保使用正确的数据库上下文
|
603
|
+
self.ensure_database_context(cursor)
|
604
|
+
|
482
605
|
# 插入请求日志
|
483
606
|
columns = ', '.join([f"`{key}`" for key in request_data.keys()])
|
484
607
|
placeholders = ', '.join(['%s'] * len(request_data))
|
@@ -490,19 +613,46 @@ class RouteMonitor:
|
|
490
613
|
|
491
614
|
cursor.execute(sql, list(request_data.values()))
|
492
615
|
connection.commit()
|
616
|
+
|
617
|
+
logger.debug("✅ 请求日志保存成功", {
|
618
|
+
"请求ID": request_id,
|
619
|
+
"写入状态": "成功",
|
620
|
+
"数据库表": "api_request_logs"
|
621
|
+
})
|
622
|
+
|
493
623
|
finally:
|
494
624
|
connection.close()
|
495
625
|
|
496
626
|
except Exception as e:
|
627
|
+
logger.error("❌ 保存请求日志失败", {
|
628
|
+
"请求ID": request_id,
|
629
|
+
"错误信息": str(e),
|
630
|
+
"错误类型": type(e).__name__,
|
631
|
+
"影响": "日志丢失,但不影响主业务"
|
632
|
+
})
|
497
633
|
# 静默处理错误,不影响主业务
|
498
634
|
pass
|
499
635
|
|
500
636
|
def update_statistics(self, request_data: Dict[str, Any]):
|
501
637
|
"""更新统计数据"""
|
638
|
+
request_id = request_data.get('request_id', 'unknown')
|
639
|
+
endpoint = request_data.get('endpoint', '')
|
640
|
+
status_code = request_data.get('response_status', 500)
|
641
|
+
|
502
642
|
try:
|
643
|
+
logger.debug("📈 更新统计数据", {
|
644
|
+
"请求ID": request_id,
|
645
|
+
"端点": endpoint,
|
646
|
+
"状态码": status_code,
|
647
|
+
"统计类型": "API访问统计和IP统计"
|
648
|
+
})
|
649
|
+
|
503
650
|
connection = self.pool.connection()
|
504
651
|
try:
|
505
652
|
with connection.cursor() as cursor:
|
653
|
+
# 确保使用正确的数据库上下文
|
654
|
+
self.ensure_database_context(cursor)
|
655
|
+
|
506
656
|
now = datetime.now()
|
507
657
|
date = now.date()
|
508
658
|
hour = now.hour
|
@@ -552,10 +702,24 @@ class RouteMonitor:
|
|
552
702
|
))
|
553
703
|
|
554
704
|
connection.commit()
|
705
|
+
|
706
|
+
logger.debug("✅ 统计数据更新成功", {
|
707
|
+
"请求ID": request_id,
|
708
|
+
"更新表": "api_access_statistics, ip_access_statistics",
|
709
|
+
"日期": str(date),
|
710
|
+
"小时": hour
|
711
|
+
})
|
712
|
+
|
555
713
|
finally:
|
556
714
|
connection.close()
|
557
715
|
|
558
716
|
except Exception as e:
|
717
|
+
logger.error("❌ 更新统计数据失败", {
|
718
|
+
"请求ID": request_id,
|
719
|
+
"错误信息": str(e),
|
720
|
+
"错误类型": type(e).__name__,
|
721
|
+
"影响": "统计数据缺失,但不影响主业务"
|
722
|
+
})
|
559
723
|
# 静默处理错误
|
560
724
|
pass
|
561
725
|
|
@@ -569,6 +733,15 @@ class RouteMonitor:
|
|
569
733
|
|
570
734
|
# 收集请求数据
|
571
735
|
request_data = self.collect_request_data(request)
|
736
|
+
request_id = request_data.get('request_id', 'unknown')
|
737
|
+
|
738
|
+
logger.debug("🎯 开始监控请求", {
|
739
|
+
"请求ID": request_id,
|
740
|
+
"函数名": func.__name__,
|
741
|
+
"端点": request_data.get('endpoint', ''),
|
742
|
+
"方法": request_data.get('method', ''),
|
743
|
+
"来源IP": request_data.get('real_ip', request_data.get('client_ip'))
|
744
|
+
})
|
572
745
|
|
573
746
|
try:
|
574
747
|
# 执行原函数
|
@@ -578,12 +751,24 @@ class RouteMonitor:
|
|
578
751
|
end_time = time.time()
|
579
752
|
process_time = round((end_time - start_time) * 1000, 3)
|
580
753
|
|
754
|
+
response_status = getattr(response, 'status_code', 200) if hasattr(response, 'status_code') else 200
|
755
|
+
response_size = len(str(response.get_data() if hasattr(response, 'get_data') else ''))
|
756
|
+
|
581
757
|
response_data = {
|
582
|
-
'response_status':
|
758
|
+
'response_status': response_status,
|
583
759
|
'process_time': process_time,
|
584
|
-
'response_size':
|
760
|
+
'response_size': response_size
|
585
761
|
}
|
586
762
|
|
763
|
+
logger.debug("✅ 请求处理完成", {
|
764
|
+
"请求ID": request_id,
|
765
|
+
"函数名": func.__name__,
|
766
|
+
"状态码": response_status,
|
767
|
+
"处理时间": f"{process_time}ms",
|
768
|
+
"响应大小": f"{response_size} bytes",
|
769
|
+
"结果": "成功"
|
770
|
+
})
|
771
|
+
|
587
772
|
# 保存日志
|
588
773
|
self.save_request_log(request_data, response_data)
|
589
774
|
|
@@ -605,6 +790,15 @@ class RouteMonitor:
|
|
605
790
|
'response_size': 0
|
606
791
|
}
|
607
792
|
|
793
|
+
logger.error("❌ 请求处理异常", {
|
794
|
+
"请求ID": request_id,
|
795
|
+
"函数名": func.__name__,
|
796
|
+
"错误信息": str(e),
|
797
|
+
"错误类型": type(e).__name__,
|
798
|
+
"处理时间": f"{process_time}ms",
|
799
|
+
"结果": "异常"
|
800
|
+
})
|
801
|
+
|
608
802
|
# 保存错误日志
|
609
803
|
self.save_request_log(request_data, error_data)
|
610
804
|
|
@@ -620,12 +814,26 @@ class RouteMonitor:
|
|
620
814
|
def get_statistics_summary(self, days: int = 7) -> Dict[str, Any]:
|
621
815
|
"""获取统计摘要"""
|
622
816
|
try:
|
817
|
+
logger.debug("📊 开始获取统计摘要", {
|
818
|
+
"查询天数": days,
|
819
|
+
"操作": "统计数据查询"
|
820
|
+
})
|
821
|
+
|
623
822
|
connection = self.pool.connection()
|
624
823
|
try:
|
625
824
|
with connection.cursor() as cursor:
|
825
|
+
# 确保使用正确的数据库上下文
|
826
|
+
self.ensure_database_context(cursor)
|
827
|
+
|
626
828
|
end_date = datetime.now().date()
|
627
829
|
start_date = end_date - timedelta(days=days)
|
628
830
|
|
831
|
+
logger.debug("📅 统计查询时间范围", {
|
832
|
+
"开始日期": str(start_date),
|
833
|
+
"结束日期": str(end_date),
|
834
|
+
"查询天数": days
|
835
|
+
})
|
836
|
+
|
629
837
|
# 总体统计
|
630
838
|
cursor.execute("""
|
631
839
|
SELECT
|
@@ -640,6 +848,14 @@ class RouteMonitor:
|
|
640
848
|
|
641
849
|
summary = cursor.fetchone() or {}
|
642
850
|
|
851
|
+
logger.debug("📈 总体统计查询完成", {
|
852
|
+
"总请求数": summary.get('total_requests', 0) or 0,
|
853
|
+
"成功请求数": summary.get('success_requests', 0) or 0,
|
854
|
+
"错误请求数": summary.get('error_requests', 0) or 0,
|
855
|
+
"平均响应时间": f"{(summary.get('avg_response_time', 0) or 0):.2f}ms",
|
856
|
+
"唯一端点数": summary.get('unique_endpoints', 0) or 0
|
857
|
+
})
|
858
|
+
|
643
859
|
# 热门端点
|
644
860
|
cursor.execute("""
|
645
861
|
SELECT endpoint, SUM(total_requests) as requests
|
@@ -652,6 +868,12 @@ class RouteMonitor:
|
|
652
868
|
|
653
869
|
top_endpoints = cursor.fetchall()
|
654
870
|
|
871
|
+
logger.debug("🔥 热门端点查询完成", {
|
872
|
+
"查询结果数": len(top_endpoints),
|
873
|
+
"最热门端点": top_endpoints[0]['endpoint'] if top_endpoints else "无数据",
|
874
|
+
"最高请求数": top_endpoints[0]['requests'] if top_endpoints else 0
|
875
|
+
})
|
876
|
+
|
655
877
|
# 活跃IP统计
|
656
878
|
cursor.execute("""
|
657
879
|
SELECT COUNT(DISTINCT ip_address) as unique_ips,
|
@@ -662,16 +884,36 @@ class RouteMonitor:
|
|
662
884
|
|
663
885
|
ip_stats = cursor.fetchone() or {}
|
664
886
|
|
665
|
-
|
887
|
+
logger.debug("🌐 IP统计查询完成", {
|
888
|
+
"唯一IP数": ip_stats.get('unique_ips', 0),
|
889
|
+
"IP总请求数": ip_stats.get('total_ip_requests', 0)
|
890
|
+
})
|
891
|
+
|
892
|
+
result = {
|
666
893
|
'period': f'{start_date} to {end_date}',
|
667
894
|
'summary': summary,
|
668
895
|
'top_endpoints': top_endpoints,
|
669
896
|
'ip_statistics': ip_stats
|
670
897
|
}
|
898
|
+
|
899
|
+
logger.debug("✅ 统计摘要获取成功", {
|
900
|
+
"查询天数": days,
|
901
|
+
"数据完整性": "完整",
|
902
|
+
"结果状态": "成功"
|
903
|
+
})
|
904
|
+
|
905
|
+
return result
|
906
|
+
|
671
907
|
finally:
|
672
908
|
connection.close()
|
673
909
|
|
674
910
|
except Exception as e:
|
911
|
+
logger.error("❌ 获取统计摘要失败", {
|
912
|
+
"查询天数": days,
|
913
|
+
"错误信息": str(e),
|
914
|
+
"错误类型": type(e).__name__,
|
915
|
+
"影响": "统计摘要不可用"
|
916
|
+
})
|
675
917
|
return {'error': str(e)}
|
676
918
|
|
677
919
|
|
mdbq-4.0.81/mdbq/__version__.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
VERSION = '4.0.81'
|
@@ -1,378 +0,0 @@
|
|
1
|
-
"""
|
2
|
-
路由监控系统集成示例
|
3
|
-
展示如何在现有的Flask应用中集成路由监控功能
|
4
|
-
|
5
|
-
重点:批量装饰器应用 - 无需手动为每个函数添加装饰器!
|
6
|
-
|
7
|
-
使用方法:
|
8
|
-
1. 导入监控模块
|
9
|
-
2. 定义需要监控的路由列表
|
10
|
-
3. 批量应用装饰器
|
11
|
-
4. 可选择性地添加全局监控
|
12
|
-
|
13
|
-
"""
|
14
|
-
|
15
|
-
# ========================================
|
16
|
-
# 方法1:批量装饰器应用(强烈推荐)
|
17
|
-
# ========================================
|
18
|
-
|
19
|
-
from monitor import monitor_request, get_request_id, get_statistics_summary
|
20
|
-
from routes import register_routes
|
21
|
-
|
22
|
-
# 在 dpflask.py 文件末尾添加以下代码:
|
23
|
-
|
24
|
-
# 第一步:定义需要监控的路由函数名列表
|
25
|
-
MONITORED_ROUTES = [
|
26
|
-
# DeepSeek相关接口
|
27
|
-
'deepseek_request', # 主要接口
|
28
|
-
'deepseek_public_key', # 获取公钥接口
|
29
|
-
'deepseek_update', # 更新日志接口
|
30
|
-
'deepseek_feedback', # 客户端反馈接口
|
31
|
-
'deepseek_chat_history', # 加载历史记录接口
|
32
|
-
'deepseek_client_id_save_user_info', # 存储client id
|
33
|
-
'deepseek_client_id_auth_user_info', # 校检client id
|
34
|
-
|
35
|
-
# 生意参谋相关接口
|
36
|
-
'sycm_access_control_info', # 访问控制信息
|
37
|
-
'sycm_load_userinfo', # 用户信息加载
|
38
|
-
'sycm_list_database', # 数据库列表
|
39
|
-
'sycm_list_tables', # 数据表列表
|
40
|
-
'sycm_view_table', # 查看表数据
|
41
|
-
]
|
42
|
-
|
43
|
-
# 第二步:批量应用监控装饰器的函数
|
44
|
-
def apply_monitor_to_routes(app_globals, route_names):
|
45
|
-
"""
|
46
|
-
为指定的路由函数批量添加监控装饰器
|
47
|
-
|
48
|
-
优势:
|
49
|
-
- 无需手动修改每个函数
|
50
|
-
- 统一管理监控配置
|
51
|
-
- 避免遗漏或重复
|
52
|
-
- 便于维护
|
53
|
-
"""
|
54
|
-
applied_count = 0
|
55
|
-
skipped_count = 0
|
56
|
-
|
57
|
-
print("🚀 开始批量应用监控装饰器...")
|
58
|
-
|
59
|
-
for route_name in route_names:
|
60
|
-
if route_name in app_globals and callable(app_globals[route_name]):
|
61
|
-
# 检查是否已经添加过监控装饰器
|
62
|
-
if not hasattr(app_globals[route_name], '_is_monitored'):
|
63
|
-
# 应用监控装饰器
|
64
|
-
app_globals[route_name] = monitor_request(app_globals[route_name])
|
65
|
-
app_globals[route_name]._is_monitored = True
|
66
|
-
applied_count += 1
|
67
|
-
print(f" ✅ 已为 {route_name} 添加监控")
|
68
|
-
else:
|
69
|
-
skipped_count += 1
|
70
|
-
print(f" ⚠️ {route_name} 已经有监控装饰器,跳过")
|
71
|
-
else:
|
72
|
-
print(f" ❌ 未找到函数: {route_name}")
|
73
|
-
|
74
|
-
print(f"📊 批量装饰器应用完成:")
|
75
|
-
print(f" - 成功添加: {applied_count} 个")
|
76
|
-
print(f" - 已存在跳过: {skipped_count} 个")
|
77
|
-
print(f" - 总计处理: {len(route_names)} 个")
|
78
|
-
|
79
|
-
# 第三步:执行批量应用(在 dpflask.py 文件最后添加)
|
80
|
-
"""
|
81
|
-
# 批量应用监控装饰器
|
82
|
-
apply_monitor_to_routes(globals(), MONITORED_ROUTES)
|
83
|
-
|
84
|
-
# 注册管理界面路由
|
85
|
-
register_routes(app)
|
86
|
-
|
87
|
-
print("🎯 路由监控系统初始化完成!")
|
88
|
-
"""
|
89
|
-
|
90
|
-
# ========================================
|
91
|
-
# 方法2:按模式自动应用(更智能)
|
92
|
-
# ========================================
|
93
|
-
|
94
|
-
def auto_apply_monitor_by_pattern(app_globals, pattern_prefixes=None, exclude_patterns=None):
|
95
|
-
"""
|
96
|
-
根据函数名模式自动添加监控装饰器
|
97
|
-
|
98
|
-
Args:
|
99
|
-
pattern_prefixes: 匹配的函数名前缀,如 ['deepseek_', 'sycm_']
|
100
|
-
exclude_patterns: 排除的模式,如 ['_test', '_internal']
|
101
|
-
"""
|
102
|
-
if pattern_prefixes is None:
|
103
|
-
pattern_prefixes = ['deepseek_', 'sycm_', 'api_']
|
104
|
-
|
105
|
-
if exclude_patterns is None:
|
106
|
-
exclude_patterns = ['_test', '_internal', '_helper', '_debug']
|
107
|
-
|
108
|
-
applied_count = 0
|
109
|
-
|
110
|
-
print("🔍 开始自动扫描并应用监控装饰器...")
|
111
|
-
|
112
|
-
for name, obj in app_globals.items():
|
113
|
-
# 检查是否为函数且符合命名模式
|
114
|
-
if callable(obj) and any(name.startswith(prefix) for prefix in pattern_prefixes):
|
115
|
-
# 检查是否需要排除
|
116
|
-
if any(exclude in name for exclude in exclude_patterns):
|
117
|
-
print(f" 🚫 跳过排除的函数: {name}")
|
118
|
-
continue
|
119
|
-
|
120
|
-
# 检查是否已经有监控装饰器
|
121
|
-
if not hasattr(obj, '_is_monitored'):
|
122
|
-
app_globals[name] = monitor_request(obj)
|
123
|
-
app_globals[name]._is_monitored = True
|
124
|
-
applied_count += 1
|
125
|
-
print(f" ✅ 自动为 {name} 添加监控")
|
126
|
-
|
127
|
-
print(f"🎯 自动扫描完成,为 {applied_count} 个路由添加了监控")
|
128
|
-
|
129
|
-
# 使用自动模式的示例:
|
130
|
-
"""
|
131
|
-
# 自动为所有以 deepseek_ 和 sycm_ 开头的函数添加监控
|
132
|
-
auto_apply_monitor_by_pattern(globals(),
|
133
|
-
pattern_prefixes=['deepseek_', 'sycm_'],
|
134
|
-
exclude_patterns=['_test', '_internal', '_debug']
|
135
|
-
)
|
136
|
-
"""
|
137
|
-
|
138
|
-
# ========================================
|
139
|
-
# 方法3:环境感知的监控配置(生产级)
|
140
|
-
# ========================================
|
141
|
-
|
142
|
-
import os
|
143
|
-
|
144
|
-
class MonitorConfig:
|
145
|
-
"""监控配置管理类 - 根据环境应用不同的监控策略"""
|
146
|
-
|
147
|
-
# 核心业务接口(所有环境都监控)
|
148
|
-
CRITICAL_ROUTES = [
|
149
|
-
'deepseek_request',
|
150
|
-
'sycm_list_database',
|
151
|
-
'sycm_view_table',
|
152
|
-
]
|
153
|
-
|
154
|
-
# 重要接口(生产和测试环境监控)
|
155
|
-
IMPORTANT_ROUTES = [
|
156
|
-
'deepseek_public_key',
|
157
|
-
'deepseek_chat_history',
|
158
|
-
'sycm_load_userinfo',
|
159
|
-
'sycm_list_tables',
|
160
|
-
]
|
161
|
-
|
162
|
-
# 辅助接口(仅生产环境监控)
|
163
|
-
AUXILIARY_ROUTES = [
|
164
|
-
'deepseek_update',
|
165
|
-
'deepseek_feedback',
|
166
|
-
'sycm_access_control_info',
|
167
|
-
'deepseek_client_id_save_user_info',
|
168
|
-
'deepseek_client_id_auth_user_info',
|
169
|
-
]
|
170
|
-
|
171
|
-
@classmethod
|
172
|
-
def apply_monitoring(cls, app_globals, environment=None):
|
173
|
-
"""
|
174
|
-
根据环境应用相应的监控策略
|
175
|
-
|
176
|
-
Args:
|
177
|
-
environment: 'production', 'staging', 'development' 或 None(自动检测)
|
178
|
-
"""
|
179
|
-
if environment is None:
|
180
|
-
environment = os.getenv('FLASK_ENV', 'development')
|
181
|
-
|
182
|
-
routes_to_monitor = []
|
183
|
-
|
184
|
-
# 根据环境确定监控范围
|
185
|
-
if environment == 'production':
|
186
|
-
routes_to_monitor.extend(cls.CRITICAL_ROUTES)
|
187
|
-
routes_to_monitor.extend(cls.IMPORTANT_ROUTES)
|
188
|
-
routes_to_monitor.extend(cls.AUXILIARY_ROUTES)
|
189
|
-
print("🔥 生产环境:启用完整监控")
|
190
|
-
|
191
|
-
elif environment == 'staging':
|
192
|
-
routes_to_monitor.extend(cls.CRITICAL_ROUTES)
|
193
|
-
routes_to_monitor.extend(cls.IMPORTANT_ROUTES)
|
194
|
-
print("🧪 测试环境:启用重要接口监控")
|
195
|
-
|
196
|
-
else: # development
|
197
|
-
routes_to_monitor.extend(cls.CRITICAL_ROUTES)
|
198
|
-
print("🔧 开发环境:仅监控核心接口")
|
199
|
-
|
200
|
-
# 应用监控
|
201
|
-
apply_monitor_to_routes(app_globals, routes_to_monitor)
|
202
|
-
print(f"📊 {environment} 环境监控配置已应用")
|
203
|
-
|
204
|
-
# 环境感知监控的使用示例:
|
205
|
-
"""
|
206
|
-
# 在 dpflask.py 文件末尾添加:
|
207
|
-
|
208
|
-
# 根据环境自动应用监控配置
|
209
|
-
MonitorConfig.apply_monitoring(globals())
|
210
|
-
|
211
|
-
# 手动指定环境
|
212
|
-
# MonitorConfig.apply_monitoring(globals(), 'production')
|
213
|
-
|
214
|
-
# 注册管理界面
|
215
|
-
register_routes(app)
|
216
|
-
"""
|
217
|
-
|
218
|
-
# ========================================
|
219
|
-
# 方法4:带条件的智能监控(高级用法)
|
220
|
-
# ========================================
|
221
|
-
|
222
|
-
def conditional_monitor_routes(app_globals, route_conditions):
|
223
|
-
"""
|
224
|
-
根据条件应用监控装饰器
|
225
|
-
|
226
|
-
Args:
|
227
|
-
route_conditions: 字典,格式为 {函数名: 条件函数}
|
228
|
-
"""
|
229
|
-
applied_count = 0
|
230
|
-
|
231
|
-
for route_name, condition_func in route_conditions.items():
|
232
|
-
if route_name in app_globals and callable(app_globals[route_name]):
|
233
|
-
if condition_func():
|
234
|
-
if not hasattr(app_globals[route_name], '_is_monitored'):
|
235
|
-
app_globals[route_name] = monitor_request(app_globals[route_name])
|
236
|
-
app_globals[route_name]._is_monitored = True
|
237
|
-
applied_count += 1
|
238
|
-
print(f" ✅ 条件满足,为 {route_name} 添加监控")
|
239
|
-
else:
|
240
|
-
print(f" ⚠️ {route_name} 已有监控装饰器")
|
241
|
-
else:
|
242
|
-
print(f" 🚫 条件不满足,跳过 {route_name}")
|
243
|
-
|
244
|
-
print(f"🎯 条件监控应用完成,处理了 {applied_count} 个路由")
|
245
|
-
|
246
|
-
# 条件监控示例:
|
247
|
-
"""
|
248
|
-
# 定义条件函数
|
249
|
-
def is_production():
|
250
|
-
return os.getenv('FLASK_ENV') == 'production'
|
251
|
-
|
252
|
-
def is_debug_enabled():
|
253
|
-
return os.getenv('DEBUG', '').lower() == 'true'
|
254
|
-
|
255
|
-
# 定义条件监控规则
|
256
|
-
CONDITIONAL_ROUTES = {
|
257
|
-
'deepseek_request': lambda: True, # 总是监控
|
258
|
-
'deepseek_update': is_production, # 仅生产环境监控
|
259
|
-
'sycm_view_table': is_production, # 仅生产环境监控
|
260
|
-
'debug_endpoint': is_debug_enabled, # 仅调试模式监控
|
261
|
-
}
|
262
|
-
|
263
|
-
# 应用条件监控
|
264
|
-
conditional_monitor_routes(globals(), CONDITIONAL_ROUTES)
|
265
|
-
"""
|
266
|
-
|
267
|
-
# ========================================
|
268
|
-
# 完整的集成示例代码
|
269
|
-
# ========================================
|
270
|
-
|
271
|
-
COMPLETE_INTEGRATION_EXAMPLE = '''
|
272
|
-
# ================================================
|
273
|
-
# 在 dpflask.py 文件末尾添加以下完整代码
|
274
|
-
# ================================================
|
275
|
-
|
276
|
-
# 1. 导入监控模块
|
277
|
-
from route.monitor import monitor_request, get_request_id, get_statistics_summary
|
278
|
-
from route.routes import register_routes
|
279
|
-
import os
|
280
|
-
|
281
|
-
# 2. 批量装饰器应用函数
|
282
|
-
def apply_monitor_to_routes(app_globals, route_names):
|
283
|
-
"""批量为路由函数添加监控装饰器"""
|
284
|
-
applied_count = 0
|
285
|
-
for route_name in route_names:
|
286
|
-
if route_name in app_globals and callable(app_globals[route_name]):
|
287
|
-
if not hasattr(app_globals[route_name], '_is_monitored'):
|
288
|
-
app_globals[route_name] = monitor_request(app_globals[route_name])
|
289
|
-
app_globals[route_name]._is_monitored = True
|
290
|
-
applied_count += 1
|
291
|
-
print(f"✅ 已为 {route_name} 添加监控")
|
292
|
-
print(f"🎯 总计为 {applied_count} 个路由添加了监控")
|
293
|
-
|
294
|
-
# 3. 定义需要监控的路由列表
|
295
|
-
MONITORED_ROUTES = [
|
296
|
-
'deepseek_request',
|
297
|
-
'deepseek_public_key',
|
298
|
-
'deepseek_chat_history',
|
299
|
-
'sycm_list_database',
|
300
|
-
'sycm_view_table',
|
301
|
-
# 根据需要添加更多路由
|
302
|
-
]
|
303
|
-
|
304
|
-
# 4. 应用监控
|
305
|
-
print("🚀 正在初始化路由监控系统...")
|
306
|
-
apply_monitor_to_routes(globals(), MONITORED_ROUTES)
|
307
|
-
|
308
|
-
# 5. 注册管理界面
|
309
|
-
register_routes(app)
|
310
|
-
|
311
|
-
# 6. 完成提示
|
312
|
-
current_env = os.getenv('FLASK_ENV', 'development')
|
313
|
-
print(f"✨ 路由监控系统初始化完成!")
|
314
|
-
print(f"🔧 当前环境: {current_env}")
|
315
|
-
print(f"📊 管理界面: http://localhost:5000/admin/monitor/dashboard")
|
316
|
-
|
317
|
-
# 可选:添加一些简单的管理接口
|
318
|
-
@app.route('/admin/monitor/stats', methods=['GET'])
|
319
|
-
def get_monitor_stats():
|
320
|
-
"""获取监控统计信息"""
|
321
|
-
try:
|
322
|
-
days = request.args.get('days', 7, type=int)
|
323
|
-
stats = get_statistics_summary(days)
|
324
|
-
return jsonify({
|
325
|
-
'code': 0,
|
326
|
-
'status': 'success',
|
327
|
-
'data': stats
|
328
|
-
})
|
329
|
-
except Exception as e:
|
330
|
-
return jsonify({
|
331
|
-
'code': 500,
|
332
|
-
'status': 'error',
|
333
|
-
'message': str(e)
|
334
|
-
}), 500
|
335
|
-
'''
|
336
|
-
|
337
|
-
# ========================================
|
338
|
-
# 总结说明
|
339
|
-
# ========================================
|
340
|
-
|
341
|
-
print("""
|
342
|
-
🎯 批量装饰器的核心优势:
|
343
|
-
|
344
|
-
1. ✅ 无需手动修改每个函数定义
|
345
|
-
- 原始函数保持不变
|
346
|
-
- 所有装饰器在一个地方统一管理
|
347
|
-
|
348
|
-
2. ✅ 灵活的监控策略
|
349
|
-
- 可以根据环境启用不同的监控范围
|
350
|
-
- 支持条件性监控
|
351
|
-
- 便于测试和调试
|
352
|
-
|
353
|
-
3. ✅ 维护性强
|
354
|
-
- 新增监控只需在列表中添加函数名
|
355
|
-
- 移除监控只需从列表中删除
|
356
|
-
- 避免在代码中到处添加装饰器
|
357
|
-
|
358
|
-
4. ✅ 性能优化
|
359
|
-
- 避免重复装饰
|
360
|
-
- 自动检测已有装饰器
|
361
|
-
- 支持条件性启用
|
362
|
-
|
363
|
-
📋 推荐使用方式:
|
364
|
-
- 开发阶段:使用方法1(明确的函数名列表)
|
365
|
-
- 生产环境:使用方法3(环境感知配置)
|
366
|
-
- 大型项目:结合多种方法,分模块管理
|
367
|
-
|
368
|
-
🔧 集成步骤:
|
369
|
-
1. 复制 COMPLETE_INTEGRATION_EXAMPLE 中的代码
|
370
|
-
2. 添加到 dpflask.py 文件末尾
|
371
|
-
3. 根据需要调整 MONITORED_ROUTES 列表
|
372
|
-
4. 重启应用即可生效
|
373
|
-
|
374
|
-
这样就实现了批量添加监控装饰器,无需手动修改每个函数!
|
375
|
-
""")
|
376
|
-
|
377
|
-
if __name__ == "__main__":
|
378
|
-
print(COMPLETE_INTEGRATION_EXAMPLE)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|