mdbq 4.2.18__tar.gz → 4.2.19__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- {mdbq-4.2.18 → mdbq-4.2.19}/PKG-INFO +1 -1
- mdbq-4.2.19/mdbq/__version__.py +1 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/redis/redis_cache.py +15 -2
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/route/monitor.py +38 -13
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq.egg-info/PKG-INFO +1 -1
- mdbq-4.2.18/mdbq/__version__.py +0 -1
- {mdbq-4.2.18 → mdbq-4.2.19}/README.txt +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/auth/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/auth/auth_backend.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/auth/crypto.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/auth/rate_limiter.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/js/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/js/jc.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/log/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/log/mylogger.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/myconf/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/myconf/myconf.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/deduplicator.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/mysql.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/s_query.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/unique_.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/mysql/uploader.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/download_sku_picture.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/error_handler.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/otk.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/pov_city.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/other/ua_sj.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/pbix/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/pbix/pbix_refresh.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/pbix/refresh_all.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/redis/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/redis/getredis.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/route/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/route/analytics.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/route/routes.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/selenium/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/selenium/get_driver.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq/spider/__init__.py +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq.egg-info/SOURCES.txt +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq.egg-info/dependency_links.txt +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/mdbq.egg-info/top_level.txt +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/setup.cfg +0 -0
- {mdbq-4.2.18 → mdbq-4.2.19}/setup.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = '4.2.19'
|
|
@@ -914,16 +914,23 @@ def flask_redis_cache(cache_key_func=None, ttl=1200, namespace="default",
|
|
|
914
914
|
def wrapper(*args, **kwargs):
|
|
915
915
|
# 导入Flask相关模块(延迟导入避免依赖问题)
|
|
916
916
|
try:
|
|
917
|
-
from flask import request, jsonify
|
|
917
|
+
from flask import request, jsonify, g
|
|
918
918
|
except ImportError:
|
|
919
919
|
# 如果没有Flask环境,直接执行原函数
|
|
920
920
|
return func(*args, **kwargs)
|
|
921
921
|
|
|
922
|
+
# 初始化数据源标记(默认为 none)
|
|
923
|
+
g.data_source = 'none'
|
|
924
|
+
g.cache_hit = False
|
|
925
|
+
|
|
922
926
|
# 获取缓存系统
|
|
923
927
|
cache_system = cache_manager.get_cache()
|
|
924
928
|
|
|
925
929
|
# 如果缓存系统不可用,直接执行原函数
|
|
926
930
|
if not cache_system:
|
|
931
|
+
# 标记为直接查询数据库(无缓存系统)
|
|
932
|
+
g.data_source = 'mysql'
|
|
933
|
+
g.cache_hit = False
|
|
927
934
|
return func(*args, **kwargs)
|
|
928
935
|
|
|
929
936
|
try:
|
|
@@ -952,12 +959,18 @@ def flask_redis_cache(cache_key_func=None, ttl=1200, namespace="default",
|
|
|
952
959
|
try:
|
|
953
960
|
cached_result = cache_system.get(cache_key, namespace)
|
|
954
961
|
if cached_result is not None:
|
|
962
|
+
# 缓存命中:标记为 redis
|
|
963
|
+
g.data_source = 'redis'
|
|
964
|
+
g.cache_hit = True
|
|
955
965
|
return jsonify(cached_result)
|
|
956
966
|
except Exception as e:
|
|
957
967
|
if not skip_cache_on_error:
|
|
958
968
|
raise
|
|
959
969
|
|
|
960
|
-
#
|
|
970
|
+
# 缓存未命中,执行原函数(会查询数据库)
|
|
971
|
+
# 标记为 mysql 查询
|
|
972
|
+
g.data_source = 'mysql'
|
|
973
|
+
g.cache_hit = False
|
|
961
974
|
result = func(*args, **kwargs)
|
|
962
975
|
|
|
963
976
|
# 如果结果是Flask Response对象,提取JSON数据进行缓存
|
|
@@ -176,6 +176,8 @@ class RouteMonitor:
|
|
|
176
176
|
`客户端ip` VARCHAR(45) NOT NULL COMMENT '客户端 ip 地址(支持 IPv6)',
|
|
177
177
|
`状态码` SMALLINT COMMENT 'HTTP 状态码',
|
|
178
178
|
`响应耗时` DECIMAL(10,3) COMMENT '请求处理耗时(毫秒)',
|
|
179
|
+
`数据源` VARCHAR(20) COMMENT '数据源类型:redis/mysql/hybrid/none',
|
|
180
|
+
`缓存命中` TINYINT(1) DEFAULT 0 COMMENT '是否命中缓存:1-命中,0-未命中',
|
|
179
181
|
`用户标识` VARCHAR(64) COMMENT '用户id或标识(如有)',
|
|
180
182
|
`用户代理` VARCHAR(500) COMMENT '浏览器 User-Agent(精简版)',
|
|
181
183
|
`请求参数` TEXT COMMENT '请求参数(JSON格式,可选记录)',
|
|
@@ -188,8 +190,11 @@ class RouteMonitor:
|
|
|
188
190
|
INDEX `idx_路由地址` (`路由地址`(191)),
|
|
189
191
|
INDEX `idx_客户端ip` (`客户端ip`),
|
|
190
192
|
INDEX `idx_状态码` (`状态码`),
|
|
193
|
+
INDEX `idx_数据源` (`数据源`),
|
|
194
|
+
INDEX `idx_缓存命中` (`缓存命中`),
|
|
191
195
|
INDEX `idx_用户标识` (`用户标识`),
|
|
192
|
-
INDEX `idx_时间_接口` (`请求时间`, `路由地址`(191))
|
|
196
|
+
INDEX `idx_时间_接口` (`请求时间`, `路由地址`(191)),
|
|
197
|
+
INDEX `idx_时间_数据源` (`请求时间`, `数据源`)
|
|
193
198
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
194
199
|
COMMENT='API 访问日志表 - 记录每次请求的核心信息'
|
|
195
200
|
ROW_FORMAT=COMPRESSED;
|
|
@@ -207,6 +212,8 @@ class RouteMonitor:
|
|
|
207
212
|
`请求总数` INT UNSIGNED DEFAULT 0 COMMENT '总请求次数',
|
|
208
213
|
`成功次数` INT UNSIGNED DEFAULT 0 COMMENT '成功响应次数(状态码 < 400)',
|
|
209
214
|
`失败次数` INT UNSIGNED DEFAULT 0 COMMENT '失败响应次数(状态码 >= 400)',
|
|
215
|
+
`缓存命中次数` INT UNSIGNED DEFAULT 0 COMMENT '缓存命中次数',
|
|
216
|
+
`数据库查询次数` INT UNSIGNED DEFAULT 0 COMMENT '没有命中缓存,直接查询数据库的次数',
|
|
210
217
|
`平均耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '平均响应耗时(毫秒)',
|
|
211
218
|
`最大耗时` DECIMAL(10,3) DEFAULT 0 COMMENT '最大响应耗时(毫秒)',
|
|
212
219
|
`最小耗时` DECIMAL(10,3) DEFAULT NULL COMMENT '最小响应耗时(毫秒)',
|
|
@@ -499,6 +506,10 @@ class RouteMonitor:
|
|
|
499
506
|
if params_dict:
|
|
500
507
|
request_params = self.sanitize_params(params_dict)
|
|
501
508
|
|
|
509
|
+
# 获取数据源信息(从 g 对象中读取,由缓存装饰器设置)
|
|
510
|
+
data_source = getattr(g, 'data_source', 'none')
|
|
511
|
+
cache_hit = getattr(g, 'cache_hit', False)
|
|
512
|
+
|
|
502
513
|
# 构建请求数据字典
|
|
503
514
|
request_data = {
|
|
504
515
|
'请求id': request_id,
|
|
@@ -506,6 +517,8 @@ class RouteMonitor:
|
|
|
506
517
|
'请求方法': request.method,
|
|
507
518
|
'路由地址': request.endpoint or request.path,
|
|
508
519
|
'客户端ip': client_ip,
|
|
520
|
+
'数据源': data_source,
|
|
521
|
+
'缓存命中': 1 if cache_hit else 0,
|
|
509
522
|
'用户标识': user_id,
|
|
510
523
|
'用户代理': user_agent,
|
|
511
524
|
'请求参数': request_params,
|
|
@@ -539,9 +552,10 @@ class RouteMonitor:
|
|
|
539
552
|
sql = """
|
|
540
553
|
INSERT INTO `api_访问日志` (
|
|
541
554
|
`请求id`, `请求时间`, `请求方法`, `路由地址`, `客户端ip`,
|
|
542
|
-
`状态码`, `响应耗时`,
|
|
555
|
+
`状态码`, `响应耗时`, `数据源`, `缓存命中`,
|
|
556
|
+
`用户标识`, `用户代理`, `请求参数`, `错误信息`
|
|
543
557
|
) VALUES (
|
|
544
|
-
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
558
|
+
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
|
|
545
559
|
)
|
|
546
560
|
"""
|
|
547
561
|
|
|
@@ -553,6 +567,8 @@ class RouteMonitor:
|
|
|
553
567
|
request_data.get('客户端ip'),
|
|
554
568
|
request_data.get('状态码'),
|
|
555
569
|
request_data.get('响应耗时'),
|
|
570
|
+
request_data.get('数据源', 'none'),
|
|
571
|
+
request_data.get('缓存命中', 0),
|
|
556
572
|
request_data.get('用户标识'),
|
|
557
573
|
request_data.get('用户代理'),
|
|
558
574
|
request_data.get('请求参数'),
|
|
@@ -597,18 +613,27 @@ class RouteMonitor:
|
|
|
597
613
|
is_error = 1 if status_code >= 400 else 0
|
|
598
614
|
response_time = request_data.get('响应耗时', 0)
|
|
599
615
|
|
|
616
|
+
# 获取数据源信息
|
|
617
|
+
cache_hit = request_data.get('缓存命中', 0)
|
|
618
|
+
data_source = request_data.get('数据源', 'none')
|
|
619
|
+
is_cache_hit = 1 if cache_hit else 0
|
|
620
|
+
is_db_query = 1 if data_source in ['mysql', 'hybrid'] else 0
|
|
621
|
+
|
|
600
622
|
# 1. 更新接口统计表
|
|
601
623
|
cursor.execute("""
|
|
602
624
|
INSERT INTO `api_接口统计` (
|
|
603
625
|
`统计日期`, `统计小时`, `路由地址`, `请求方法`,
|
|
604
626
|
`请求总数`, `成功次数`, `失败次数`,
|
|
627
|
+
`缓存命中次数`, `数据库查询次数`,
|
|
605
628
|
`平均耗时`, `最大耗时`, `最小耗时`
|
|
606
629
|
) VALUES (
|
|
607
|
-
%s, %s, %s, %s, 1, %s, %s, %s, %s, %s
|
|
630
|
+
%s, %s, %s, %s, 1, %s, %s, %s, %s, %s, %s, %s
|
|
608
631
|
) ON DUPLICATE KEY UPDATE
|
|
609
632
|
`请求总数` = `请求总数` + 1,
|
|
610
633
|
`成功次数` = `成功次数` + %s,
|
|
611
634
|
`失败次数` = `失败次数` + %s,
|
|
635
|
+
`缓存命中次数` = `缓存命中次数` + %s,
|
|
636
|
+
`数据库查询次数` = `数据库查询次数` + %s,
|
|
612
637
|
`平均耗时` = (
|
|
613
638
|
(`平均耗时` * `请求总数` + %s) / (`请求总数` + 1)
|
|
614
639
|
),
|
|
@@ -623,13 +648,13 @@ class RouteMonitor:
|
|
|
623
648
|
date, hour,
|
|
624
649
|
request_data.get('路由地址', ''),
|
|
625
650
|
request_data.get('请求方法', ''),
|
|
626
|
-
is_success, is_error,
|
|
651
|
+
is_success, is_error, is_cache_hit, is_db_query,
|
|
627
652
|
response_time, response_time, response_time,
|
|
628
|
-
is_success, is_error,
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
653
|
+
is_success, is_error, is_cache_hit, is_db_query,
|
|
654
|
+
response_time,
|
|
655
|
+
response_time,
|
|
656
|
+
response_time,
|
|
657
|
+
response_time
|
|
633
658
|
))
|
|
634
659
|
|
|
635
660
|
# 2. 更新 IP 统计表
|
|
@@ -702,7 +727,7 @@ class RouteMonitor:
|
|
|
702
727
|
|
|
703
728
|
# 收集请求数据
|
|
704
729
|
request_data = self.collect_request_data(request)
|
|
705
|
-
request_id = request_data.get('请求id', 'unknown')
|
|
730
|
+
# request_id = request_data.get('请求id', 'unknown')
|
|
706
731
|
|
|
707
732
|
# 统计总请求数(线程安全)
|
|
708
733
|
with self._stats_lock:
|
|
@@ -739,7 +764,7 @@ class RouteMonitor:
|
|
|
739
764
|
# 合并数据
|
|
740
765
|
request_data.update(response_data)
|
|
741
766
|
|
|
742
|
-
#
|
|
767
|
+
# 优先使用 Redis 队列(非阻塞)
|
|
743
768
|
queued = self._push_to_queue({
|
|
744
769
|
'type': 'request_log',
|
|
745
770
|
'data': request_data,
|
|
@@ -770,7 +795,7 @@ class RouteMonitor:
|
|
|
770
795
|
# 合并数据
|
|
771
796
|
request_data.update(error_data)
|
|
772
797
|
|
|
773
|
-
#
|
|
798
|
+
# 优先使用 Redis 队列(非阻塞)
|
|
774
799
|
queued = self._push_to_queue({
|
|
775
800
|
'type': 'request_log',
|
|
776
801
|
'data': request_data,
|
mdbq-4.2.18/mdbq/__version__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
VERSION = '4.2.18'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|