mdbq 4.2.9__tar.gz → 4.2.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mdbq might be problematic. Click here for more details.
- {mdbq-4.2.9 → mdbq-4.2.10}/PKG-INFO +2 -2
- mdbq-4.2.10/mdbq/__version__.py +1 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/redis/redis_cache.py +283 -1
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq.egg-info/PKG-INFO +2 -2
- mdbq-4.2.9/mdbq/__version__.py +0 -1
- {mdbq-4.2.9 → mdbq-4.2.10}/README.txt +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/auth/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/auth/auth_backend.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/auth/crypto.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/auth/rate_limiter.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/js/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/js/jc.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/log/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/log/mylogger.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/myconf/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/myconf/myconf.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/deduplicator.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/mysql.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/s_query.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/unique_.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/mysql/uploader.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/download_sku_picture.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/error_handler.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/otk.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/pov_city.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/other/ua_sj.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/pbix/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/pbix/pbix_refresh.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/pbix/refresh_all.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/redis/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/redis/getredis.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/route/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/route/analytics.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/route/monitor.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/route/routes.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/selenium/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/selenium/get_driver.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq/spider/__init__.py +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq.egg-info/SOURCES.txt +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq.egg-info/dependency_links.txt +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/mdbq.egg-info/top_level.txt +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/setup.cfg +0 -0
- {mdbq-4.2.9 → mdbq-4.2.10}/setup.py +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
VERSION = '4.2.10'
|
|
@@ -879,4 +879,286 @@ class CacheManager:
|
|
|
879
879
|
|
|
880
880
|
|
|
881
881
|
# 全局缓存管理器实例
|
|
882
|
-
cache_manager = CacheManager()
|
|
882
|
+
cache_manager = CacheManager()
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
# ===== 装饰器功能 =====
|
|
886
|
+
|
|
887
|
+
def flask_redis_cache(cache_key_func=None, ttl=1200, namespace="default",
|
|
888
|
+
data_validator=None, skip_cache_on_error=True):
|
|
889
|
+
"""
|
|
890
|
+
Flask路由函数的Redis缓存装饰器
|
|
891
|
+
|
|
892
|
+
Args:
|
|
893
|
+
cache_key_func: 缓存键生成函数,接收请求数据作为参数,返回缓存键字符串
|
|
894
|
+
如果为None,则使用默认的键生成策略
|
|
895
|
+
ttl: 缓存过期时间(秒),默认20分钟
|
|
896
|
+
namespace: 缓存命名空间,默认为"default"
|
|
897
|
+
data_validator: 数据验证函数,用于验证数据是否应该被缓存
|
|
898
|
+
skip_cache_on_error: 当缓存操作出错时是否跳过缓存,默认True
|
|
899
|
+
|
|
900
|
+
Usage:
|
|
901
|
+
@flask_redis_cache(
|
|
902
|
+
cache_key_func=lambda data: f"tables_{data.get('database', 'unknown')}",
|
|
903
|
+
ttl=1200,
|
|
904
|
+
namespace="sycm_tables"
|
|
905
|
+
)
|
|
906
|
+
def my_flask_route():
|
|
907
|
+
pass
|
|
908
|
+
"""
|
|
909
|
+
def decorator(func):
|
|
910
|
+
import functools
|
|
911
|
+
import hashlib
|
|
912
|
+
|
|
913
|
+
@functools.wraps(func)
|
|
914
|
+
def wrapper(*args, **kwargs):
|
|
915
|
+
# 导入Flask相关模块(延迟导入避免依赖问题)
|
|
916
|
+
try:
|
|
917
|
+
from flask import request, jsonify
|
|
918
|
+
except ImportError:
|
|
919
|
+
# 如果没有Flask环境,直接执行原函数
|
|
920
|
+
return func(*args, **kwargs)
|
|
921
|
+
|
|
922
|
+
# 获取缓存系统
|
|
923
|
+
cache_system = cache_manager.get_cache()
|
|
924
|
+
|
|
925
|
+
# 如果缓存系统不可用,直接执行原函数
|
|
926
|
+
if not cache_system:
|
|
927
|
+
return func(*args, **kwargs)
|
|
928
|
+
|
|
929
|
+
try:
|
|
930
|
+
# 获取请求数据用于生成缓存键
|
|
931
|
+
request_data = {}
|
|
932
|
+
if request.method == 'POST':
|
|
933
|
+
try:
|
|
934
|
+
request_data = request.get_json() or {}
|
|
935
|
+
except Exception:
|
|
936
|
+
request_data = {}
|
|
937
|
+
elif request.method == 'GET':
|
|
938
|
+
request_data = dict(request.args)
|
|
939
|
+
|
|
940
|
+
# 生成缓存键
|
|
941
|
+
if cache_key_func:
|
|
942
|
+
cache_key = cache_key_func(request_data)
|
|
943
|
+
else:
|
|
944
|
+
# 默认缓存键生成策略
|
|
945
|
+
func_name = func.__name__
|
|
946
|
+
# 将请求数据转换为字符串并生成哈希
|
|
947
|
+
data_str = str(sorted(request_data.items()))
|
|
948
|
+
data_hash = hashlib.md5(data_str.encode()).hexdigest()[:8]
|
|
949
|
+
cache_key = f"{func_name}_{data_hash}"
|
|
950
|
+
|
|
951
|
+
# 尝试从缓存获取数据
|
|
952
|
+
try:
|
|
953
|
+
cached_result = cache_system.get(cache_key, namespace)
|
|
954
|
+
if cached_result is not None:
|
|
955
|
+
return jsonify(cached_result)
|
|
956
|
+
except Exception as e:
|
|
957
|
+
if not skip_cache_on_error:
|
|
958
|
+
raise
|
|
959
|
+
|
|
960
|
+
# 缓存未命中,执行原函数
|
|
961
|
+
result = func(*args, **kwargs)
|
|
962
|
+
|
|
963
|
+
# 如果结果是Flask Response对象,提取JSON数据进行缓存
|
|
964
|
+
if hasattr(result, 'get_json'):
|
|
965
|
+
try:
|
|
966
|
+
response_data = result.get_json()
|
|
967
|
+
if response_data:
|
|
968
|
+
# 使用安全缓存写入
|
|
969
|
+
_safe_cache_set(
|
|
970
|
+
cache_system=cache_system,
|
|
971
|
+
cache_key=cache_key,
|
|
972
|
+
response_data=response_data,
|
|
973
|
+
ttl=ttl,
|
|
974
|
+
namespace=namespace,
|
|
975
|
+
data_validator=data_validator
|
|
976
|
+
)
|
|
977
|
+
except Exception as e:
|
|
978
|
+
if not skip_cache_on_error:
|
|
979
|
+
raise
|
|
980
|
+
elif isinstance(result, tuple) and len(result) == 2:
|
|
981
|
+
# 处理 (response, status_code) 格式的返回值
|
|
982
|
+
try:
|
|
983
|
+
response_data, status_code = result
|
|
984
|
+
if hasattr(response_data, 'get_json'):
|
|
985
|
+
json_data = response_data.get_json()
|
|
986
|
+
elif isinstance(response_data, dict):
|
|
987
|
+
json_data = response_data
|
|
988
|
+
else:
|
|
989
|
+
json_data = None
|
|
990
|
+
|
|
991
|
+
if json_data and status_code == 200:
|
|
992
|
+
_safe_cache_set(
|
|
993
|
+
cache_system=cache_system,
|
|
994
|
+
cache_key=cache_key,
|
|
995
|
+
response_data=json_data,
|
|
996
|
+
ttl=ttl,
|
|
997
|
+
namespace=namespace,
|
|
998
|
+
data_validator=data_validator
|
|
999
|
+
)
|
|
1000
|
+
except Exception as e:
|
|
1001
|
+
if not skip_cache_on_error:
|
|
1002
|
+
raise
|
|
1003
|
+
|
|
1004
|
+
return result
|
|
1005
|
+
|
|
1006
|
+
except Exception as e:
|
|
1007
|
+
if not skip_cache_on_error:
|
|
1008
|
+
raise
|
|
1009
|
+
return func(*args, **kwargs)
|
|
1010
|
+
|
|
1011
|
+
return wrapper
|
|
1012
|
+
return decorator
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def function_redis_cache(cache_key_func=None, ttl=1800, namespace="default",
|
|
1016
|
+
skip_cache_on_error=True):
|
|
1017
|
+
"""
|
|
1018
|
+
普通函数的Redis缓存装饰器
|
|
1019
|
+
|
|
1020
|
+
Args:
|
|
1021
|
+
cache_key_func: 缓存键生成函数,接收函数参数作为输入,返回缓存键字符串
|
|
1022
|
+
如果为None,则使用默认的键生成策略
|
|
1023
|
+
ttl: 缓存过期时间(秒),默认30分钟
|
|
1024
|
+
namespace: 缓存命名空间,默认为"default"
|
|
1025
|
+
skip_cache_on_error: 当缓存操作出错时是否跳过缓存,默认True
|
|
1026
|
+
|
|
1027
|
+
Usage:
|
|
1028
|
+
@function_redis_cache(
|
|
1029
|
+
cache_key_func=lambda _key, shop_name: f"cookies_{_key}_{shop_name}",
|
|
1030
|
+
ttl=1800,
|
|
1031
|
+
namespace="cookies_cache"
|
|
1032
|
+
)
|
|
1033
|
+
def my_function(_key, shop_name):
|
|
1034
|
+
pass
|
|
1035
|
+
"""
|
|
1036
|
+
def decorator(func):
|
|
1037
|
+
import functools
|
|
1038
|
+
import inspect
|
|
1039
|
+
import hashlib
|
|
1040
|
+
|
|
1041
|
+
@functools.wraps(func)
|
|
1042
|
+
def wrapper(*args, **kwargs):
|
|
1043
|
+
# 获取缓存系统
|
|
1044
|
+
cache_system = cache_manager.get_cache()
|
|
1045
|
+
|
|
1046
|
+
# 如果缓存系统不可用,直接执行原函数
|
|
1047
|
+
if not cache_system:
|
|
1048
|
+
return func(*args, **kwargs)
|
|
1049
|
+
|
|
1050
|
+
try:
|
|
1051
|
+
# 获取函数签名和参数
|
|
1052
|
+
sig = inspect.signature(func)
|
|
1053
|
+
bound_args = sig.bind(*args, **kwargs)
|
|
1054
|
+
bound_args.apply_defaults()
|
|
1055
|
+
|
|
1056
|
+
# 生成缓存键
|
|
1057
|
+
if cache_key_func:
|
|
1058
|
+
cache_key = cache_key_func(*args, **kwargs)
|
|
1059
|
+
else:
|
|
1060
|
+
# 默认缓存键生成策略
|
|
1061
|
+
func_name = func.__name__
|
|
1062
|
+
# 将参数转换为字符串并生成哈希
|
|
1063
|
+
args_str = str(args) + str(sorted(kwargs.items()))
|
|
1064
|
+
args_hash = hashlib.md5(args_str.encode()).hexdigest()[:8]
|
|
1065
|
+
cache_key = f"{func_name}_{args_hash}"
|
|
1066
|
+
|
|
1067
|
+
# 尝试从缓存获取数据
|
|
1068
|
+
try:
|
|
1069
|
+
cached_result = cache_system.get(cache_key, namespace)
|
|
1070
|
+
if cached_result is not None:
|
|
1071
|
+
return cached_result
|
|
1072
|
+
except Exception as e:
|
|
1073
|
+
if not skip_cache_on_error:
|
|
1074
|
+
raise
|
|
1075
|
+
|
|
1076
|
+
# 缓存未命中,执行原函数
|
|
1077
|
+
result = func(*args, **kwargs)
|
|
1078
|
+
|
|
1079
|
+
# 缓存结果(只缓存非空结果)
|
|
1080
|
+
if result is not None and result != {} and result != []:
|
|
1081
|
+
try:
|
|
1082
|
+
cache_system.set(cache_key, result, ttl=ttl, namespace=namespace)
|
|
1083
|
+
except Exception as e:
|
|
1084
|
+
if not skip_cache_on_error:
|
|
1085
|
+
raise
|
|
1086
|
+
|
|
1087
|
+
return result
|
|
1088
|
+
|
|
1089
|
+
except Exception as e:
|
|
1090
|
+
if not skip_cache_on_error:
|
|
1091
|
+
raise
|
|
1092
|
+
return func(*args, **kwargs)
|
|
1093
|
+
|
|
1094
|
+
return wrapper
|
|
1095
|
+
return decorator
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def _safe_cache_set(cache_system, cache_key, response_data, ttl, namespace,
|
|
1099
|
+
data_validator=None):
|
|
1100
|
+
"""
|
|
1101
|
+
安全的缓存写入函数,只有数据有效时才写入缓存。
|
|
1102
|
+
|
|
1103
|
+
Args:
|
|
1104
|
+
cache_system: 缓存系统实例
|
|
1105
|
+
cache_key: 缓存键
|
|
1106
|
+
response_data: 要缓存的响应数据
|
|
1107
|
+
ttl: 缓存过期时间
|
|
1108
|
+
namespace: 缓存命名空间
|
|
1109
|
+
data_validator: 数据验证函数,返回True表示数据有效
|
|
1110
|
+
|
|
1111
|
+
Returns:
|
|
1112
|
+
bool: 是否成功写入缓存
|
|
1113
|
+
"""
|
|
1114
|
+
if not cache_system:
|
|
1115
|
+
return False
|
|
1116
|
+
|
|
1117
|
+
# 默认验证逻辑:检查响应数据结构
|
|
1118
|
+
if data_validator is None:
|
|
1119
|
+
def default_validator(data):
|
|
1120
|
+
if not isinstance(data, dict):
|
|
1121
|
+
return False
|
|
1122
|
+
|
|
1123
|
+
# 更宽松的验证逻辑,支持多种响应格式
|
|
1124
|
+
# 检查状态字段(支持多种成功状态格式)
|
|
1125
|
+
status_ok = (
|
|
1126
|
+
data.get('status') == 'success' or # 新格式
|
|
1127
|
+
data.get('code') == 0 or # 旧格式
|
|
1128
|
+
data.get('code') == 200 # HTTP状态码格式
|
|
1129
|
+
)
|
|
1130
|
+
|
|
1131
|
+
if not status_ok:
|
|
1132
|
+
return False
|
|
1133
|
+
|
|
1134
|
+
# 检查数据部分(支持多种数据字段名)
|
|
1135
|
+
has_data_fields = (
|
|
1136
|
+
'data' in data or # 标准data字段
|
|
1137
|
+
'logs' in data or # 更新日志专用
|
|
1138
|
+
'announcements' in data or # 公告数据
|
|
1139
|
+
'databases' in data or # 数据库列表
|
|
1140
|
+
'tables' in data or # 表列表
|
|
1141
|
+
'rows' in data or # 数据行
|
|
1142
|
+
'message' in data # 包含消息即认为有数据
|
|
1143
|
+
)
|
|
1144
|
+
|
|
1145
|
+
# 如果有数据字段,基本认为有效
|
|
1146
|
+
return has_data_fields
|
|
1147
|
+
|
|
1148
|
+
data_validator = default_validator
|
|
1149
|
+
|
|
1150
|
+
# 验证数据
|
|
1151
|
+
try:
|
|
1152
|
+
is_valid = data_validator(response_data)
|
|
1153
|
+
except Exception:
|
|
1154
|
+
return False
|
|
1155
|
+
|
|
1156
|
+
if is_valid:
|
|
1157
|
+
try:
|
|
1158
|
+
cache_system.set(cache_key, response_data, ttl=ttl, namespace=namespace)
|
|
1159
|
+
return True
|
|
1160
|
+
except Exception:
|
|
1161
|
+
return False
|
|
1162
|
+
else:
|
|
1163
|
+
return False
|
|
1164
|
+
|
mdbq-4.2.9/mdbq/__version__.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
VERSION = '4.2.9'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|