mdbq 4.2.8__py3-none-any.whl → 4.2.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of mdbq might be problematic. Click here for more details.

mdbq/__version__.py CHANGED
@@ -1 +1 @@
1
- VERSION = '4.2.8'
1
+ VERSION = '4.2.10'
mdbq/mysql/uploader.py CHANGED
@@ -11,8 +11,6 @@ from typing import Union, List, Dict, Optional, Any, Tuple, Iterator
11
11
  from functools import wraps
12
12
  from decimal import Decimal, InvalidOperation
13
13
  import math
14
- import concurrent.futures
15
- import threading
16
14
  import pymysql
17
15
  import pandas as pd
18
16
  import psutil
@@ -267,18 +265,26 @@ class DataTypeInferrer:
267
265
  # 采样数据进行类型推断
268
266
  sample_data = data[:sample_size] if len(data) > sample_size else data
269
267
 
268
+ # 首先收集所有列名
269
+ all_columns = set()
270
+ for row in sample_data:
271
+ for col in row.keys():
272
+ if col.lower() not in ['id', 'create_at', 'update_at']:
273
+ all_columns.add(col)
274
+
275
+ # 为每个列初始化候选类型列表
276
+ for col in all_columns:
277
+ type_candidates[col] = []
278
+
270
279
  for row in sample_data:
271
280
  for col, value in row.items():
272
281
  # 跳过系统列
273
282
  if col.lower() in ['id', 'create_at', 'update_at']:
274
283
  continue
275
284
 
276
- if value is not None and str(value).strip():
277
- mysql_type = DataTypeInferrer.infer_mysql_type(value)
278
-
279
- if col not in type_candidates:
280
- type_candidates[col] = []
281
- type_candidates[col].append(mysql_type)
285
+ # 即使值为空,也要推断类型
286
+ mysql_type = DataTypeInferrer.infer_mysql_type(value)
287
+ type_candidates[col].append(mysql_type)
282
288
 
283
289
  # 为每列选择最合适的类型
284
290
  for col, types in type_candidates.items():
@@ -684,6 +690,19 @@ class TableManager:
684
690
  db_name = self._sanitize_identifier(db_name)
685
691
  table_name = self._sanitize_identifier(table_name)
686
692
 
693
+ # 验证columns不为空
694
+ if not columns:
695
+ raise ValueError(f"创建表失败:columns不能为空。数据库: {db_name}, 表: {table_name}")
696
+
697
+ # 验证unique_keys中的列是否存在于columns中
698
+ if unique_keys:
699
+ business_columns = {k.lower(): k for k in columns.keys() if k.lower() not in ['id', 'create_at', 'update_at']}
700
+ for i, uk in enumerate(unique_keys):
701
+ for col in uk:
702
+ col_lower = col.lower()
703
+ if col_lower not in business_columns and col not in columns:
704
+ raise ValueError(f"唯一约束中的列 '{col}' 不存在于表定义中。可用列: {list(business_columns.keys())}")
705
+
687
706
  # 构建列定义
688
707
  column_defs = []
689
708
 
@@ -716,8 +735,15 @@ class TableManager:
716
735
  safe_uk_parts = []
717
736
  for col in filtered_uk:
718
737
  safe_col_name = self._sanitize_identifier(col)
719
- # 检查是否需要前缀索引
720
- col_type = columns.get(col, 'varchar(255)').lower()
738
+ # 检查是否需要前缀索引 - 优先使用原始列名,然后尝试小写
739
+ col_lower = col.lower()
740
+ if col in columns:
741
+ col_type = columns[col].lower()
742
+ elif col_lower in columns:
743
+ col_type = columns[col_lower].lower()
744
+ else:
745
+ col_type = 'varchar(255)'
746
+
721
747
  if 'varchar' in col_type:
722
748
  # 提取varchar长度
723
749
  match = re.search(r'varchar\((\d+)\)', col_type)
@@ -727,20 +753,11 @@ class TableManager:
727
753
  if length > 191:
728
754
  prefix_length = 191
729
755
  safe_uk_parts.append(f"`{safe_col_name}`({prefix_length})")
730
- logger.debug('应用前缀索引', {
731
- '列名': col,
732
- '原始长度': length,
733
- '前缀长度': prefix_length
734
- })
735
756
  else:
736
757
  safe_uk_parts.append(f"`{safe_col_name}`")
737
758
  else:
738
759
  # 如果没有指定长度,默认使用前缀索引
739
760
  safe_uk_parts.append(f"`{safe_col_name}`(191)")
740
- logger.debug('应用默认前缀索引', {
741
- '列名': col,
742
- '前缀长度': 191
743
- })
744
761
  else:
745
762
  # 非varchar字段保持原样
746
763
  safe_uk_parts.append(f"`{safe_col_name}`")
@@ -760,9 +777,17 @@ class TableManager:
760
777
 
761
778
  with self.conn_mgr.get_connection() as conn:
762
779
  with conn.cursor() as cursor:
763
- cursor.execute(sql)
764
- conn.commit()
765
- logger.debug('表已创建', {'database': db_name, 'table': table_name})
780
+ try:
781
+ cursor.execute(sql)
782
+ conn.commit()
783
+ logger.debug('表已创建', {'database': db_name, 'table': table_name})
784
+ except Exception as e:
785
+ logger.error('创建表失败', {
786
+ 'database': db_name,
787
+ 'table': table_name,
788
+ 'error': str(e)
789
+ })
790
+ raise
766
791
 
767
792
  def get_partition_table_name(self, base_name: str, date_value: str, partition_by: str) -> str:
768
793
  """获取分表名称"""
@@ -806,8 +831,6 @@ class TableManager:
806
831
  return cleaned
807
832
 
808
833
 
809
-
810
-
811
834
  class DataProcessor:
812
835
  """数据处理器"""
813
836
 
@@ -1169,21 +1192,35 @@ class MySQLUploader:
1169
1192
  normalized_data = DataProcessor.normalize_data(data)
1170
1193
 
1171
1194
  # 推断或验证列类型
1172
- if set_typ is None:
1195
+ if set_typ is None or not set_typ:
1173
1196
  # 取第一个chunk进行类型推断
1174
1197
  first_chunk = next(iter(normalized_data))
1198
+
1199
+ if not first_chunk:
1200
+ raise ValueError("数据为空,无法推断列类型")
1201
+
1175
1202
  set_typ = DataTypeInferrer.infer_types_from_data(first_chunk)
1176
1203
  # 重新创建迭代器
1177
1204
  normalized_data = DataProcessor.normalize_data(data)
1178
1205
  logger.debug('自动推断数据类型', {'类型映射': set_typ})
1206
+
1207
+ # 验证推断结果
1208
+ if not set_typ or not any(col for col in set_typ.keys() if col.lower() not in ['id', 'create_at', 'update_at']):
1209
+ raise ValueError(f"类型推断失败,无有效业务列。推断结果: {set_typ}")
1179
1210
 
1180
1211
  # 将set_typ的键统一转为小写
1181
1212
  set_typ = self.tran_set_typ_to_lower(set_typ)
1182
1213
 
1214
+ # 最终验证:确保有业务列定义
1215
+ business_columns = {k: v for k, v in set_typ.items() if k.lower() not in ['id', 'create_at', 'update_at']}
1216
+ if not business_columns:
1217
+ raise ValueError(f"没有有效的业务列定义。set_typ: {set_typ}")
1218
+
1183
1219
  # 确保数据库存在
1184
1220
  self.table_mgr.ensure_database_exists(db_name)
1185
1221
 
1186
1222
  # 处理分表逻辑
1223
+
1187
1224
  if partition_by:
1188
1225
  upload_result = self._handle_partitioned_upload(
1189
1226
  db_name, table_name, normalized_data, set_typ,
@@ -1389,6 +1426,16 @@ class MySQLUploader:
1389
1426
  main_result['failed_rows'] += partition_result['failed_rows']
1390
1427
  main_result['tables_created'].extend(partition_result['tables_created'])
1391
1428
 
1429
+ def tran_set_typ_to_lower(self, set_typ: Dict[str, str]) -> Dict[str, str]:
1430
+ if not isinstance(set_typ, dict) or set_typ is None:
1431
+ return {}
1432
+
1433
+ set_typ_lower = {}
1434
+ for key, value in set_typ.items():
1435
+ set_typ_lower[key.lower()] = value
1436
+
1437
+ return set_typ_lower
1438
+
1392
1439
  def close(self):
1393
1440
  """关闭连接"""
1394
1441
  if self.conn_mgr:
@@ -1406,178 +1453,6 @@ class MySQLUploader:
1406
1453
  def __exit__(self, exc_type, exc_val, exc_tb):
1407
1454
  self.close()
1408
1455
 
1409
- def upload_data_concurrent(self, db_name: str, table_name: str,
1410
- data: Union[Dict, List[Dict], pd.DataFrame],
1411
- set_typ: Optional[Dict[str, str]] = None,
1412
- allow_null: bool = False,
1413
- partition_by: Optional[str] = None,
1414
- partition_date_column: str = '日期',
1415
- update_on_duplicate: bool = False,
1416
- unique_keys: Optional[List[List[str]]] = None,
1417
- max_workers: int = 3) -> Dict[str, Any]:
1418
- """
1419
- 并发上传数据到MySQL数据库
1420
-
1421
- :param max_workers: 最大并发工作线程数
1422
- :return: 上传结果详情
1423
- """
1424
- db_name = db_name.lower()
1425
- table_name = table_name.lower()
1426
-
1427
- result = {
1428
- 'success': False,
1429
- 'inserted_rows': 0,
1430
- 'skipped_rows': 0,
1431
- 'failed_rows': 0,
1432
- 'tables_created': []
1433
- }
1434
-
1435
- try:
1436
- # 标准化数据为流式迭代器
1437
- normalized_data = DataProcessor.normalize_data(data, chunk_size=2000) # 更小的chunk用于并发
1438
-
1439
- # 推断或验证列类型
1440
- if set_typ is None:
1441
- first_chunk = next(iter(normalized_data))
1442
- set_typ = DataTypeInferrer.infer_types_from_data(first_chunk)
1443
- normalized_data = DataProcessor.normalize_data(data, chunk_size=2000)
1444
- logger.debug('自动推断数据类型', {'类型映射': set_typ})
1445
-
1446
- # 将set_typ的键统一转为小写
1447
- set_typ = self.tran_set_typ_to_lower(set_typ)
1448
-
1449
- # 确保数据库存在
1450
- self.table_mgr.ensure_database_exists(db_name)
1451
-
1452
- # 创建线程锁用于表创建的线程安全
1453
- table_creation_lock = threading.Lock()
1454
- created_tables_set = set()
1455
-
1456
- def process_chunk_worker(chunk_data):
1457
- """工作线程函数"""
1458
- try:
1459
- if partition_by:
1460
- # 分表处理
1461
- partitioned_chunk = DataProcessor.partition_data_by_date(
1462
- chunk_data, partition_date_column, partition_by
1463
- )
1464
-
1465
- chunk_result = {
1466
- 'inserted_rows': 0,
1467
- 'skipped_rows': 0,
1468
- 'failed_rows': 0,
1469
- 'tables_created': []
1470
- }
1471
-
1472
- for partition_suffix, partition_data in partitioned_chunk.items():
1473
- partition_table_name = f"{table_name}_{partition_suffix}"
1474
- table_key = f"{db_name}.{partition_table_name}"
1475
-
1476
- # 确保表存在(线程安全)
1477
- with table_creation_lock:
1478
- if table_key not in created_tables_set:
1479
- if not self.table_mgr.table_exists(db_name, partition_table_name):
1480
- self.table_mgr.create_table(db_name, partition_table_name, set_typ,
1481
- unique_keys=unique_keys, allow_null=allow_null)
1482
- chunk_result['tables_created'].append(table_key)
1483
- else:
1484
- self.table_mgr.ensure_system_columns(db_name, partition_table_name)
1485
- created_tables_set.add(table_key)
1486
-
1487
- # 准备并插入数据
1488
- prepared_data = DataProcessor.prepare_data_for_insert(
1489
- partition_data, set_typ, allow_null
1490
- )
1491
-
1492
- inserted, skipped, failed = self.data_inserter.insert_data(
1493
- db_name, partition_table_name, prepared_data, set_typ, update_on_duplicate
1494
- )
1495
-
1496
- chunk_result['inserted_rows'] += inserted
1497
- chunk_result['skipped_rows'] += skipped
1498
- chunk_result['failed_rows'] += failed
1499
- else:
1500
- # 单表处理
1501
- table_key = f"{db_name}.{table_name}"
1502
- with table_creation_lock:
1503
- if table_key not in created_tables_set:
1504
- if not self.table_mgr.table_exists(db_name, table_name):
1505
- self.table_mgr.create_table(db_name, table_name, set_typ,
1506
- unique_keys=unique_keys, allow_null=allow_null)
1507
- chunk_result = {'tables_created': [table_key]}
1508
- else:
1509
- self.table_mgr.ensure_system_columns(db_name, table_name)
1510
- chunk_result = {'tables_created': []}
1511
- created_tables_set.add(table_key)
1512
- else:
1513
- chunk_result = {'tables_created': []}
1514
-
1515
- prepared_chunk = DataProcessor.prepare_data_for_insert(
1516
- chunk_data, set_typ, allow_null
1517
- )
1518
-
1519
- inserted, skipped, failed = self.data_inserter.insert_data(
1520
- db_name, table_name, prepared_chunk, set_typ, update_on_duplicate
1521
- )
1522
-
1523
- chunk_result.update({
1524
- 'inserted_rows': inserted,
1525
- 'skipped_rows': skipped,
1526
- 'failed_rows': failed
1527
- })
1528
-
1529
- return chunk_result
1530
-
1531
- except Exception as e:
1532
- logger.error('并发处理chunk失败', {'错误': str(e)})
1533
- return {
1534
- 'inserted_rows': 0,
1535
- 'skipped_rows': 0,
1536
- 'failed_rows': len(chunk_data) if chunk_data else 0,
1537
- 'tables_created': []
1538
- }
1539
-
1540
- # 使用线程池执行并发处理
1541
- with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
1542
- # 提交所有任务
1543
- future_to_chunk = {}
1544
- for chunk in normalized_data:
1545
- if chunk:
1546
- future = executor.submit(process_chunk_worker, chunk)
1547
- future_to_chunk[future] = len(chunk)
1548
-
1549
- # 收集结果
1550
- for future in concurrent.futures.as_completed(future_to_chunk):
1551
- chunk_result = future.result()
1552
- result['inserted_rows'] += chunk_result['inserted_rows']
1553
- result['skipped_rows'] += chunk_result['skipped_rows']
1554
- result['failed_rows'] += chunk_result['failed_rows']
1555
- result['tables_created'].extend(chunk_result['tables_created'])
1556
-
1557
- # 去重tables_created
1558
- result['tables_created'] = list(set(result['tables_created']))
1559
- result['success'] = result['failed_rows'] == 0
1560
-
1561
- except Exception as e:
1562
- logger.error('并发数据上传失败', {
1563
- '数据库': db_name,
1564
- '表名': table_name,
1565
- '错误': str(e)
1566
- })
1567
- result['success'] = False
1568
-
1569
- return result
1570
-
1571
- def tran_set_typ_to_lower(self, set_typ: Dict[str, str]) -> Dict[str, str]:
1572
- if not isinstance(set_typ, dict):
1573
- return set_typ
1574
-
1575
- set_typ_lower = {}
1576
- for key, value in set_typ.items():
1577
- set_typ_lower[key.lower()] = value
1578
-
1579
- return set_typ_lower
1580
-
1581
1456
 
1582
1457
  # 使用示例
1583
1458
  if __name__ == '__main__':
mdbq/redis/redis_cache.py CHANGED
@@ -879,4 +879,286 @@ class CacheManager:
879
879
 
880
880
 
881
881
  # 全局缓存管理器实例
882
- cache_manager = CacheManager()
882
+ cache_manager = CacheManager()
883
+
884
+
885
+ # ===== 装饰器功能 =====
886
+
887
+ def flask_redis_cache(cache_key_func=None, ttl=1200, namespace="default",
888
+ data_validator=None, skip_cache_on_error=True):
889
+ """
890
+ Flask路由函数的Redis缓存装饰器
891
+
892
+ Args:
893
+ cache_key_func: 缓存键生成函数,接收请求数据作为参数,返回缓存键字符串
894
+ 如果为None,则使用默认的键生成策略
895
+ ttl: 缓存过期时间(秒),默认20分钟
896
+ namespace: 缓存命名空间,默认为"default"
897
+ data_validator: 数据验证函数,用于验证数据是否应该被缓存
898
+ skip_cache_on_error: 当缓存操作出错时是否跳过缓存,默认True
899
+
900
+ Usage:
901
+ @flask_redis_cache(
902
+ cache_key_func=lambda data: f"tables_{data.get('database', 'unknown')}",
903
+ ttl=1200,
904
+ namespace="sycm_tables"
905
+ )
906
+ def my_flask_route():
907
+ pass
908
+ """
909
+ def decorator(func):
910
+ import functools
911
+ import hashlib
912
+
913
+ @functools.wraps(func)
914
+ def wrapper(*args, **kwargs):
915
+ # 导入Flask相关模块(延迟导入避免依赖问题)
916
+ try:
917
+ from flask import request, jsonify
918
+ except ImportError:
919
+ # 如果没有Flask环境,直接执行原函数
920
+ return func(*args, **kwargs)
921
+
922
+ # 获取缓存系统
923
+ cache_system = cache_manager.get_cache()
924
+
925
+ # 如果缓存系统不可用,直接执行原函数
926
+ if not cache_system:
927
+ return func(*args, **kwargs)
928
+
929
+ try:
930
+ # 获取请求数据用于生成缓存键
931
+ request_data = {}
932
+ if request.method == 'POST':
933
+ try:
934
+ request_data = request.get_json() or {}
935
+ except Exception:
936
+ request_data = {}
937
+ elif request.method == 'GET':
938
+ request_data = dict(request.args)
939
+
940
+ # 生成缓存键
941
+ if cache_key_func:
942
+ cache_key = cache_key_func(request_data)
943
+ else:
944
+ # 默认缓存键生成策略
945
+ func_name = func.__name__
946
+ # 将请求数据转换为字符串并生成哈希
947
+ data_str = str(sorted(request_data.items()))
948
+ data_hash = hashlib.md5(data_str.encode()).hexdigest()[:8]
949
+ cache_key = f"{func_name}_{data_hash}"
950
+
951
+ # 尝试从缓存获取数据
952
+ try:
953
+ cached_result = cache_system.get(cache_key, namespace)
954
+ if cached_result is not None:
955
+ return jsonify(cached_result)
956
+ except Exception as e:
957
+ if not skip_cache_on_error:
958
+ raise
959
+
960
+ # 缓存未命中,执行原函数
961
+ result = func(*args, **kwargs)
962
+
963
+ # 如果结果是Flask Response对象,提取JSON数据进行缓存
964
+ if hasattr(result, 'get_json'):
965
+ try:
966
+ response_data = result.get_json()
967
+ if response_data:
968
+ # 使用安全缓存写入
969
+ _safe_cache_set(
970
+ cache_system=cache_system,
971
+ cache_key=cache_key,
972
+ response_data=response_data,
973
+ ttl=ttl,
974
+ namespace=namespace,
975
+ data_validator=data_validator
976
+ )
977
+ except Exception as e:
978
+ if not skip_cache_on_error:
979
+ raise
980
+ elif isinstance(result, tuple) and len(result) == 2:
981
+ # 处理 (response, status_code) 格式的返回值
982
+ try:
983
+ response_data, status_code = result
984
+ if hasattr(response_data, 'get_json'):
985
+ json_data = response_data.get_json()
986
+ elif isinstance(response_data, dict):
987
+ json_data = response_data
988
+ else:
989
+ json_data = None
990
+
991
+ if json_data and status_code == 200:
992
+ _safe_cache_set(
993
+ cache_system=cache_system,
994
+ cache_key=cache_key,
995
+ response_data=json_data,
996
+ ttl=ttl,
997
+ namespace=namespace,
998
+ data_validator=data_validator
999
+ )
1000
+ except Exception as e:
1001
+ if not skip_cache_on_error:
1002
+ raise
1003
+
1004
+ return result
1005
+
1006
+ except Exception as e:
1007
+ if not skip_cache_on_error:
1008
+ raise
1009
+ return func(*args, **kwargs)
1010
+
1011
+ return wrapper
1012
+ return decorator
1013
+
1014
+
1015
+ def function_redis_cache(cache_key_func=None, ttl=1800, namespace="default",
1016
+ skip_cache_on_error=True):
1017
+ """
1018
+ 普通函数的Redis缓存装饰器
1019
+
1020
+ Args:
1021
+ cache_key_func: 缓存键生成函数,接收函数参数作为输入,返回缓存键字符串
1022
+ 如果为None,则使用默认的键生成策略
1023
+ ttl: 缓存过期时间(秒),默认30分钟
1024
+ namespace: 缓存命名空间,默认为"default"
1025
+ skip_cache_on_error: 当缓存操作出错时是否跳过缓存,默认True
1026
+
1027
+ Usage:
1028
+ @function_redis_cache(
1029
+ cache_key_func=lambda _key, shop_name: f"cookies_{_key}_{shop_name}",
1030
+ ttl=1800,
1031
+ namespace="cookies_cache"
1032
+ )
1033
+ def my_function(_key, shop_name):
1034
+ pass
1035
+ """
1036
+ def decorator(func):
1037
+ import functools
1038
+ import inspect
1039
+ import hashlib
1040
+
1041
+ @functools.wraps(func)
1042
+ def wrapper(*args, **kwargs):
1043
+ # 获取缓存系统
1044
+ cache_system = cache_manager.get_cache()
1045
+
1046
+ # 如果缓存系统不可用,直接执行原函数
1047
+ if not cache_system:
1048
+ return func(*args, **kwargs)
1049
+
1050
+ try:
1051
+ # 获取函数签名和参数
1052
+ sig = inspect.signature(func)
1053
+ bound_args = sig.bind(*args, **kwargs)
1054
+ bound_args.apply_defaults()
1055
+
1056
+ # 生成缓存键
1057
+ if cache_key_func:
1058
+ cache_key = cache_key_func(*args, **kwargs)
1059
+ else:
1060
+ # 默认缓存键生成策略
1061
+ func_name = func.__name__
1062
+ # 将参数转换为字符串并生成哈希
1063
+ args_str = str(args) + str(sorted(kwargs.items()))
1064
+ args_hash = hashlib.md5(args_str.encode()).hexdigest()[:8]
1065
+ cache_key = f"{func_name}_{args_hash}"
1066
+
1067
+ # 尝试从缓存获取数据
1068
+ try:
1069
+ cached_result = cache_system.get(cache_key, namespace)
1070
+ if cached_result is not None:
1071
+ return cached_result
1072
+ except Exception as e:
1073
+ if not skip_cache_on_error:
1074
+ raise
1075
+
1076
+ # 缓存未命中,执行原函数
1077
+ result = func(*args, **kwargs)
1078
+
1079
+ # 缓存结果(只缓存非空结果)
1080
+ if result is not None and result != {} and result != []:
1081
+ try:
1082
+ cache_system.set(cache_key, result, ttl=ttl, namespace=namespace)
1083
+ except Exception as e:
1084
+ if not skip_cache_on_error:
1085
+ raise
1086
+
1087
+ return result
1088
+
1089
+ except Exception as e:
1090
+ if not skip_cache_on_error:
1091
+ raise
1092
+ return func(*args, **kwargs)
1093
+
1094
+ return wrapper
1095
+ return decorator
1096
+
1097
+
1098
+ def _safe_cache_set(cache_system, cache_key, response_data, ttl, namespace,
1099
+ data_validator=None):
1100
+ """
1101
+ 安全的缓存写入函数,只有数据有效时才写入缓存。
1102
+
1103
+ Args:
1104
+ cache_system: 缓存系统实例
1105
+ cache_key: 缓存键
1106
+ response_data: 要缓存的响应数据
1107
+ ttl: 缓存过期时间
1108
+ namespace: 缓存命名空间
1109
+ data_validator: 数据验证函数,返回True表示数据有效
1110
+
1111
+ Returns:
1112
+ bool: 是否成功写入缓存
1113
+ """
1114
+ if not cache_system:
1115
+ return False
1116
+
1117
+ # 默认验证逻辑:检查响应数据结构
1118
+ if data_validator is None:
1119
+ def default_validator(data):
1120
+ if not isinstance(data, dict):
1121
+ return False
1122
+
1123
+ # 更宽松的验证逻辑,支持多种响应格式
1124
+ # 检查状态字段(支持多种成功状态格式)
1125
+ status_ok = (
1126
+ data.get('status') == 'success' or # 新格式
1127
+ data.get('code') == 0 or # 旧格式
1128
+ data.get('code') == 200 # HTTP状态码格式
1129
+ )
1130
+
1131
+ if not status_ok:
1132
+ return False
1133
+
1134
+ # 检查数据部分(支持多种数据字段名)
1135
+ has_data_fields = (
1136
+ 'data' in data or # 标准data字段
1137
+ 'logs' in data or # 更新日志专用
1138
+ 'announcements' in data or # 公告数据
1139
+ 'databases' in data or # 数据库列表
1140
+ 'tables' in data or # 表列表
1141
+ 'rows' in data or # 数据行
1142
+ 'message' in data # 包含消息即认为有数据
1143
+ )
1144
+
1145
+ # 如果有数据字段,基本认为有效
1146
+ return has_data_fields
1147
+
1148
+ data_validator = default_validator
1149
+
1150
+ # 验证数据
1151
+ try:
1152
+ is_valid = data_validator(response_data)
1153
+ except Exception:
1154
+ return False
1155
+
1156
+ if is_valid:
1157
+ try:
1158
+ cache_system.set(cache_key, response_data, ttl=ttl, namespace=namespace)
1159
+ return True
1160
+ except Exception:
1161
+ return False
1162
+ else:
1163
+ return False
1164
+
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: mdbq
3
- Version: 4.2.8
3
+ Version: 4.2.10
4
4
  Home-page: https://pypi.org/project/mdbq
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
@@ -1,5 +1,5 @@
1
1
  mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
2
- mdbq/__version__.py,sha256=bRBQbqec4RjxzcwssgAcf6Xb3LHtO4NQTk0fuRkUXaY,17
2
+ mdbq/__version__.py,sha256=MrwjVWD5rtRL9-vXKFMy6HJ06DkpWqj_fUcianwbRG8,18
3
3
  mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
4
4
  mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
5
5
  mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -15,7 +15,7 @@ mdbq/mysql/deduplicator.py,sha256=VGRBcIEsWUqQovkGpGCOittSW1l5fuMpiNWLbiOA5tI,72
15
15
  mdbq/mysql/mysql.py,sha256=eSrEE6DdS3GJ_EPPSggEZXhnVz6IqHPwUSsWfZJVFkY,57081
16
16
  mdbq/mysql/s_query.py,sha256=rCJZ0XPalwk3Z7HkqsvF9o0UX0sZT4Wk1JXryRrV6yQ,50472
17
17
  mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
18
- mdbq/mysql/uploader.py,sha256=GAJIJO3uQg8G8OciZlH6duXkEqroUaU5j5hgET7Ck8Y,65370
18
+ mdbq/mysql/uploader.py,sha256=bUT8Xhh97a5zdS0YqsKIJpdievcRnxswmJMrZcKfsbo,59157
19
19
  mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
20
20
  mdbq/other/download_sku_picture.py,sha256=pxcoNYoTP5KAhyvyHO2NQIrfhige5UN-LuB9Z8Iw3g8,45017
21
21
  mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -27,7 +27,7 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
27
27
  mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
28
28
  mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
29
29
  mdbq/redis/getredis.py,sha256=vdg7YQEjhoMp5QzxygNGx5DQKRnePrcwPYgUrDypA6g,23672
30
- mdbq/redis/redis_cache.py,sha256=hjl-oh1r0ONdMeWN28cTMsdkcjc7HG71NmGZ9H6jWUE,34559
30
+ mdbq/redis/redis_cache.py,sha256=D8VlzzbLVE_U9jgbkCjKtg8ak2UvryT8Ysi8C_DL0jM,45462
31
31
  mdbq/route/__init__.py,sha256=BT_dAY7V-U2o72bevq1B9Mq9QA7GodwtkxyLNdGaoE8,22
32
32
  mdbq/route/analytics.py,sha256=dngj5hVwKddEUy59nSYbOoJ9C7OVrtCmCkvW6Uj9RYM,28097
33
33
  mdbq/route/monitor.py,sha256=lyowGUU8c2GykeZLrdxd7nXpNMqXWcOsuQsbS8l0pwU,36595
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
35
35
  mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
36
36
  mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
37
37
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
38
- mdbq-4.2.8.dist-info/METADATA,sha256=TEvwYOF6KRkyAWFQWoCiVz4R7cJJyBAAChZdiMIcgew,363
39
- mdbq-4.2.8.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
40
- mdbq-4.2.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
41
- mdbq-4.2.8.dist-info/RECORD,,
38
+ mdbq-4.2.10.dist-info/METADATA,sha256=zwELJYHcPLZqzOYxMznf_iGEG4SKk8TjriZfcA3xvHo,364
39
+ mdbq-4.2.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
40
+ mdbq-4.2.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
41
+ mdbq-4.2.10.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.8.2)
2
+ Generator: setuptools (80.9.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5