mdbq 3.10.2__py3-none-any.whl → 3.10.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/__version__.py +1 -1
- mdbq/mysql/uploader.py +20 -20
- {mdbq-3.10.2.dist-info → mdbq-3.10.3.dist-info}/METADATA +1 -1
- {mdbq-3.10.2.dist-info → mdbq-3.10.3.dist-info}/RECORD +6 -6
- {mdbq-3.10.2.dist-info → mdbq-3.10.3.dist-info}/WHEEL +0 -0
- {mdbq-3.10.2.dist-info → mdbq-3.10.3.dist-info}/top_level.txt +0 -0
mdbq/__version__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
VERSION = '3.10.
|
1
|
+
VERSION = '3.10.3'
|
mdbq/mysql/uploader.py
CHANGED
@@ -303,8 +303,8 @@ class MySQLUploader:
|
|
303
303
|
return exists
|
304
304
|
except Exception as e:
|
305
305
|
logger.error(sys._getframe().f_code.co_name, {
|
306
|
+
'库': db_name,
|
306
307
|
'检查数据库是否存在时出错': str(e),
|
307
|
-
'库': db_name
|
308
308
|
})
|
309
309
|
raise
|
310
310
|
|
@@ -546,9 +546,9 @@ class MySQLUploader:
|
|
546
546
|
|
547
547
|
except Exception as e:
|
548
548
|
logger.error(sys._getframe().f_code.co_name, {
|
549
|
-
'建表失败': str(e),
|
550
549
|
'库': db_name,
|
551
550
|
'表': table_name,
|
551
|
+
'建表失败': str(e),
|
552
552
|
})
|
553
553
|
conn.rollback()
|
554
554
|
raise
|
@@ -678,9 +678,9 @@ class MySQLUploader:
|
|
678
678
|
return set_typ
|
679
679
|
except Exception as e:
|
680
680
|
logger.error(sys._getframe().f_code.co_name, {
|
681
|
-
'无法获取表列信息': str(e),
|
682
681
|
'库': db_name,
|
683
682
|
'表': table_name,
|
683
|
+
'无法获取表列信息': str(e),
|
684
684
|
})
|
685
685
|
raise
|
686
686
|
|
@@ -709,9 +709,9 @@ class MySQLUploader:
|
|
709
709
|
allow_null=allow_null)
|
710
710
|
else:
|
711
711
|
logger.error(sys._getframe().f_code.co_name, {
|
712
|
-
'数据表不存在': table_name,
|
713
712
|
'库': db_name,
|
714
713
|
'表': table_name,
|
714
|
+
'数据表不存在': table_name,
|
715
715
|
})
|
716
716
|
raise ValueError(f"数据表不存在: `{db_name}`.`{table_name}`")
|
717
717
|
|
@@ -719,9 +719,9 @@ class MySQLUploader:
|
|
719
719
|
table_columns = self._get_table_columns(db_name, table_name)
|
720
720
|
if not table_columns:
|
721
721
|
logger.error(sys._getframe().f_code.co_name, {
|
722
|
-
'获取列失败': table_columns,
|
723
722
|
'库': db_name,
|
724
723
|
'表': table_name,
|
724
|
+
'获取列失败': table_columns,
|
725
725
|
})
|
726
726
|
raise ValueError(f"获取列失败 `{db_name}`.`{table_name}`")
|
727
727
|
|
@@ -729,9 +729,9 @@ class MySQLUploader:
|
|
729
729
|
for col in set_typ:
|
730
730
|
if col not in table_columns:
|
731
731
|
logger.error(sys._getframe().f_code.co_name, {
|
732
|
-
'列不存在': f'{col} -> {table_columns}',
|
733
732
|
'库': db_name,
|
734
733
|
'表': table_name,
|
734
|
+
'列不存在': f'{col} -> {table_columns}',
|
735
735
|
})
|
736
736
|
raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
|
737
737
|
|
@@ -960,7 +960,7 @@ class MySQLUploader:
|
|
960
960
|
默认值为 'batch'
|
961
961
|
:raises: 可能抛出各种验证和数据库相关异常
|
962
962
|
"""
|
963
|
-
upload_start = time.time()
|
963
|
+
# upload_start = time.time()
|
964
964
|
initial_row_count = len(data) if hasattr(data, '__len__') else 1
|
965
965
|
|
966
966
|
batch_id = f"batch_{int(time.time() * 1000)}"
|
@@ -972,7 +972,7 @@ class MySQLUploader:
|
|
972
972
|
'批次': batch_id,
|
973
973
|
'分表方式': partition_by,
|
974
974
|
'排重': check_duplicate,
|
975
|
-
'
|
975
|
+
'传入': len(data) if hasattr(data, '__len__') else 1,
|
976
976
|
'自动建表': auto_create
|
977
977
|
})
|
978
978
|
|
@@ -985,10 +985,10 @@ class MySQLUploader:
|
|
985
985
|
partition_by = str(partition_by).lower()
|
986
986
|
if partition_by not in ['year', 'month']:
|
987
987
|
logger.error(sys._getframe().f_code.co_name, {
|
988
|
-
'分表方式必须是 "year" 或 "month" 或 "None"': partition_by,
|
989
988
|
'库': db_name,
|
990
989
|
'表': table_name,
|
991
|
-
'批次': batch_id
|
990
|
+
'批次': batch_id,
|
991
|
+
'分表方式必须是 "year" 或 "month" 或 "None"': partition_by,
|
992
992
|
})
|
993
993
|
raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
|
994
994
|
|
@@ -1029,10 +1029,10 @@ class MySQLUploader:
|
|
1029
1029
|
partitioned_data[part_table].append(row)
|
1030
1030
|
except Exception as e:
|
1031
1031
|
logger.error(sys._getframe().f_code.co_name, {
|
1032
|
-
'row_data': row,
|
1033
|
-
'分表处理失败': str(e),
|
1034
1032
|
'库': db_name,
|
1035
1033
|
'表': table_name,
|
1034
|
+
'row_data': row,
|
1035
|
+
'分表处理失败': str(e),
|
1036
1036
|
})
|
1037
1037
|
continue # 跳过当前行
|
1038
1038
|
|
@@ -1047,10 +1047,10 @@ class MySQLUploader:
|
|
1047
1047
|
)
|
1048
1048
|
except Exception as e:
|
1049
1049
|
logger.error(sys._getframe().f_code.co_name, {
|
1050
|
-
'分表': part_table,
|
1051
|
-
'分表上传失败': str(e),
|
1052
1050
|
'库': db_name,
|
1053
1051
|
'表': table_name,
|
1052
|
+
'分表': part_table,
|
1053
|
+
'分表上传失败': str(e),
|
1054
1054
|
})
|
1055
1055
|
continue # 跳过当前分表,继续处理其他分表
|
1056
1056
|
else:
|
@@ -1066,10 +1066,10 @@ class MySQLUploader:
|
|
1066
1066
|
|
1067
1067
|
except Exception as e:
|
1068
1068
|
logger.error(sys._getframe().f_code.co_name, {
|
1069
|
-
'上传过程发生全局错误': str(e),
|
1070
|
-
'error_type': type(e).__name__,
|
1071
1069
|
'库': db_name,
|
1072
1070
|
'表': table_name,
|
1071
|
+
'上传过程发生全局错误': str(e),
|
1072
|
+
'error_type': type(e).__name__,
|
1073
1073
|
})
|
1074
1074
|
finally:
|
1075
1075
|
logger.info("存储完成", {
|
@@ -1077,7 +1077,7 @@ class MySQLUploader:
|
|
1077
1077
|
'表': table_name,
|
1078
1078
|
'批次': batch_id,
|
1079
1079
|
'success': success_flag,
|
1080
|
-
'耗时': round(time.time() - upload_start, 2),
|
1080
|
+
# '耗时': round(time.time() - upload_start, 2),
|
1081
1081
|
'数据行': initial_row_count
|
1082
1082
|
})
|
1083
1083
|
|
@@ -1164,8 +1164,8 @@ class MySQLUploader:
|
|
1164
1164
|
update_on_duplicate: bool
|
1165
1165
|
) -> str:
|
1166
1166
|
"""准备插入SQL语句"""
|
1167
|
-
# 获取所有列名(排除id列)
|
1168
|
-
all_columns = [col for col in set_typ.keys() if col.lower()
|
1167
|
+
# 获取所有列名(排除 `id`、`更新时间` 列)
|
1168
|
+
all_columns = [col for col in set_typ.keys() if col.lower() not in ['id', '更新时间']]
|
1169
1169
|
safe_columns = [self._validate_identifier(col) for col in all_columns]
|
1170
1170
|
placeholders = ','.join(['%s'] * len(safe_columns))
|
1171
1171
|
|
@@ -1173,7 +1173,7 @@ class MySQLUploader:
|
|
1173
1173
|
if not duplicate_columns:
|
1174
1174
|
duplicate_columns = all_columns
|
1175
1175
|
else:
|
1176
|
-
duplicate_columns = [col for col in duplicate_columns if col
|
1176
|
+
duplicate_columns = [col for col in duplicate_columns if col.lower() not in ['id', '更新时间']]
|
1177
1177
|
|
1178
1178
|
conditions = []
|
1179
1179
|
for col in duplicate_columns:
|
@@ -1,5 +1,5 @@
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
2
|
-
mdbq/__version__.py,sha256=
|
2
|
+
mdbq/__version__.py,sha256=S1pYeTgXo5MtZqzwck9ASp8x1pB1PZ33oC1NI7fY9dI,18
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
4
4
|
mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
|
5
5
|
mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
|
@@ -12,7 +12,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
|
12
12
|
mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
|
13
13
|
mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
|
14
14
|
mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
|
15
|
-
mdbq/mysql/uploader.py,sha256=
|
15
|
+
mdbq/mysql/uploader.py,sha256=9wgFxsiFSAngdX2pWj57jElaspwqfPtadC-xQqvweUc,59066
|
16
16
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
17
17
|
mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
|
18
18
|
mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
|
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
|
|
25
25
|
mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
|
26
26
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
27
27
|
mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
|
28
|
-
mdbq-3.10.
|
29
|
-
mdbq-3.10.
|
30
|
-
mdbq-3.10.
|
31
|
-
mdbq-3.10.
|
28
|
+
mdbq-3.10.3.dist-info/METADATA,sha256=TM8JAb8gTTte7N0agKbaDWZ14bmkl66dgCxIbLTqCbc,364
|
29
|
+
mdbq-3.10.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
|
30
|
+
mdbq-3.10.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
31
|
+
mdbq-3.10.3.dist-info/RECORD,,
|
File without changes
|
File without changes
|