mdbq 4.0.62__tar.gz → 4.0.64__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mdbq-4.0.62 → mdbq-4.0.64}/PKG-INFO +1 -1
- mdbq-4.0.64/mdbq/__version__.py +1 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/uploader.py +26 -12
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq.egg-info/PKG-INFO +1 -1
- mdbq-4.0.62/mdbq/__version__.py +0 -1
- {mdbq-4.0.62 → mdbq-4.0.64}/README.txt +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/log/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/log/mylogger.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/myconf/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/myconf/myconf.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/deduplicator.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/mysql.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/s_query.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/mysql/unique_.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/download_sku_picture.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/error_handler.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/otk.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/pov_city.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/other/ua_sj.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/pbix/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/pbix/pbix_refresh.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/pbix/refresh_all.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/redis/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/redis/getredis.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq/spider/__init__.py +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq.egg-info/SOURCES.txt +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq.egg-info/dependency_links.txt +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/mdbq.egg-info/top_level.txt +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/setup.cfg +0 -0
- {mdbq-4.0.62 → mdbq-4.0.64}/setup.py +0 -0
@@ -0,0 +1 @@
|
|
1
|
+
VERSION = '4.0.64'
|
@@ -267,7 +267,7 @@ class MySQLUploader:
|
|
267
267
|
:return: 存在返回True,否则返回False
|
268
268
|
:raises: 可能抛出数据库相关异常
|
269
269
|
"""
|
270
|
-
db_name = self._validate_identifier(db_name)
|
270
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
271
271
|
sql = 'SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s'
|
272
272
|
conn = None
|
273
273
|
try:
|
@@ -289,7 +289,7 @@ class MySQLUploader:
|
|
289
289
|
:param db_name: 要创建的数据库名称
|
290
290
|
:raises: 可能抛出数据库相关异常
|
291
291
|
"""
|
292
|
-
db_name = self._validate_identifier(db_name)
|
292
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
293
293
|
sql = f'CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}'
|
294
294
|
conn = None
|
295
295
|
try:
|
@@ -327,11 +327,12 @@ class MySQLUploader:
|
|
327
327
|
logger.error('分表方式无效', {'表': table_name, '分表方式': partition_by})
|
328
328
|
raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
|
329
329
|
|
330
|
-
def _validate_identifier(self, identifier: str) -> str:
|
330
|
+
def _validate_identifier(self, identifier: str, is_database: bool = False) -> str:
|
331
331
|
"""
|
332
332
|
验证并清理数据库标识符(表名、列名等)
|
333
333
|
|
334
334
|
:param identifier: 要验证的标识符
|
335
|
+
:param is_database: 是否为数据库名,数据库名不能以数字开头
|
335
336
|
:return: 清理后的安全标识符
|
336
337
|
:raises ValueError: 当标识符无效时抛出
|
337
338
|
"""
|
@@ -349,9 +350,14 @@ class MySQLUploader:
|
|
349
350
|
hash_suffix = hashlib.md5(identifier.encode('utf-8')).hexdigest()[:8]
|
350
351
|
cleaned = f'unknown_col_{hash_suffix}'
|
351
352
|
|
352
|
-
#
|
353
|
-
if cleaned and cleaned[0].isdigit():
|
354
|
-
cleaned = f'
|
353
|
+
# 数据库名不能以数字开头(MySQL要求),但表名和列名可以
|
354
|
+
if is_database and cleaned and cleaned[0].isdigit():
|
355
|
+
cleaned = f'db_{cleaned}'
|
356
|
+
logger.warning('为数字开头的数据库名添加db_前缀', {
|
357
|
+
'原始标识符': identifier,
|
358
|
+
'清理后': cleaned
|
359
|
+
})
|
360
|
+
|
355
361
|
mysql_keywords = {
|
356
362
|
'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
|
357
363
|
'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
|
@@ -379,7 +385,7 @@ class MySQLUploader:
|
|
379
385
|
if time.time() - cached_time < self.metadata_cache_ttl:
|
380
386
|
logger.debug('表存在缓存命中', {'库': db_name, '表': table_name, '存在': result})
|
381
387
|
return result
|
382
|
-
db_name = self._validate_identifier(db_name)
|
388
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
383
389
|
table_name = self._validate_identifier(table_name)
|
384
390
|
sql = """
|
385
391
|
SELECT TABLE_NAME
|
@@ -413,7 +419,7 @@ class MySQLUploader:
|
|
413
419
|
"""
|
414
420
|
创建数据表,优化索引创建方式
|
415
421
|
"""
|
416
|
-
db_name = self._validate_identifier(db_name)
|
422
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
417
423
|
table_name = self._validate_identifier(table_name)
|
418
424
|
if not set_typ:
|
419
425
|
logger.error('建表时未指定set_typ', {'库': db_name, '表': table_name})
|
@@ -859,7 +865,7 @@ class MySQLUploader:
|
|
859
865
|
:return: 列名和数据类型字典 {列名: 数据类型}
|
860
866
|
:raises: 可能抛出数据库相关异常
|
861
867
|
"""
|
862
|
-
db_name = self._validate_identifier(db_name)
|
868
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
863
869
|
table_name = self._validate_identifier(table_name)
|
864
870
|
sql = """
|
865
871
|
SELECT COLUMN_NAME, DATA_TYPE
|
@@ -885,7 +891,7 @@ class MySQLUploader:
|
|
885
891
|
"""
|
886
892
|
确保某列有索引,如果没有则创建。
|
887
893
|
"""
|
888
|
-
db_name = self._validate_identifier(db_name)
|
894
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
889
895
|
table_name = self._validate_identifier(table_name)
|
890
896
|
column = self._validate_identifier(column)
|
891
897
|
# 检查索引是否已存在
|
@@ -914,7 +920,7 @@ class MySQLUploader:
|
|
914
920
|
获取表中所有UNIQUE KEY的列组合(不含主键)。
|
915
921
|
返回:[[col1, col2], ...]
|
916
922
|
"""
|
917
|
-
db_name = self._validate_identifier(db_name)
|
923
|
+
db_name = self._validate_identifier(db_name, is_database=True)
|
918
924
|
table_name = self._validate_identifier(table_name)
|
919
925
|
sql = '''
|
920
926
|
SELECT INDEX_NAME, COLUMN_NAME
|
@@ -1323,6 +1329,14 @@ class MySQLUploader:
|
|
1323
1329
|
total_inserted = 0
|
1324
1330
|
total_skipped = 0
|
1325
1331
|
total_failed = 0
|
1332
|
+
validated_primary_keys = None
|
1333
|
+
validated_indexes = None
|
1334
|
+
validated_unique_keys = None
|
1335
|
+
prepared_data = None
|
1336
|
+
filtered_set_typ = None
|
1337
|
+
inserted = None
|
1338
|
+
skipped = None
|
1339
|
+
failed = None
|
1326
1340
|
|
1327
1341
|
try:
|
1328
1342
|
# 验证参数格式
|
@@ -1487,7 +1501,7 @@ class MySQLUploader:
|
|
1487
1501
|
})
|
1488
1502
|
|
1489
1503
|
# 更新索引(只有在成功时才执行)
|
1490
|
-
if success_flag and
|
1504
|
+
if success_flag and validated_indexes:
|
1491
1505
|
try:
|
1492
1506
|
self._update_indexes(db_name, table_name, validated_indexes)
|
1493
1507
|
except Exception as e:
|
mdbq-4.0.62/mdbq/__version__.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
VERSION = '4.0.62'
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|