mdbq 4.2.2__py3-none-any.whl → 4.2.4__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.2.2'
+VERSION = '4.2.4'

mdbq/mysql/deduplicator.py

@@ -39,7 +39,7 @@ class MySQLDeduplicator:
 
     主要参数说明：
     - columns: 指定去重分组字段，控制唯一性分组行为。若 columns 有值且不包含 date_column，则全表去重，否则按天分区。
-    - exclude_columns: 去重时排除的列名列表，自动合并 ['id', '更新时间']，在分组时排除。
+    - exclude_columns: 去重时排除的列名列表，自动合并 ['id', 'update_at']，在分组时排除。
     - date_column: 指定日期分区字段，默认为 '日期'。如表存在该字段且 columns 未排除，则按天分区去重。
     - duplicate_keep_mode: 'keep_one'（默认，重复组保留一条），'remove_all'（全部删除重复组）。
     - dry_run: 是否为模拟运行，不实际删除数据。
@@ -81,7 +81,7 @@ class MySQLDeduplicator:
         :param date_range: 指定去重的日期区间 [start_date, end_date]，格式'YYYY-MM-DD'
         :param recent_month: 最近N个月的数据去重（与date_range互斥，优先生效）
         :param date_column: 时间列名，默认为'日期'
-        :param exclude_columns: 去重时排除的列名列表，默认为['id', '更新时间']
+        :param exclude_columns: 去重时排除的列名列表，默认为['id', 'update_at']
         :param exclude_databases: 排除的数据库名列表
         :param exclude_tables: 排除的表名字典 {数据库名: [表名, ...]}
         :param duplicate_keep_mode: 'keep_one'（默认，重复组保留一条），'remove_all'（全部删除重复组）
@@ -171,7 +171,7 @@ class MySQLDeduplicator:
             logger.debug('去重日期范围', {'开始': self._dedup_start_date, '结束': self._dedup_end_date})
 
         # 排除列处理，直接合并去重
-        self.exclude_columns = list(set((exclude_columns or []) + ['id', '更新时间']))
+        self.exclude_columns = list(set((exclude_columns or []) + ['id', 'update_at']))
 
         # 线程安全控制
         self._lock = threading.Lock()
@@ -450,11 +450,11 @@ class MySQLDeduplicator:
 
                 # 用Python查找重复
                 if use_python_dedup:
-                    # 判断分组字段是否有“更新时间”
-                    has_update_time = any(col == '更新时间' for col in use_columns)
+                    # 判断分组字段是否有“update_at”
+                    has_update_time = any(col == 'update_at' for col in use_columns)
                     select_cols = f'`{pk_real}`,' + ','.join([f'`{col}`' for col in use_columns])
                     if has_update_time:
-                        select_cols += ',`更新时间`'
+                        select_cols += ',`update_at`'
                     select_where = f"WHERE `{time_col}` = '{date_val}'" if date_val else ''
                     grouped = defaultdict(list)
                     for row in self._row_generator(database, table, select_cols, select_where, self.batch_size):
@@ -466,8 +466,7 @@ class MySQLDeduplicator:
                         if len(ids) > 1:
                             dup_count += 1
                             if has_update_time:
-                                # 按更新时间最大保留
-                                keep_row = max(ids, key=lambda x: x.get('更新时间') or '')
+                                keep_row = max(ids, key=lambda x: x.get('update_at') or '')
                             else:
                                 # 按id保留
                                 if self.keep_order == 'max':
@@ -490,10 +489,10 @@ class MySQLDeduplicator:
                 temp_table = self._make_temp_table_name(table)
                 drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
                 create_temp_where = f"WHERE `{time_col}` = '{date_val}'"
-                # 判断分组字段是否有“更新时间”
-                has_update_time = any(col == '更新时间' for col in use_columns)
+ 
+                has_update_time = any(col == 'update_at' for col in use_columns)
                 if has_update_time:
-                    keep_field = '更新时间'
+                    keep_field = 'update_at'
                     keep_func = 'MAX'
                 else:
                     keep_field = pk_real
@@ -1369,7 +1368,7 @@ def main():
         maxcached=5,
         # recent_month=1,
         # date_range=['2025-06-09', '2025-06-10'],
-        exclude_columns=['创建时间', '更新时间'],
+        exclude_columns=['创建时间', '更新时间', "update_at", "create_at"],
         exclude_databases=['cookie文件', '日志', '视频数据', '云电影'],
         # exclude_tables={
         #     '推广数据2': [

mdbq/mysql/mysql.py

@@ -146,6 +146,8 @@ class MysqlUpload:
                 __res_dict.update({k: 'DATE'})
             elif k == '更新时间':
                 __res_dict.update({k: 'TIMESTAMP'})
+            elif k == 'update_at' or k == 'create_at':
+                __res_dict.update({k: 'TIMESTAMP'})
             elif result2:
                 __res_dict.update({k: 'decimal(10,4)'})
             elif date_type == 1:
@@ -550,6 +552,8 @@ class MysqlUpload:
                 __res_dict.update({k: 'DATE'})
             elif k == '更新时间':
                 __res_dict.update({k: 'TIMESTAMP'})
+            elif k == 'update_at' or k == 'create_at':
+                __res_dict.update({k: 'TIMESTAMP'})
             elif result2:  # 小数
                 __res_dict.update({k: 'decimal(10,4)'})
             elif date_type == 1:  # 纯日期
@@ -605,6 +609,8 @@ class MysqlUpload:
                 __res_dict.update({k: 'date'})
             elif k == '更新时间':
                 __res_dict.update({k: 'timestamp'})
+            elif k == 'update_at' or k == 'create_at':
+                __res_dict.update({k: 'timestamp'})
             elif v == 'int64':
                 __res_dict.update({k: 'int'})
             elif v == 'float64':
@@ -936,10 +942,10 @@ class OptimizeDatas:
             self.db_name = db_name
             self.optimize()
 
-    def optimize(self, except_key: list[str] = ['更新时间']) -> None:
+    def optimize(self, except_key: list[str] = ['update_at']) -> None:
         """
         优化当前数据库，移除冗余数据。
-        :param except_key: 排除的字段名列表，默认['更新时间']
+        :param except_key: 排除的字段名列表，默认['update_at']
         """
         if not self.db_name:
             logger.info(f'尚未设置参数: self.db_name')
@@ -995,7 +1001,7 @@ class OptimizeDatas:
                 self.connection.close()
         logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
 
-    def delete_duplicate(self, table_name: str, date: pd.Timestamp, except_key: list[str] = ['更新时间']) -> None:
+    def delete_duplicate(self, table_name: str, date: pd.Timestamp, except_key: list[str] = ['update_at']) -> None:
         """
         删除指定表指定日期的冗余数据。
         :param table_name: 表名
@@ -1029,7 +1035,7 @@ class OptimizeDatas:
             logger.error(f'{self.db_name}/{table_name}, {e}')
             self.connection.rollback()
 
-    def delete_duplicate2(self, table_name: str, except_key: list[str] = ['更新时间']) -> None:
+    def delete_duplicate2(self, table_name: str, except_key: list[str] = ['update_at']) -> None:
         """
         删除指定表（无日期列）的冗余数据。
         :param table_name: 表名

mdbq/mysql/s_query.py

@@ -485,7 +485,7 @@ class QueryDatas:
             logger.error('获取列类型失败', {'库': db_name, '表': table_name, '列': columns_name, '错误': str(e)})
             return []
 
-    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = '更新时间') -> Optional[List[Dict[str, Any]]]:
+    def check_condition(self, db_name: str, table_name: str, condition: str, columns: str = 'update_at') -> Optional[List[Dict[str, Any]]]:
         """按指定条件查询数据库表"""
         if not self._get_table_info(db_name, table_name):
             return None

mdbq/mysql/uploader.py

@@ -13,7 +13,6 @@ from decimal import Decimal, InvalidOperation
 import math
 import concurrent.futures
 import threading
-from queue import Queue
 import pymysql
 import pandas as pd
 import psutil
@@ -668,7 +667,8 @@ class TableManager:
     
     def create_table(self, db_name: str, table_name: str, columns: Dict[str, str], 
                     primary_keys: Optional[List[str]] = None, 
-                    unique_keys: Optional[List[List[str]]] = None):
+                    unique_keys: Optional[List[List[str]]] = None,
+                    allow_null: bool = False):
         """创建表"""
         db_name = self._sanitize_identifier(db_name)
         table_name = self._sanitize_identifier(table_name)
@@ -684,7 +684,8 @@ class TableManager:
             if col_name.lower() in ['id', 'create_at', 'update_at']:
                 continue
             safe_col_name = self._sanitize_identifier(col_name)
-            column_defs.append(f"`{safe_col_name}` {col_type} NOT NULL")
+            null_constraint = "" if allow_null else " NOT NULL"
+            column_defs.append(f"`{safe_col_name}` {col_type}{null_constraint}")
         
         # 添加时间戳列
         column_defs.append("`create_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP")
@@ -785,6 +786,10 @@ class DataProcessor:
             chunk_size = min(chunk_size, 1000)  # 内存紧张时减小chunk
         
         if isinstance(data, pd.DataFrame):
+            # 统一将DataFrame的列名转为小写
+            data = data.copy()
+            data.columns = [col.lower() for col in data.columns]
+            
             # 对于大DataFrame，使用更高效的分块方式
             if len(data) > 50000:
                 # 大数据集使用pandas的分块读取
@@ -795,11 +800,23 @@ class DataProcessor:
                     chunk = data.iloc[i:i + chunk_size]
                     yield chunk.to_dict('records')
         elif isinstance(data, dict):
-            yield [data]
+            # 统一将字典的键转为小写
+            normalized_dict = {}
+            for key, value in data.items():
+                normalized_dict[key.lower()] = value
+            yield [normalized_dict]
         elif isinstance(data, list):
             if all(isinstance(item, dict) for item in data):
-                for i in range(0, len(data), chunk_size):
-                    yield data[i:i + chunk_size]
+                # 统一将列表中字典的键转为小写
+                normalized_data = []
+                for item in data:
+                    normalized_item = {}
+                    for key, value in item.items():
+                        normalized_item[key.lower()] = value
+                    normalized_data.append(normalized_item)
+                
+                for i in range(0, len(normalized_data), chunk_size):
+                    yield normalized_data[i:i + chunk_size]
             else:
                 raise ValueError("列表中必须全部是字典")
         else:
@@ -938,9 +955,21 @@ class DataInserter:
                             try:
                                 cursor.executemany(sql, batch_values)
                                 total_inserted += len(batch_values)
-                            except pymysql.err.IntegrityError:
-                                total_skipped += len(batch_values)
-                                logger.debug('批量插入唯一约束冲突，跳过', {'批次大小': len(batch_values)})
+                            except pymysql.err.IntegrityError as e:
+                                # 批量插入遇到唯一约束冲突，fallback到逐行插入
+                                logger.debug('批量插入唯一约束冲突，尝试逐行插入', {'批次大小': len(batch_values)})
+                                
+                                # 逐行插入处理冲突
+                                for single_value in batch_values:
+                                    try:
+                                        cursor.execute(sql, single_value)
+                                        total_inserted += 1
+                                    except pymysql.err.IntegrityError:
+                                        total_skipped += 1
+                                        logger.debug('单行插入唯一约束冲突，跳过')
+                                    except Exception as single_e:
+                                        total_failed += 1
+                                        logger.error('单行插入失败', {'错误': str(single_e)})
                             except Exception as e:
                                 logger.error('批量插入失败', {'错误': str(e), '批次大小': len(batch_values)})
                                 raise
@@ -1077,12 +1106,6 @@ class MySQLUploader:
         }
         
         try:
-            # 计算原始数据大小
-            original_data_size = 0
-            if isinstance(data, (pd.DataFrame, list)):
-                original_data_size = len(data)
-            elif isinstance(data, dict):
-                original_data_size = 1
             
             # 标准化数据为流式迭代器
             normalized_data = DataProcessor.normalize_data(data)
@@ -1096,6 +1119,9 @@ class MySQLUploader:
                 normalized_data = DataProcessor.normalize_data(data)
                 logger.debug('自动推断数据类型', {'类型映射': set_typ})
             
+            # 将set_typ的键统一转为小写
+            set_typ = self.tran_set_typ_to_lower(set_typ)
+            
             # 确保数据库存在
             self.table_mgr.ensure_database_exists(db_name)
             
@@ -1142,7 +1168,7 @@ class MySQLUploader:
         # 确保表存在
         if not self.table_mgr.table_exists(db_name, table_name):
             self.table_mgr.create_table(db_name, table_name, set_typ, 
-                                       unique_keys=unique_keys)
+                                       unique_keys=unique_keys, allow_null=allow_null)
             result['tables_created'].append(f"{db_name}.{table_name}")
         else:
             # 表已存在，确保有时间戳列（但保持原有主键结构）
@@ -1275,7 +1301,7 @@ class MySQLUploader:
         if table_key not in created_tables:
             if not self.table_mgr.table_exists(db_name, partition_table_name):
                 self.table_mgr.create_table(db_name, partition_table_name, set_typ, 
-                                           unique_keys=unique_keys)
+                                           unique_keys=unique_keys, allow_null=allow_null)
                 result['tables_created'].append(table_key)
             else:
                 # 表已存在，确保有时间戳列（但保持原有主键结构）
@@ -1358,7 +1384,10 @@ class MySQLUploader:
                 set_typ = DataTypeInferrer.infer_types_from_data(first_chunk)
                 normalized_data = DataProcessor.normalize_data(data, chunk_size=2000)
                 logger.debug('自动推断数据类型', {'类型映射': set_typ})
-            
+
+            # 将set_typ的键统一转为小写
+            set_typ = self.tran_set_typ_to_lower(set_typ)
+
             # 确保数据库存在
             self.table_mgr.ensure_database_exists(db_name)
             
@@ -1391,7 +1420,7 @@ class MySQLUploader:
                                 if table_key not in created_tables_set:
                                     if not self.table_mgr.table_exists(db_name, partition_table_name):
                                         self.table_mgr.create_table(db_name, partition_table_name, set_typ, 
-                                                                   unique_keys=unique_keys)
+                                                                   unique_keys=unique_keys, allow_null=allow_null)
                                         chunk_result['tables_created'].append(table_key)
                                     else:
                                         self.table_mgr.ensure_system_columns(db_name, partition_table_name)
@@ -1416,7 +1445,7 @@ class MySQLUploader:
                             if table_key not in created_tables_set:
                                 if not self.table_mgr.table_exists(db_name, table_name):
                                     self.table_mgr.create_table(db_name, table_name, set_typ, 
-                                                               unique_keys=unique_keys)
+                                                               unique_keys=unique_keys, allow_null=allow_null)
                                     chunk_result = {'tables_created': [table_key]}
                                 else:
                                     self.table_mgr.ensure_system_columns(db_name, table_name)
@@ -1480,6 +1509,16 @@ class MySQLUploader:
             result['success'] = False
         
         return result
+    
+    def tran_set_typ_to_lower(self, set_typ: Dict[str, str]) -> Dict[str, str]:
+        if not isinstance(set_typ, dict):
+            return set_typ
+
+        set_typ_lower = {}
+        for key, value in set_typ.items():
+            set_typ_lower[key.lower()] = value
+            
+        return set_typ_lower
 
 
 # 使用示例

mdbq/other/download_sku_picture.py

@@ -477,7 +477,6 @@ class SkuPicture:
                         '推荐卖点': desc,
                         '获取与下载': '已获取',
                         '类目': leimu,
-                        '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     }
                 )
             except Exception as e:

{mdbq-4.2.2.dist-info → mdbq-4.2.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: mdbq
-Version: 4.2.2
+Version: 4.2.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.2.2.dist-info → mdbq-4.2.4.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=gNDA6f7PmXcbqB0lTY4HIgD6dEB6SGywjhHa3HAyczA,17
+mdbq/__version__.py,sha256=aDBBA8s4yZmwb4L5nfeuXIxGeD7k5CCdzxpb3hxH4Og,17
 mdbq/auth/__init__.py,sha256=pnPMAt63sh1B6kEvmutUuro46zVf2v2YDAG7q-jV_To,24
 mdbq/auth/auth_backend.py,sha256=iLN7AqiSq7fQgFtNtge_TIlVOR1hrCSZXH6oId6uGX4,116924
 mdbq/auth/crypto.py,sha256=fcZRFCnrKVVdWDUx_zds51ynFYwS9DBvJOrRQVldrfM,15931
@@ -11,13 +11,13 @@ mdbq/log/mylogger.py,sha256=DyBftCMNLe1pTTXsa830pUtDISJxpJHFIradYtE3lFA,26418
 mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/myconf/myconf.py,sha256=x_9mS6wOfKVjCVElbruxj2yjzitbyKiTkf59quG-5Zg,32529
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=tzLIm9K9S0lGLlVTI0dDQVYpWX796XCuyufmw1lU26Y,73098
-mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=N2xHJf2CiUXjXIVBemdst-wamIP3908EGAJOFG13fCU,50475
+mdbq/mysql/deduplicator.py,sha256=VGRBcIEsWUqQovkGpGCOittSW1l5fuMpiNWLbiOA5tI,72973
+mdbq/mysql/mysql.py,sha256=eSrEE6DdS3GJ_EPPSggEZXhnVz6IqHPwUSsWfZJVFkY,57081
+mdbq/mysql/s_query.py,sha256=rCJZ0XPalwk3Z7HkqsvF9o0UX0sZT4Wk1JXryRrV6yQ,50472
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=2inrXu3PIlvowfm5_0U4Trx_mraApjII8g_5ycFbNJ0,60059
+mdbq/mysql/uploader.py,sha256=zi7Y2VQTykChKl6qvQ2fY_0T2SjVBJP8Vqm6vRI3sao,62088
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=MJX47I9jTUMFzO1kyEH-onIzAGa6QpgfmghrmyYnEsc,45111
+mdbq/other/download_sku_picture.py,sha256=pxcoNYoTP5KAhyvyHO2NQIrfhige5UN-LuB9Z8Iw3g8,45017
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
@@ -35,7 +35,7 @@ mdbq/route/routes.py,sha256=QVGfTvDgu0CpcKCvk1ra74H8uojgqTLUav1fnVAqLEA,29433
 mdbq/selenium/__init__.py,sha256=AKzeEceqZyvqn2dEDoJSzDQnbuENkJSHAlbHAD0u0ZI,10
 mdbq/selenium/get_driver.py,sha256=1NTlVUE6QsyjTrVVVqTO2LOnYf578ccFWlWnvIXGtic,20903
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.2.2.dist-info/METADATA,sha256=vfhvk7DXQ267-NOPdqKJ_AWCWSEbWKdDjIf7bilbCXo,363
-mdbq-4.2.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.2.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.2.2.dist-info/RECORD,,
+mdbq-4.2.4.dist-info/METADATA,sha256=wGQTV-yfALM95nxMQDpRY2r-khPAK5ogMHXlrH0sB8c,363
+mdbq-4.2.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mdbq-4.2.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.2.4.dist-info/RECORD,,

{mdbq-4.2.2.dist-info → mdbq-4.2.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any