mdbq 3.12.0__py3-none-any.whl → 3.12.2__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.12.0'
+VERSION = '3.12.2'

mdbq/mysql/deduplicator.py

@@ -1294,32 +1294,32 @@ def main():
         password='pwd',
         host='localhost',
         port=3306,
-        max_workers= 2,
+        max_workers= 3,
         batch_size=1000,
         skip_system_dbs=True,
         max_retries=3,
         retry_waiting_time=5,
-        # pool_size=30,
+        pool_size=30,
         recent_month=1,
         # date_range=['2025-06-09', '2025-06-10'],
         date_column='日期',
-        exclude_databases=['测试库4'],
-        exclude_tables={
-            '推广数据2': [
-                '地域报表_城市_2025_04', 
-                # '地域报表_城市_2025_04_copy1', 
-            ],
-            "生意参谋3": [
-                "商品排行_2025",
-            ],
-        },
+        # exclude_databases=['测试库4'],
+        # exclude_tables={
+        #     '推广数据2': [
+        #         '地域报表_城市_2025_04', 
+        #         # '地域报表_城市_2025_04_copy1', 
+        #     ],
+        #     "生意参谋3": [
+        #         "商品排行_2025",
+        #     ],
+        # },
     )
 
     # 全库去重(单线程)
     deduplicator.deduplicate_all(dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定数据库去重(多线程)
-    # deduplicator.deduplicate_database('my_db', dry_run=False, parallel=True, reorder_id=True)
+    # deduplicator.deduplicate_database('数据引擎2', dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定表去重(使用特定列)
     # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'data'], dry_run=False, reorder_id=True)
@@ -1331,5 +1331,5 @@ def main():
     deduplicator.close()
 
 if __name__ == '__main__':
-    # main()
+    main()
     pass

mdbq/mysql/uploader.py

@@ -23,8 +23,8 @@ logger = mylogger.MyLogger(
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=1,  # 采样50%的DEBUG/INFO日志
-    sensitive_fields=[],  #  敏感字段列表
+    sample_rate=1,  # 采样DEBUG/INFO日志, 0.5表示50%的日志会被采样
+    sensitive_fields=[],  #  过滤敏感字段列表
 )
 
 
@@ -83,7 +83,7 @@ class MySQLUploader:
             charset: str = 'utf8mb4',
             collation: str = 'utf8mb4_0900_ai_ci',
             max_retries: int = 10,
-            retry_interval: int = 10,
+            retry_waiting_time: int = 10,
             pool_size: int = 5,
             connect_timeout: int = 10,
             read_timeout: int = 30,
@@ -100,7 +100,7 @@ class MySQLUploader:
         :param charset: 字符集，默认为utf8mb4
         :param collation: 排序规则，默认为utf8mb4_0900_ai_ci，对大小写不敏感，utf8mb4_0900_as_cs/utf8mb4_bin: 对大小写敏感
         :param max_retries: 最大重试次数，默认为10
-        :param retry_interval: 重试间隔(秒)，默认为10
+        :param retry_waiting_time: 重试间隔(秒)，默认为10
         :param pool_size: 连接池大小，默认为5
         :param connect_timeout: 连接超时(秒)，默认为10
         :param read_timeout: 读取超时(秒)，默认为30
@@ -114,7 +114,7 @@ class MySQLUploader:
         self.charset = charset
         self.collation = collation
         self.max_retries = max(max_retries, 1)
-        self.retry_interval = max(retry_interval, 1)
+        self.retry_waiting_time = max(retry_waiting_time, 1)
         self.pool_size = max(pool_size, 1)
         self.connect_timeout = connect_timeout
         self.read_timeout = read_timeout
@@ -169,7 +169,7 @@ class MySQLUploader:
             }
         try:
             pool = PooledDB(**pool_params)
-            logger.info('连接池创建成功', {'连接池': self.pool_size, 'host': self.host, 'port': self.port})
+            logger.debug('连接池创建成功', {'连接池': self.pool_size, 'host': self.host, 'port': self.port})
             return pool
         except Exception as e:
             self.pool = None
@@ -188,14 +188,11 @@ class MySQLUploader:
         def wrapper(self, *args, **kwargs):
             last_exception = None
             operation = func.__name__
-            logger.debug(f'开始执行操作: {operation}', {'max_retries': self.max_retries})
             for attempt in range(self.max_retries):
                 try:
                     result = func(self, *args, **kwargs)
                     if attempt > 0:
                         logger.info('操作成功(重试后)', {'operation': operation, 'attempts': attempt + 1})
-                    else:
-                        logger.debug('操作成功', {'operation': operation})
                     return result
                 except (pymysql.OperationalError, pymysql.err.MySQLError) as e:
                     last_exception = e
@@ -207,7 +204,7 @@ class MySQLUploader:
                         'max_retries': self.max_retries
                     }
                     if attempt < self.max_retries - 1:
-                        wait_time = self.retry_interval * (attempt + 1)
+                        wait_time = self.retry_waiting_time * (attempt + 1)
                         error_details['wait_time'] = wait_time
                         logger.warning('数据库操作失败，准备重试', error_details)
                         time.sleep(wait_time)
@@ -218,13 +215,6 @@ class MySQLUploader:
                             logger.error('重连失败', {'error': str(reconnect_error)})
                     else:
                         logger.error('操作最终失败', error_details)
-                except pymysql.IntegrityError as e:
-                    logger.error('完整性约束错误', {
-                        'operation': operation,
-                        'error_code': e.args[0] if e.args else None,
-                        'error_message': e.args[1] if len(e.args) > 1 else None
-                    })
-                    raise e
                 except Exception as e:
                     last_exception = e
                     logger.error('发生意外错误', {
@@ -247,10 +237,9 @@ class MySQLUploader:
         """
         try:
             conn = self.pool.connection()
-            logger.debug('获取数据库连接', {'host': self.host, 'port': self.port})
             return conn
         except Exception as e:
-            logger.error('获取数据库连接失败', {'error': str(e)})
+            logger.error('从连接池获取数据库连接失败', {'error': str(e)})
             raise ConnectionError(f'连接数据库失败: {str(e)}')
 
     @_execute_with_retry
@@ -392,7 +381,8 @@ class MySQLUploader:
             primary_keys: Optional[List[str]] = None,
             date_column: Optional[str] = None,
             indexes: Optional[List[str]] = None,
-            allow_null: bool = False
+            allow_null: bool = False,
+            unique_keys: Optional[List[List[str]]] = None
     ) -> None:
         """
         创建数据表，优化索引创建方式
@@ -402,39 +392,48 @@ class MySQLUploader:
         if not set_typ:
             logger.error('建表时未指定set_typ', {'库': db_name, '表': table_name})
             raise ValueError('set_typ 未指定')
+        # set_typ的键清洗
+        set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
         column_defs = ["`id` INT NOT NULL AUTO_INCREMENT"]
         for col_name, col_type in set_typ.items():
-            if col_name.lower() == 'id':
+            if col_name == 'id':
                 continue
-            safe_col_name = self._validate_identifier(col_name)
+            safe_col_name = self._normalize_col(col_name)
             col_def = f"`{safe_col_name}` {col_type}"
             if not allow_null and not col_type.lower().startswith('json'):
                 col_def += " NOT NULL"
             column_defs.append(col_def)
-        if primary_keys:
-            if 'id' not in [pk.lower() for pk in primary_keys]:
-                primary_keys = ['id'] + primary_keys
+        # 主键处理逻辑调整
+        if primary_keys and len(primary_keys) > 0:
+            safe_primary_keys = [self._normalize_col(pk) for pk in primary_keys]
+            primary_key_sql = f"PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
         else:
-            primary_keys = ['id']
-        safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
-        primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
+            safe_primary_keys = [self._normalize_col('id')]
+            primary_key_sql = f"PRIMARY KEY (`id`)"
         # 索引统一在CREATE TABLE中定义
         index_defs = []
         if date_column and date_column in set_typ:
-            safe_date_col = self._validate_identifier(date_column)
+            safe_date_col = self._normalize_col(date_column)
             index_defs.append(f"INDEX `idx_{safe_date_col}` (`{safe_date_col}`)")
         if indexes:
             for idx_col in indexes:
                 if idx_col in set_typ:
-                    safe_idx_col = self._validate_identifier(idx_col)
+                    safe_idx_col = self._normalize_col(idx_col)
                     index_defs.append(f"INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)")
+        # UNIQUE KEY定义
+        unique_defs = []
+        if unique_keys:
+            for idx, unique_cols in enumerate(unique_keys):
+                if not unique_cols:
+                    continue
+                safe_unique_cols = [self._normalize_col(col) for col in unique_cols]
+                unique_name = f"uniq_{'_'.join(safe_unique_cols)}_{idx}"
+                unique_defs.append(f"UNIQUE KEY `{unique_name}` (`{'`,`'.join(safe_unique_cols)}`)")
         index_defs = list(set(index_defs))
-        index_sql = (',' + ','.join(index_defs)) if index_defs else ''
+        all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
         sql = f"""
         CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
-            {','.join(column_defs)}
-            {primary_key_sql}
-            {index_sql}
+            {','.join(all_defs)}
         ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
         """
         conn = None
@@ -443,7 +442,7 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
                 conn.commit()
-                logger.info('数据表及索引已创建', {'库': db_name, '表': table_name, '索引': indexes})
+                logger.info('数据表及索引已创建', {'库': db_name, '表': table_name, '索引': indexes, '唯一约束': unique_keys})
         except Exception as e:
             logger.error('建表失败', {'库': db_name, '表': table_name, '错误': str(e)})
             if conn is not None:
@@ -476,11 +475,9 @@ class MySQLUploader:
             try:
                 if date_type:
                     result = pd.to_datetime(datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d'))
-                    logger.debug('日期格式化成功', {'原始': value, '格式': fmt, '结果': str(result)})
                     return result
                 else:
                     result = datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
-                    logger.debug('日期格式化成功', {'原始': value, '格式': fmt, '结果': str(result)})
                     return result
             except ValueError:
                 continue
@@ -613,7 +610,7 @@ class MySQLUploader:
                     cursor.execute(sql_check, (db_name, table_name, column))
                     exists = cursor.fetchone()
                     if exists and list(exists.values())[0] > 0:
-                        logger.debug('索引已存在', {'库': db_name, '表': table_name, '列': column})
+                        logger.debug('索引检查', {'库': db_name, '表': table_name, '索引列': column})
                         return
                     cursor.execute(sql_create)
                 conn.commit()
@@ -622,6 +619,49 @@ class MySQLUploader:
             logger.error('创建索引失败', {'库': db_name, '表': table_name, '列': column, '错误': str(e)})
             raise
 
+    def _get_existing_unique_keys(self, db_name: str, table_name: str) -> List[List[str]]:
+        """
+        获取表中所有UNIQUE KEY的列组合（不含主键）。
+        返回：[[col1, col2], ...]
+        """
+        db_name = self._validate_identifier(db_name)
+        table_name = self._validate_identifier(table_name)
+        sql = '''
+            SELECT INDEX_NAME, COLUMN_NAME
+            FROM INFORMATION_SCHEMA.STATISTICS
+            WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND NON_UNIQUE = 0 AND INDEX_NAME != 'PRIMARY'
+            ORDER BY INDEX_NAME, SEQ_IN_INDEX
+        '''
+        unique_map = {}
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql, (db_name, table_name))
+                    for row in cursor.fetchall():
+                        idx = row['INDEX_NAME']
+                        col = row['COLUMN_NAME']
+                        unique_map.setdefault(idx, []).append(col)
+        except Exception as e:
+            logger.warning('获取UNIQUE KEY信息失败', {'库': db_name, '表': table_name, '错误': str(e)})
+        # 只返回列名组合，全部清洗小写
+        return [[self._normalize_col(c) for c in cols] for cols in unique_map.values() if cols]
+
+    def _add_unique_key(self, db_name: str, table_name: str, unique_cols: List[str]):
+        """
+        添加UNIQUE KEY
+        """
+        safe_cols = [self._normalize_col(col) for col in unique_cols]
+        unique_name = f"uniq_{'_'.join(safe_cols)}_{int(time.time()*1000)%100000}"
+        sql = f'ALTER TABLE `{db_name}`.`{table_name}` ADD UNIQUE KEY `{unique_name}` ({','.join(f'`{col}`' for col in safe_cols)})'
+        try:
+            with self._get_connection() as conn:
+                with conn.cursor() as cursor:
+                    cursor.execute(sql)
+                conn.commit()
+            logger.info('添加唯一约束列成功', {'库': db_name, '表': table_name, '列': unique_cols})
+        except Exception as e:
+            logger.warning('唯一约束列添加失败', {'库': db_name, '表': table_name, '列': unique_cols, '错误': str(e)})
+
     def _upload_to_table(
             self,
             db_name: str,
@@ -637,14 +677,15 @@ class MySQLUploader:
             indexes: Optional[List[str]],
             batch_id: Optional[str] = None,
             update_on_duplicate: bool = False,
-            transaction_mode: str = "batch"
+            transaction_mode: str = "batch",
+            unique_keys: Optional[List[List[str]]] = None
     ):
         """实际执行表上传的方法"""
-        # 检查表是否存在
-        if not self._check_table_exists(db_name, table_name):
+        table_existed = self._check_table_exists(db_name, table_name)
+        if not table_existed:
             if auto_create:
                 self._create_table(db_name, table_name, set_typ, primary_keys, date_column, indexes,
-                                   allow_null=allow_null)
+                                   allow_null=allow_null, unique_keys=unique_keys)
             else:
                 logger.error('数据表不存在', {
                     '库': db_name,
@@ -652,8 +693,30 @@ class MySQLUploader:
                     'func': sys._getframe().f_code.co_name,
                 })
                 raise ValueError(f"数据表不存在: `{db_name}`.`{table_name}`")
-
-        # 获取表结构并验证
+        if table_existed and unique_keys:
+            try:
+                exist_ukeys = self._get_existing_unique_keys(db_name, table_name)
+                exist_ukeys_norm = [sorted([c.lower() for c in uk]) for uk in exist_ukeys]
+                filtered_ukeys = [uk for uk in unique_keys if 1 <= len(uk) <= 20]
+                to_add = []
+                for uk in filtered_ukeys:
+                    norm_uk = sorted([c.lower() for c in uk])
+                    if norm_uk not in exist_ukeys_norm:
+                        to_add.append(uk)
+                max_unique_keys = 10
+                if len(exist_ukeys) + len(to_add) > max_unique_keys:
+                    logger.warning('unique_keys超限', {
+                        '库': db_name,
+                        '表': table_name,
+                        '已存在': exist_ukeys,
+                        '本次待添加': to_add,
+                        '最大数量': max_unique_keys
+                    })
+                    to_add = to_add[:max_unique_keys - len(exist_ukeys)]
+                for uk in to_add:
+                    self._add_unique_key(db_name, table_name, uk)
+            except Exception as e:
+                logger.warning('动态unique key处理异常', {'库': db_name, '表': table_name, '错误': str(e)})
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
             logger.error('获取列失败', {
@@ -663,8 +726,6 @@ class MySQLUploader:
                 'func': sys._getframe().f_code.co_name,
             })
             raise ValueError(f"获取列失败 `{db_name}`.`{table_name}`")
-
-        # 验证数据列与表列匹配
         for col in set_typ:
             if col not in table_columns:
                 logger.error('列不存在', {
@@ -674,22 +735,19 @@ class MySQLUploader:
                     'func': sys._getframe().f_code.co_name,
                 })
                 raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
-
-        # 确保分表参考字段为索引
         if date_column and date_column in table_columns:
             try:
                 self._ensure_index(db_name, table_name, date_column)
             except Exception as e:
                 logger.warning('分表参考字段索引创建失败', {'库': db_name, '表': table_name, '列': date_column, '错误': str(e)})
-
-        # 插入数据
-        self._insert_data(
+        inserted, skipped, failed = self._insert_data(
             db_name, table_name, data, set_typ,
             check_duplicate, duplicate_columns,
             batch_id=batch_id,
             update_on_duplicate=update_on_duplicate,
             transaction_mode=transaction_mode
         )
+        return inserted, skipped, failed
 
     def _infer_data_type(self, value: Any, no_log: bool = False) -> str:
         """
@@ -817,11 +875,8 @@ class MySQLUploader:
         # 统一处理原始数据中列名的特殊字符
         data = self.normalize_column_names(data)
 
-        # set_typ的键处理
-        if self.case_sensitive:
-            set_typ = {k: v for k, v in set_typ.items()}
-        else:
-            set_typ = {k.lower(): v for k, v in set_typ.items()}
+        # set_typ的键清洗
+        set_typ = {self._normalize_col(k): v for k, v in set_typ.items()}
 
         # 获取数据中实际存在的列名
         data_columns = set()
@@ -890,7 +945,8 @@ class MySQLUploader:
             auto_create: bool = True,
             indexes: Optional[List[str]] = None,
             update_on_duplicate: bool = False,
-            transaction_mode: str = "batch"
+            transaction_mode: str = "batch",
+            unique_keys: Optional[List[List[str]]] = None
     ):
         """
         上传数据到数据库的主入口方法，分表逻辑异常处理统计丢弃数据
@@ -912,6 +968,7 @@ class MySQLUploader:
             - 'row'     : 逐行提交事务（错误隔离性好）
             - 'batch'   : 整批提交事务（性能最优）
             - 'hybrid'  : 混合模式（每N行提交，平衡性能与安全性）
+        :param unique_keys: 唯一约束列表，每个元素为列名列表，支持多列组合唯一约束
         :raises: 可能抛出各种验证和数据库相关异常
         """
         # upload_start = time.time()
@@ -936,7 +993,8 @@ class MySQLUploader:
                 # '自动建表': auto_create,
                 '索引': indexes,
                 '更新旧数据': update_on_duplicate,
-                '事务模式': transaction_mode
+                '事务模式': transaction_mode,
+                '唯一约束': unique_keys
             },
             # '数据样例': self._shorten_for_log(data, 2)
         })
@@ -1005,15 +1063,21 @@ class MySQLUploader:
                         continue
 
                 # 对每个分表执行上传
+                total_inserted = 0
+                total_skipped = dropped_rows  # 分表异常丢弃
+                total_failed = 0
                 for part_table, part_data in partitioned_data.items():
                     try:
-                        self._upload_to_table(
+                        inserted, skipped, failed = self._upload_to_table(
                             db_name, part_table, part_data, filtered_set_typ,
                             primary_keys, check_duplicate, duplicate_columns,
                             allow_null, auto_create, partition_date_column,
-                            indexes, batch_id, update_on_duplicate, transaction_mode
+                            indexes, batch_id, update_on_duplicate, transaction_mode,
+                            unique_keys
                         )
-                        # 确保分表参考字段为索引
+                        total_inserted += inserted
+                        total_skipped += skipped
+                        total_failed += failed
                         if partition_date_column in filtered_set_typ:
                             try:
                                 self._ensure_index(db_name, part_table, partition_date_column)
@@ -1031,13 +1095,16 @@ class MySQLUploader:
                         continue  # 跳过当前分表，继续处理其他分表
             else:
                 # 不分表，直接上传
-                self._upload_to_table(
+                inserted, skipped, failed = self._upload_to_table(
                     db_name, table_name, prepared_data, filtered_set_typ,
                     primary_keys, check_duplicate, duplicate_columns,
                     allow_null, auto_create, partition_date_column,
-                    indexes, batch_id, update_on_duplicate, transaction_mode
+                    indexes, batch_id, update_on_duplicate, transaction_mode,
+                    unique_keys
                 )
-                # 确保分表参考字段为索引
+                total_inserted = inserted
+                total_skipped = skipped
+                total_failed = failed
                 if partition_date_column in filtered_set_typ:
                     try:
                         self._ensure_index(db_name, table_name, partition_date_column)
@@ -1062,7 +1129,9 @@ class MySQLUploader:
                 '批次': batch_id,
                 'finish': success_flag,
                 '数据行': initial_row_count,
-                '丢弃行数': dropped_rows
+                '插入': total_inserted,
+                '跳过': total_skipped,
+                '失败': total_failed
             })
 
     @_execute_with_retry
@@ -1095,26 +1164,19 @@ class MySQLUploader:
             - 'hybrid'  : 混合模式（每N行提交，平衡性能与安全性）
         """
         if not data:
-            return
-
-        # 验证事务模式
+            return 0, 0, 0
         transaction_mode = self._validate_transaction_mode(transaction_mode)
-
-        # 准备SQL语句
         sql = self._prepare_insert_sql(
             db_name, table_name, set_typ,
             check_duplicate, duplicate_columns,
             update_on_duplicate
         )
-
-        # 执行批量插入
         total_inserted, total_skipped, total_failed = self._execute_batch_insert(
             db_name, table_name, data, set_typ,
             sql, check_duplicate, duplicate_columns,
             batch_id, transaction_mode,
             update_on_duplicate
         )
-
         logger.info('插入完成', {
             '库': db_name,
             '表': table_name,
@@ -1124,6 +1186,7 @@ class MySQLUploader:
             '失败': total_failed,
             '事务模式': transaction_mode,
         })
+        return total_inserted, total_skipped, total_failed
 
     def _validate_transaction_mode(self, mode: str) -> str:
         """验证并标准化事务模式"""
@@ -1266,6 +1329,7 @@ class MySQLUploader:
             update_on_duplicate: bool = False
     ) -> Tuple[int, int, int]:
         """执行批量插入操作，优化batch和hybrid模式"""
+        import pymysql  # 确保异常类型可用
         def get_optimal_batch_size(total_rows: int) -> int:
             if total_rows <= 100:
                 return total_rows
@@ -1295,7 +1359,13 @@ class MySQLUploader:
                         try:
                             cursor.executemany(sql, values_list)
                             conn.commit()
-                            total_inserted += len(batch)
+                            inserted = cursor.rowcount if cursor.rowcount is not None else 0
+                            total_inserted += inserted
+                            total_skipped += len(batch) - inserted
+                        except pymysql.err.IntegrityError as e:
+                            conn.rollback()
+                            total_skipped += len(batch)
+                            logger.debug('批量插入唯一约束冲突，全部跳过', {'库': db_name, '表': table_name, '错误': str(e)})
                         except Exception as e:
                             conn.rollback()
                             total_failed += len(batch)
@@ -1311,7 +1381,15 @@ class MySQLUploader:
                                     dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
                                     values += [row.get(col) for col in dup_cols]
                                 cursor.execute(sql, values)
-                                total_inserted += 1
+                                affected = cursor.rowcount if cursor.rowcount is not None else 0
+                                if affected > 0:
+                                    total_inserted += 1
+                                else:
+                                    total_skipped += 1
+                            except pymysql.err.IntegrityError as e:
+                                conn.rollback()
+                                total_skipped += 1
+                                logger.debug('hybrid单行插入唯一约束冲突，跳过', {'库': db_name, '表': table_name, '错误': str(e)})
                             except Exception as e:
                                 conn.rollback()
                                 total_failed += 1
@@ -1325,8 +1403,16 @@ class MySQLUploader:
                                 dup_cols = duplicate_columns if duplicate_columns else [col for col in all_columns if col.lower() not in self.base_excute_col]
                                 values += [row.get(col) for col in dup_cols]
                             cursor.execute(sql, values)
+                            affected = cursor.rowcount if cursor.rowcount is not None else 0
+                            if affected > 0:
+                                total_inserted += 1
+                            else:
+                                total_skipped += 1
                             conn.commit()
-                            total_inserted += 1
+                        except pymysql.err.IntegrityError as e:
+                            conn.rollback()
+                            total_skipped += 1
+                            logger.debug('单行插入唯一约束冲突，跳过', {'库': db_name, '表': table_name, '错误': str(e)})
                         except Exception as e:
                             conn.rollback()
                             total_failed += 1
@@ -1347,9 +1433,9 @@ class MySQLUploader:
                     self.pool = None
                 except Exception as e:
                     logger.warning('关闭连接池时出错', {'error': str(e)})
-                logger.info('连接池关闭', {'uploader.py': '连接池关闭'})
+                logger.debug('finished', {'uploader.py': '连接池关闭'})
         except Exception as e:
-            logger.error('关闭连接池失败', {'error': str(e)})
+            logger.error('关闭连接池失败', {'uploader.py': str(e)})
             raise
 
     def _check_pool_health(self) -> bool:
@@ -1431,6 +1517,13 @@ class MySQLUploader:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+    def _normalize_col(self, col: str) -> str:
+        """
+        列名自动清洗并转小写（如case_sensitive为False），保证和表结构一致。
+        """
+        safe = self._validate_identifier(col)
+        return safe if self.case_sensitive else safe.lower()
+
 
 def main():
     """
@@ -1443,7 +1536,7 @@ def main():
     """
     uploader = MySQLUploader(
         username='root',
-        password='pw',
+        password='pwd',
         host='localhost',
         port=3306,
     )
@@ -1462,7 +1555,7 @@ def main():
         {'日期': '2023-01-8', 'name': 'JACk', 'AGE': '24', 'salary': 555.1545},
         {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 35, 'salary': '100'},
         {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 30, 'salary': 0.0},
-        {'日期': '2023-02-20', 'name': 'Bob', 'AGE': 25, 'salary': 45000.75}
+        {'日期': '2023-02-20', 'name': 'Bob', 'AGE': 25, 'salary': 45000.75},
     ]
 
     # 上传数据
@@ -1474,12 +1567,13 @@ def main():
         primary_keys=[],  # 创建唯一主键
         check_duplicate=False,  # 检查重复数据
         duplicate_columns=[],  # 指定排重的组合键
+        update_on_duplicate=False,  # 更新旧数据
         allow_null=False,  # 允许插入空值
-        partition_by='year',  # 按月分表
+        partition_by='year',  # 分表方式
         partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
-        auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
-        indexes=[],  # 指定索引列
+        indexes=[],  # 普通索引列
         transaction_mode='row',  # 事务模式
+        unique_keys=[['日期', 'name', 'age']],  # 唯一约束列表
     )
 
     uploader.close()

{mdbq-3.12.0.dist-info → mdbq-3.12.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.12.0
+Version: 3.12.2
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.12.0.dist-info → mdbq-3.12.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=W8WVhYkHLU0SBDlL9Q6XQVTqIrzYjc1kFBZgqzS_NEI,18
+mdbq/__version__.py,sha256=7f_XGixBIsGAP5w1W3y_kTPfLVlurDoRsv3FOOJLDIA,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=nxL8hSy8yI1QLlqnkTNHHQSxRfo-6WKL5OA-N4xLB7c,179832
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -8,10 +8,10 @@ mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=Crw6LwVo3I3IUbzIETu8f46Quza3CTCh-qYf4edbBPo,24139
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=KMJ_YyqAniaLVRqOHLgO92PgwknIDB-EgaOY7S6iMZ4,68599
+mdbq/mysql/deduplicator.py,sha256=eILGFxFtMCSR6dvCgEgsOjwlK_hEWBe2dFSgJTPHfj8,68623
 mdbq/mysql/mysql.py,sha256=Kjpi-LL00WQUmTTOfhEBsNrmo4-4kFFJzrHbVKfqiBE,56770
 mdbq/mysql/s_query.py,sha256=dlnrVJ3-Vp1Suv9CNbPxyYSRqRJUHjOpF39tb2F-wBc,10190
-mdbq/mysql/uploader.py,sha256=8Px_W2bYOr1wQgMXMK0DggNiuE6a6Ul4BlJake8LSo8,64469
+mdbq/mysql/uploader.py,sha256=szX6t4SObBF6fbHT2s5ixfh1-c288cigsJ66pFE02Qg,70266
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -24,7 +24,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=YHgCKO8mEsslwet33K5tGss-nrDDwPnOSlhA9iBu0jY,24078
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=cqK-JRd_DHbToC7hyo83m8o97NZkJFqmB2xBtr6aAVU,20961
-mdbq-3.12.0.dist-info/METADATA,sha256=Q6EyaC61H4okFva6YFV2a0Y3Iqun8L8mnpSkeVXcFdc,364
-mdbq-3.12.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.12.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.12.0.dist-info/RECORD,,
+mdbq-3.12.2.dist-info/METADATA,sha256=1V071qDvuX5apscDGFLY_ICX6mMVBUhjzMGh_a_wd7w,364
+mdbq-3.12.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.12.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.12.2.dist-info/RECORD,,