mdbq 3.9.17__tar.gz → 3.9.18__tar.gz

{mdbq-3.9.17 → mdbq-3.9.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.17
+Version: 3.9.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.9.18/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '3.9.18'

{mdbq-3.9.17 → mdbq-3.9.18}/mdbq/mysql/uploader.py

@@ -1106,6 +1106,11 @@ class MySQLUploader:
         safe_columns = [self._validate_identifier(col) for col in all_columns]
         placeholders = ','.join(['%s'] * len(safe_columns))
 
+        # 初始化统计变量
+        total_inserted = 0
+        total_skipped = 0
+        total_failed = 0
+
         # 构建基础SQL语句
         if check_duplicate:
             if not duplicate_columns:
@@ -1116,10 +1121,7 @@ class MySQLUploader:
             conditions = []
             for col in duplicate_columns:
                 col_type = set_typ.get(col, '').lower()
-
-                # 处理DECIMAL类型，使用ROUND确保精度一致
                 if col_type.startswith('decimal'):
-                    # 提取小数位数，如DECIMAL(10,2)提取2
                     scale_match = re.search(r'decimal\(\d+,(\d+)\)', col_type)
                     scale = int(scale_match.group(1)) if scale_match else 2
                     conditions.append(f"ROUND(`{self._validate_identifier(col)}`, {scale}) = ROUND(%s, {scale})")
@@ -1137,10 +1139,6 @@ class MySQLUploader:
                     VALUES ({placeholders})
                     ON DUPLICATE KEY UPDATE {update_clause}
                     """
-
-                # 注意：在update_on_duplicate模式下，row_values只需要插入数据，不需要排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns]
             else:
                 sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
@@ -1151,10 +1149,6 @@ class MySQLUploader:
                         WHERE {where_clause}
                     )
                     """
-
-                # 在check_duplicate模式下，row_values需要插入数据+排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns] + [row.get(col) for col in duplicate_columns]
         else:
             sql = f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
@@ -1162,35 +1156,38 @@ class MySQLUploader:
                 VALUES ({placeholders})
                 """
 
-            # 普通模式下，row_values只需要插入数据
-            def prepare_values(row):
-                return [row.get(col) for col in all_columns]
-
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0  # 失败计数器
-
         # 分批插入数据
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 for i in range(0, len(data), batch_size):
-                    batch_start = time.time()
                     batch = data[i:i + batch_size]
-                    successful_rows = 0  # 当前批次成功数
+                    batch_inserted = 0
+                    batch_skipped = 0
+                    batch_failed = 0
 
                     for row in batch:
                         try:
                             # 准备参数
-                            row_values = prepare_values(row)
+                            row_values = [row.get(col) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                row_values += [row.get(col) for col in duplicate_columns]
+
                             cursor.execute(sql, row_values)
-                            successful_rows += 1
-                            conn.commit()  # 每次成功插入后提交
 
-                        except Exception as e:
-                            conn.rollback()  # 回滚当前行的事务
-                            total_failed += 1
+                            if check_duplicate:
+                                # 检查是否实际插入了行
+                                if cursor.rowcount > 0:
+                                    batch_inserted += 1
+                                else:
+                                    batch_skipped += 1
+                            else:
+                                batch_inserted += 1
 
-                            # 记录失败行详细信息
+                            conn.commit()
+
+                        except Exception as e:
+                            conn.rollback()
+                            batch_failed += 1
                             logger.error(sys._getframe().f_code.co_name, {
                                 '库': db_name,
                                 '表': table_name,
@@ -1201,29 +1198,22 @@ class MySQLUploader:
                                 '是否排重': check_duplicate,
                                 '排重列': duplicate_columns
                             })
-                            continue  # 跳过当前行，继续处理下一行
-
-                    # 更新统计信息
-                    if check_duplicate:
-                        cursor.execute("SELECT ROW_COUNT()")
-                        affected_rows = cursor.rowcount
-                        total_inserted += affected_rows
-                        total_skipped += len(batch) - affected_rows - (len(batch) - successful_rows)
-                    else:
-                        total_inserted += successful_rows
 
-                    batch_elapsed = round(time.time() - batch_start, 2)
+                    # 更新总统计
+                    total_inserted += batch_inserted
+                    total_skipped += batch_skipped
+                    total_failed += batch_failed
+
                     logger.debug(sys._getframe().f_code.co_name, {
                         '库': db_name,
                         '表': table_name,
                         '批次': batch_id,
                         '批次处理完成': i // batch_size + 1,
-                        'total_batches': (len(data) + batch_size - 1) // batch_size,
-                        'batch_size': len(batch),
-                        'successful_rows': successful_rows,
-                        'failed_rows': len(batch) - successful_rows,
-                        '耗时': batch_elapsed,
-                        'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
+                        '总批次': (len(data) + batch_size - 1) // batch_size,
+                        '数据量': len(batch),
+                        '插入': batch_inserted,
+                        '跳过': batch_skipped,
+                        '失败': batch_failed
                     })
 
         logger.info('插入完成', {

{mdbq-3.9.17 → mdbq-3.9.18}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.17
+Version: 3.9.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.9.17/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '3.9.17'