mdbq 3.10.0__tar.gz → 3.10.1__tar.gz

{mdbq-3.10.0 → mdbq-3.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.0
+Version: 3.10.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.10.1/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '3.10.1'

{mdbq-3.10.0 → mdbq-3.10.1}/mdbq/mysql/uploader.py

@@ -1125,7 +1125,6 @@ class MySQLUploader:
                 '自动使用默认模式': 'batch'
             })
             transaction_mode = 'batch'
-            # return
 
         # 获取所有列名（排除id列）
         all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
@@ -1191,50 +1190,103 @@ class MySQLUploader:
                     batch_skipped = 0
                     batch_failed = 0
 
-                    for row_idx, row in enumerate(batch, 1):
+                    if transaction_mode == 'batch':
+                        # 批量模式特殊处理 - 尝试逐行插入但保持事务
                         try:
-                            # 准备参数
-                            row_values = [row.get(col) for col in all_columns]
-                            if check_duplicate and not update_on_duplicate:
-                                row_values += [row.get(col) for col in duplicate_columns]
-
-                            cursor.execute(sql, row_values)
-
-                            if check_duplicate:
-                                # 检查是否实际插入了行
-                                if cursor.rowcount > 0:
-                                    batch_inserted += 1
-                                else:
-                                    batch_skipped += 1
-                            else:
-                                batch_inserted += 1
-
-                            # 根据模式决定提交时机
-                            if transaction_mode == 'row':
-                                conn.commit()  # 逐行提交
-                            elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
-                                conn.commit()  # 每100行提交一次
+                            for row_idx, row in enumerate(batch, 1):
+                                try:
+                                    # 准备参数
+                                    row_values = [row.get(col) for col in all_columns]
+                                    if check_duplicate and not update_on_duplicate:
+                                        row_values += [row.get(col) for col in duplicate_columns]
+
+                                    cursor.execute(sql, row_values)
+
+                                    if check_duplicate:
+                                        # 检查是否实际插入了行
+                                        if cursor.rowcount > 0:
+                                            batch_inserted += 1
+                                        else:
+                                            batch_skipped += 1
+                                    else:
+                                        batch_inserted += 1
+
+                                except Exception as e:
+                                    batch_failed += 1
+                                    logger.error(sys._getframe().f_code.co_name, {
+                                        '库': db_name,
+                                        '表': table_name,
+                                        '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
+                                        'error_type': type(e).__name__,
+                                        '单行插入失败': str(e),
+                                        '数据类型': set_typ,
+                                        '是否排重': check_duplicate,
+                                        '排重列': duplicate_columns,
+                                        '事务提交模式': transaction_mode,
+                                        '处理方式': '继续处理剩余行'
+                                    })
+                                    continue  # 继续处理下一行
+
+                            # 批量模式最后统一提交
+                            conn.commit()
 
                         except Exception as e:
-                            # if transaction_mode == 'row':
-                            #     conn.rollback()
+                            # 如果整个批量操作失败，回滚
                             conn.rollback()
-                            batch_failed += 1
+                            batch_failed = len(batch)  # 标记整个批次失败
                             logger.error(sys._getframe().f_code.co_name, {
                                 '库': db_name,
                                 '表': table_name,
-                                '批次': batch_id,
+                                '批次': f'{batch_id}  {i+1}/{len(data)}',
                                 'error_type': type(e).__name__,
-                                '单行插入失败': str(e),
-                                '数据类型': set_typ,
-                                '是否排重': check_duplicate,
-                                '排重列': duplicate_columns,
+                                '批量操作失败': str(e),
                                 '事务提交模式': transaction_mode,
+                                '处理方式': '整个批次回滚'
                             })
 
-                    # 批量模式最后统一提交
-                    if transaction_mode in ('batch', 'hybrid'):
-                        conn.commit()
+                    else:  # row 或 hybrid 模式
+                        for row_idx, row in enumerate(batch, 1):
+                            try:
+                                # 准备参数
+                                row_values = [row.get(col) for col in all_columns]
+                                if check_duplicate and not update_on_duplicate:
+                                    row_values += [row.get(col) for col in duplicate_columns]
+
+                                cursor.execute(sql, row_values)
+
+                                if check_duplicate:
+                                    # 检查是否实际插入了行
+                                    if cursor.rowcount > 0:
+                                        batch_inserted += 1
+                                    else:
+                                        batch_skipped += 1
+                                else:
+                                    batch_inserted += 1
+
+                                # 根据模式决定提交时机
+                                if transaction_mode == 'row':
+                                    conn.commit()  # 逐行提交
+                                elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
+                                    conn.commit()  # 每100行提交一次
+
+                            except Exception as e:
+                                conn.rollback()
+                                batch_failed += 1
+                                logger.error(sys._getframe().f_code.co_name, {
+                                    '库': db_name,
+                                    '表': table_name,
+                                    '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
+                                    'error_type': type(e).__name__,
+                                    '单行插入失败': str(e),
+                                    '数据类型': set_typ,
+                                    '是否排重': check_duplicate,
+                                    '排重列': duplicate_columns,
+                                    '事务提交模式': transaction_mode,
+                                })
+
+                        # 混合模式最后统一提交
+                        if transaction_mode == 'hybrid':
+                            conn.commit()
 
                     # 更新总统计
                     total_inserted += batch_inserted

{mdbq-3.10.0 → mdbq-3.10.1}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.0
+Version: 3.10.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.10.0/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '3.10.0'