mdbq 3.9.17__py3-none-any.whl → 3.9.18__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.17'
+VERSION = '3.9.18'

mdbq/mysql/uploader.py

@@ -1106,6 +1106,11 @@ class MySQLUploader:
         safe_columns = [self._validate_identifier(col) for col in all_columns]
         placeholders = ','.join(['%s'] * len(safe_columns))
 
+        # 初始化统计变量
+        total_inserted = 0
+        total_skipped = 0
+        total_failed = 0
+
         # 构建基础SQL语句
         if check_duplicate:
             if not duplicate_columns:
@@ -1116,10 +1121,7 @@ class MySQLUploader:
             conditions = []
             for col in duplicate_columns:
                 col_type = set_typ.get(col, '').lower()
-
-                # 处理DECIMAL类型，使用ROUND确保精度一致
                 if col_type.startswith('decimal'):
-                    # 提取小数位数，如DECIMAL(10,2)提取2
                     scale_match = re.search(r'decimal\(\d+,(\d+)\)', col_type)
                     scale = int(scale_match.group(1)) if scale_match else 2
                     conditions.append(f"ROUND(`{self._validate_identifier(col)}`, {scale}) = ROUND(%s, {scale})")
@@ -1137,10 +1139,6 @@ class MySQLUploader:
                     VALUES ({placeholders})
                     ON DUPLICATE KEY UPDATE {update_clause}
                     """
-
-                # 注意：在update_on_duplicate模式下，row_values只需要插入数据，不需要排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns]
             else:
                 sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
@@ -1151,10 +1149,6 @@ class MySQLUploader:
                         WHERE {where_clause}
                     )
                     """
-
-                # 在check_duplicate模式下，row_values需要插入数据+排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns] + [row.get(col) for col in duplicate_columns]
         else:
             sql = f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
@@ -1162,35 +1156,38 @@ class MySQLUploader:
                 VALUES ({placeholders})
                 """
 
-            # 普通模式下，row_values只需要插入数据
-            def prepare_values(row):
-                return [row.get(col) for col in all_columns]
-
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0  # 失败计数器
-
         # 分批插入数据
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 for i in range(0, len(data), batch_size):
-                    batch_start = time.time()
                     batch = data[i:i + batch_size]
-                    successful_rows = 0  # 当前批次成功数
+                    batch_inserted = 0
+                    batch_skipped = 0
+                    batch_failed = 0
 
                     for row in batch:
                         try:
                             # 准备参数
-                            row_values = prepare_values(row)
+                            row_values = [row.get(col) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                row_values += [row.get(col) for col in duplicate_columns]
+
                             cursor.execute(sql, row_values)
-                            successful_rows += 1
-                            conn.commit()  # 每次成功插入后提交
 
-                        except Exception as e:
-                            conn.rollback()  # 回滚当前行的事务
-                            total_failed += 1
+                            if check_duplicate:
+                                # 检查是否实际插入了行
+                                if cursor.rowcount > 0:
+                                    batch_inserted += 1
+                                else:
+                                    batch_skipped += 1
+                            else:
+                                batch_inserted += 1
 
-                            # 记录失败行详细信息
+                            conn.commit()
+
+                        except Exception as e:
+                            conn.rollback()
+                            batch_failed += 1
                             logger.error(sys._getframe().f_code.co_name, {
                                 '库': db_name,
                                 '表': table_name,
@@ -1201,29 +1198,22 @@ class MySQLUploader:
                                 '是否排重': check_duplicate,
                                 '排重列': duplicate_columns
                             })
-                            continue  # 跳过当前行，继续处理下一行
-
-                    # 更新统计信息
-                    if check_duplicate:
-                        cursor.execute("SELECT ROW_COUNT()")
-                        affected_rows = cursor.rowcount
-                        total_inserted += affected_rows
-                        total_skipped += len(batch) - affected_rows - (len(batch) - successful_rows)
-                    else:
-                        total_inserted += successful_rows
 
-                    batch_elapsed = round(time.time() - batch_start, 2)
+                    # 更新总统计
+                    total_inserted += batch_inserted
+                    total_skipped += batch_skipped
+                    total_failed += batch_failed
+
                     logger.debug(sys._getframe().f_code.co_name, {
                         '库': db_name,
                         '表': table_name,
                         '批次': batch_id,
                         '批次处理完成': i // batch_size + 1,
-                        'total_batches': (len(data) + batch_size - 1) // batch_size,
-                        'batch_size': len(batch),
-                        'successful_rows': successful_rows,
-                        'failed_rows': len(batch) - successful_rows,
-                        '耗时': batch_elapsed,
-                        'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
+                        '总批次': (len(data) + batch_size - 1) // batch_size,
+                        '数据量': len(batch),
+                        '插入': batch_inserted,
+                        '跳过': batch_skipped,
+                        '失败': batch_failed
                     })
 
         logger.info('插入完成', {

{mdbq-3.9.17.dist-info → mdbq-3.9.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.17
+Version: 3.9.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.17.dist-info → mdbq-3.9.18.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=i4IAUZBywP8zVvl8Pf78xv-EnW_gcBYlRkrSW52LW00,18
+mdbq/__version__.py,sha256=ZrDWol2b_rsMyMYX5Mye1GkN4Kl5Fbba7vG9AQ17vqo,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -12,7 +12,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=rQe_a58WRKNzhSEqj4naCnurZUucpWEo-ZpIRVAPf8Q,53624
+mdbq/mysql/uploader.py,sha256=OTUKnAxOLpd_SpXsSiJDTWQFuyU2hh5WweY_n3pMijc,52756
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.17.dist-info/METADATA,sha256=H3t8v9L5lRv3RoRaA71sOOYA8Nx4j0Mx0DiOd-xRYS4,364
-mdbq-3.9.17.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.17.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.17.dist-info/RECORD,,
+mdbq-3.9.18.dist-info/METADATA,sha256=kVj3t7BkNCh6hcmJ4qR5xcSD34FC6vb_ce-sxuZXCpY,364
+mdbq-3.9.18.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.18.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.18.dist-info/RECORD,,