mdbq 3.10.0__py3-none-any.whl → 3.10.2__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.0'
+VERSION = '3.10.2'

mdbq/mysql/uploader.py

@@ -1115,29 +1115,60 @@ class MySQLUploader:
         if not data:
             return
 
+        # 验证事务模式
+        transaction_mode = self._validate_transaction_mode(transaction_mode)
+
+        # 准备SQL语句
+        sql = self._prepare_insert_sql(
+            db_name, table_name, set_typ,
+            check_duplicate, duplicate_columns,
+            update_on_duplicate
+        )
+
+        # 执行批量插入
+        total_inserted, total_skipped, total_failed = self._execute_batch_insert(
+            db_name, table_name, data, set_typ,
+            sql, check_duplicate, duplicate_columns,
+            batch_size, batch_id, transaction_mode
+        )
+
+        logger.info('插入完成', {
+            '库': db_name,
+            '表': table_name,
+            '完成总计': len(data),
+            '插入': total_inserted,
+            '跳过': total_skipped,
+            '失败': total_failed,
+            '事务提交模式': transaction_mode,
+        })
+
+    def _validate_transaction_mode(self, mode: str) -> str:
+        """验证并标准化事务模式"""
         valid_modes = ('row', 'batch', 'hybrid')
-        if transaction_mode.lower() not in valid_modes:
+        if mode.lower() not in valid_modes:
             logger.error(sys._getframe().f_code.co_name, {
-                '库': db_name,
-                '表': table_name,
-                '参数异常': f'transaction_mode -> {transaction_mode}',
+                '参数异常': f'transaction_mode -> {mode}',
                 '可选值': valid_modes,
                 '自动使用默认模式': 'batch'
             })
-            transaction_mode = 'batch'
-            # return
+            return 'batch'
+        return mode.lower()
 
+    def _prepare_insert_sql(
+            self,
+            db_name: str,
+            table_name: str,
+            set_typ: Dict[str, str],
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            update_on_duplicate: bool
+    ) -> str:
+        """准备插入SQL语句"""
         # 获取所有列名（排除id列）
         all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         safe_columns = [self._validate_identifier(col) for col in all_columns]
         placeholders = ','.join(['%s'] * len(safe_columns))
 
-        # 初始化统计变量
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0
-
-        # 构建基础SQL语句
         if check_duplicate:
             if not duplicate_columns:
                 duplicate_columns = all_columns
@@ -1159,14 +1190,14 @@ class MySQLUploader:
             if update_on_duplicate:
                 # 更新模式 - 使用ON DUPLICATE KEY UPDATE语法
                 update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)" for col in all_columns])
-                sql = f"""
+                return f"""
                     INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
                     VALUES ({placeholders})
                     ON DUPLICATE KEY UPDATE {update_clause}
                     """
             else:
-                sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
+                return f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
                     SELECT {placeholders}
                     FROM DUAL
@@ -1176,94 +1207,190 @@ class MySQLUploader:
                     )
                     """
         else:
-            sql = f"""
+            return f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
                 (`{'`,`'.join(safe_columns)}`) 
                 VALUES ({placeholders})
                 """
 
-        # 分批插入数据
+    def _execute_batch_insert(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            set_typ: Dict[str, str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            batch_size: int,
+            batch_id: Optional[str],
+            transaction_mode: str
+    ) -> Tuple[int, int, int]:
+        """执行批量插入操作"""
+        # 获取所有列名（排除id列）
+        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+
+        total_inserted = 0
+        total_skipped = 0
+        total_failed = 0
+
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 for i in range(0, len(data), batch_size):
                     batch = data[i:i + batch_size]
-                    batch_inserted = 0
-                    batch_skipped = 0
-                    batch_failed = 0
-
-                    for row_idx, row in enumerate(batch, 1):
-                        try:
-                            # 准备参数
-                            row_values = [row.get(col) for col in all_columns]
-                            if check_duplicate and not update_on_duplicate:
-                                row_values += [row.get(col) for col in duplicate_columns]
-
-                            cursor.execute(sql, row_values)
-
-                            if check_duplicate:
-                                # 检查是否实际插入了行
-                                if cursor.rowcount > 0:
-                                    batch_inserted += 1
-                                else:
-                                    batch_skipped += 1
-                            else:
-                                batch_inserted += 1
-
-                            # 根据模式决定提交时机
-                            if transaction_mode == 'row':
-                                conn.commit()  # 逐行提交
-                            elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
-                                conn.commit()  # 每100行提交一次
-
-                        except Exception as e:
-                            # if transaction_mode == 'row':
-                            #     conn.rollback()
-                            conn.rollback()
-                            batch_failed += 1
-                            logger.error(sys._getframe().f_code.co_name, {
-                                '库': db_name,
-                                '表': table_name,
-                                '批次': batch_id,
-                                'error_type': type(e).__name__,
-                                '单行插入失败': str(e),
-                                '数据类型': set_typ,
-                                '是否排重': check_duplicate,
-                                '排重列': duplicate_columns,
-                                '事务提交模式': transaction_mode,
-                            })
-
-                    # 批量模式最后统一提交
-                    if transaction_mode in ('batch', 'hybrid'):
-                        conn.commit()
+                    batch_inserted, batch_skipped, batch_failed = self._process_batch(
+                        conn, cursor, db_name, table_name, batch, all_columns,
+                        sql, check_duplicate, duplicate_columns, batch_id,
+                        transaction_mode, i, len(data)
+                    )
 
                     # 更新总统计
                     total_inserted += batch_inserted
                     total_skipped += batch_skipped
                     total_failed += batch_failed
 
-                    logger.debug(sys._getframe().f_code.co_name, {
+        return total_inserted, total_skipped, total_failed
+
+    def _process_batch(
+            self,
+            conn,
+            cursor,
+            db_name: str,
+            table_name: str,
+            batch: List[Dict],
+            all_columns: List[str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            batch_id: Optional[str],
+            transaction_mode: str,
+            batch_index: int,
+            total_data_length: int
+    ) -> Tuple[int, int, int]:
+        """处理单个批次的数据插入"""
+        batch_inserted = 0
+        batch_skipped = 0
+        batch_failed = 0
+
+        if transaction_mode == 'batch':
+            # 批量模式特殊处理 - 尝试逐行插入但保持事务
+            try:
+                for row_idx, row in enumerate(batch, 1):
+                    result = self._process_single_row(
+                        cursor, row, all_columns, sql,
+                        check_duplicate, duplicate_columns
+                    )
+                    if result == 'inserted':
+                        batch_inserted += 1
+                    elif result == 'skipped':
+                        batch_skipped += 1
+                    else:
+                        batch_failed += 1
+
+                # 批量模式最后统一提交
+                conn.commit()
+
+            except Exception as e:
+                # 如果整个批量操作失败，回滚
+                conn.rollback()
+                batch_failed = len(batch)  # 标记整个批次失败
+                logger.error(sys._getframe().f_code.co_name, {
+                    '库': db_name,
+                    '表': table_name,
+                    '批次': f'{batch_id}  {batch_index + 1}/{total_data_length}',
+                    'error_type': type(e).__name__,
+                    '批量操作失败': str(e),
+                    '事务提交模式': transaction_mode,
+                    '处理方式': '整个批次回滚'
+                })
+
+        else:  # row 或 hybrid 模式
+            for row_idx, row in enumerate(batch, 1):
+                try:
+                    result = self._process_single_row(
+                        cursor, row, all_columns, sql,
+                        check_duplicate, duplicate_columns
+                    )
+                    if result == 'inserted':
+                        batch_inserted += 1
+                    elif result == 'skipped':
+                        batch_skipped += 1
+                    else:
+                        batch_failed += 1
+
+                    # 根据模式决定提交时机
+                    if transaction_mode == 'row':
+                        conn.commit()  # 逐行提交
+                    elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
+                        conn.commit()  # 每100行提交一次
+
+                except Exception as e:
+                    conn.rollback()
+                    batch_failed += 1
+                    logger.error(sys._getframe().f_code.co_name, {
                         '库': db_name,
                         '表': table_name,
-                        '批次': batch_id,
-                        '批次处理完成': i // batch_size + 1,
-                        '总批次': (len(data) + batch_size - 1) // batch_size,
-                        '数据量': len(batch),
-                        '插入': batch_inserted,
-                        '跳过': batch_skipped,
-                        '失败': batch_failed,
+                        '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
+                        'error_type': type(e).__name__,
+                        '单行插入失败': str(e),
+                        '数据类型': set_typ,
+                        '是否排重': check_duplicate,
+                        '排重列': duplicate_columns,
                         '事务提交模式': transaction_mode,
                     })
 
-        logger.info('插入完成', {
+            # 混合模式最后统一提交
+            if transaction_mode == 'hybrid':
+                conn.commit()
+
+        logger.debug(sys._getframe().f_code.co_name, {
             '库': db_name,
             '表': table_name,
-            '完成总计': len(data),
-            '插入': total_inserted,
-            '跳过': total_skipped,
-            '失败': total_failed,
+            '批次': batch_id,
+            '批次处理完成': batch_index // len(batch) + 1,
+            '总批次': (total_data_length + len(batch) - 1) // len(batch),
+            '数据量': len(batch),
+            '插入': batch_inserted,
+            '跳过': batch_skipped,
+            '失败': batch_failed,
             '事务提交模式': transaction_mode,
         })
 
+        return batch_inserted, batch_skipped, batch_failed
+
+    def _process_single_row(
+            self,
+            cursor,
+            row: Dict,
+            all_columns: List[str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]]
+    ) -> str:
+        """处理单行数据插入"""
+        try:
+            # 准备参数
+            row_values = [row.get(col) for col in all_columns]
+            if check_duplicate:
+                row_values += [row.get(col) for col in duplicate_columns]
+
+            cursor.execute(sql, row_values)
+
+            if check_duplicate:
+                # 检查是否实际插入了行
+                return 'inserted' if cursor.rowcount > 0 else 'skipped'
+            return 'inserted'
+
+        except Exception as e:
+            logger.error(sys._getframe().f_code.co_name, {
+                'error_type': type(e).__name__,
+                '单行插入失败': str(e),
+                '是否排重': check_duplicate,
+                '排重列': duplicate_columns,
+                '处理方式': '继续处理剩余行'
+            })
+            return 'failed'
+
     def close(self):
         """
         关闭连接池并清理资源
@@ -1333,6 +1460,7 @@ class MySQLUploader:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+
 def main():
     uploader = MySQLUploader(
         username='root',

{mdbq-3.10.0.dist-info → mdbq-3.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.0
+Version: 3.10.2
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.10.0.dist-info → mdbq-3.10.2.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=5KePxgTqsEgArDghwVLPBSZN4Cwjr4f6goTdfGdj5cE,18
+mdbq/__version__.py,sha256=tL5iFQ6j9Svg-3tbUuEZAgDFN3ipIhdJjFUPU6EHSRQ,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -12,7 +12,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=oK9LD5ydDRk5FvyKzBxQTXmTgnSMlpn31gq1Ht9uJCE,54884
+mdbq/mysql/uploader.py,sha256=XOSeGg74zN3qYFfWmLqr98H7tCj74dIMCS3C0cvS3kU,58994
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.10.0.dist-info/METADATA,sha256=I6UtECQJCo-8lp2QiudMEWPG_EHctYjesMdcDPUb2wg,364
-mdbq-3.10.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.10.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.10.0.dist-info/RECORD,,
+mdbq-3.10.2.dist-info/METADATA,sha256=D9d_UixDPHEbrdRE1yjA4SHjo4tYoY60_R4cBGPF3ms,364
+mdbq-3.10.2.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.10.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.10.2.dist-info/RECORD,,