mdbq 3.10.1__py3-none-any.whl → 3.10.3__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.10.1'
+VERSION = '3.10.3'

mdbq/mysql/uploader.py

@@ -303,8 +303,8 @@ class MySQLUploader:
                     return exists
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
+                '库': db_name,
                 '检查数据库是否存在时出错': str(e),
-                '库': db_name
             })
             raise
 
@@ -546,9 +546,9 @@ class MySQLUploader:
 
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '建表失败': str(e),
                 '库': db_name,
                 '表': table_name,
+                '建表失败': str(e),
             })
             conn.rollback()
             raise
@@ -678,9 +678,9 @@ class MySQLUploader:
                     return set_typ
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '无法获取表列信息': str(e),
                 '库': db_name,
                 '表': table_name,
+                '无法获取表列信息': str(e),
             })
             raise
 
@@ -709,9 +709,9 @@ class MySQLUploader:
                                    allow_null=allow_null)
             else:
                 logger.error(sys._getframe().f_code.co_name, {
-                    '数据表不存在': table_name,
                     '库': db_name,
                     '表': table_name,
+                    '数据表不存在': table_name,
                 })
                 raise ValueError(f"数据表不存在: `{db_name}`.`{table_name}`")
 
@@ -719,9 +719,9 @@ class MySQLUploader:
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
             logger.error(sys._getframe().f_code.co_name, {
-                '获取列失败': table_columns,
                 '库': db_name,
                 '表': table_name,
+                '获取列失败': table_columns,
             })
             raise ValueError(f"获取列失败 `{db_name}`.`{table_name}`")
 
@@ -729,9 +729,9 @@ class MySQLUploader:
         for col in set_typ:
             if col not in table_columns:
                 logger.error(sys._getframe().f_code.co_name, {
-                    '列不存在': f'{col} -> {table_columns}',
                     '库': db_name,
                     '表': table_name,
+                    '列不存在': f'{col} -> {table_columns}',
                 })
                 raise ValueError(f"列不存在: `{col}` -> `{db_name}`.`{table_name}`")
 
@@ -960,7 +960,7 @@ class MySQLUploader:
             默认值为 'batch'
         :raises: 可能抛出各种验证和数据库相关异常
         """
-        upload_start = time.time()
+        # upload_start = time.time()
         initial_row_count = len(data) if hasattr(data, '__len__') else 1
 
         batch_id = f"batch_{int(time.time() * 1000)}"
@@ -972,7 +972,7 @@ class MySQLUploader:
             '批次': batch_id,
             '分表方式': partition_by,
             '排重': check_duplicate,
-            '传入总计': len(data) if hasattr(data, '__len__') else 1,
+            '传入': len(data) if hasattr(data, '__len__') else 1,
             '自动建表': auto_create
         })
 
@@ -985,10 +985,10 @@ class MySQLUploader:
                 partition_by = str(partition_by).lower()
                 if partition_by not in ['year', 'month']:
                     logger.error(sys._getframe().f_code.co_name, {
-                        '分表方式必须是 "year" 或 "month" 或 "None"':  partition_by,
                         '库': db_name,
                         '表': table_name,
-                        '批次': batch_id
+                        '批次': batch_id,
+                        '分表方式必须是 "year" 或 "month" 或 "None"': partition_by,
                     })
                     raise ValueError("分表方式必须是 'year' 或 'month' 或 'None'")
 
@@ -1029,10 +1029,10 @@ class MySQLUploader:
                         partitioned_data[part_table].append(row)
                     except Exception as e:
                         logger.error(sys._getframe().f_code.co_name, {
-                            'row_data': row,
-                            '分表处理失败': str(e),
                             '库': db_name,
                             '表': table_name,
+                            'row_data': row,
+                            '分表处理失败': str(e),
                         })
                         continue  # 跳过当前行
 
@@ -1047,10 +1047,10 @@ class MySQLUploader:
                         )
                     except Exception as e:
                         logger.error(sys._getframe().f_code.co_name, {
-                            '分表': part_table,
-                            '分表上传失败': str(e),
                             '库': db_name,
                             '表': table_name,
+                            '分表': part_table,
+                            '分表上传失败': str(e),
                         })
                         continue  # 跳过当前分表，继续处理其他分表
             else:
@@ -1066,10 +1066,10 @@ class MySQLUploader:
 
         except Exception as e:
             logger.error(sys._getframe().f_code.co_name, {
-                '上传过程发生全局错误': str(e),
-                'error_type': type(e).__name__,
                 '库': db_name,
                 '表': table_name,
+                '上传过程发生全局错误': str(e),
+                'error_type': type(e).__name__,
             })
         finally:
             logger.info("存储完成", {
@@ -1077,7 +1077,7 @@ class MySQLUploader:
                 '表': table_name,
                 '批次': batch_id,
                 'success': success_flag,
-                '耗时': round(time.time() - upload_start, 2),
+                # '耗时': round(time.time() - upload_start, 2),
                 '数据行': initial_row_count
             })
 
@@ -1115,33 +1115,65 @@ class MySQLUploader:
         if not data:
             return
 
+        # 验证事务模式
+        transaction_mode = self._validate_transaction_mode(transaction_mode)
+
+        # 准备SQL语句
+        sql = self._prepare_insert_sql(
+            db_name, table_name, set_typ,
+            check_duplicate, duplicate_columns,
+            update_on_duplicate
+        )
+
+        # 执行批量插入
+        total_inserted, total_skipped, total_failed = self._execute_batch_insert(
+            db_name, table_name, data, set_typ,
+            sql, check_duplicate, duplicate_columns,
+            batch_size, batch_id, transaction_mode
+        )
+
+        logger.info('插入完成', {
+            '库': db_name,
+            '表': table_name,
+            '完成总计': len(data),
+            '插入': total_inserted,
+            '跳过': total_skipped,
+            '失败': total_failed,
+            '事务提交模式': transaction_mode,
+        })
+
+    def _validate_transaction_mode(self, mode: str) -> str:
+        """验证并标准化事务模式"""
         valid_modes = ('row', 'batch', 'hybrid')
-        if transaction_mode.lower() not in valid_modes:
+        if mode.lower() not in valid_modes:
             logger.error(sys._getframe().f_code.co_name, {
-                '库': db_name,
-                '表': table_name,
-                '参数异常': f'transaction_mode -> {transaction_mode}',
+                '参数异常': f'transaction_mode -> {mode}',
                 '可选值': valid_modes,
                 '自动使用默认模式': 'batch'
             })
-            transaction_mode = 'batch'
+            return 'batch'
+        return mode.lower()
 
-        # 获取所有列名（排除id列）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+    def _prepare_insert_sql(
+            self,
+            db_name: str,
+            table_name: str,
+            set_typ: Dict[str, str],
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            update_on_duplicate: bool
+    ) -> str:
+        """准备插入SQL语句"""
+        # 获取所有列名（排除 `id`、`更新时间` 列）
+        all_columns = [col for col in set_typ.keys() if col.lower() not in ['id', '更新时间']]
         safe_columns = [self._validate_identifier(col) for col in all_columns]
         placeholders = ','.join(['%s'] * len(safe_columns))
 
-        # 初始化统计变量
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0
-
-        # 构建基础SQL语句
         if check_duplicate:
             if not duplicate_columns:
                 duplicate_columns = all_columns
             else:
-                duplicate_columns = [col for col in duplicate_columns if col != 'id']
+                duplicate_columns = [col for col in duplicate_columns if col.lower() not in ['id', '更新时间']]
 
             conditions = []
             for col in duplicate_columns:
@@ -1158,14 +1190,14 @@ class MySQLUploader:
             if update_on_duplicate:
                 # 更新模式 - 使用ON DUPLICATE KEY UPDATE语法
                 update_clause = ", ".join([f"`{col}` = VALUES(`{col}`)" for col in all_columns])
-                sql = f"""
+                return f"""
                     INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
                     VALUES ({placeholders})
                     ON DUPLICATE KEY UPDATE {update_clause}
                     """
             else:
-                sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
+                return f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
                     SELECT {placeholders}
                     FROM DUAL
@@ -1175,147 +1207,190 @@ class MySQLUploader:
                     )
                     """
         else:
-            sql = f"""
+            return f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
                 (`{'`,`'.join(safe_columns)}`) 
                 VALUES ({placeholders})
                 """
 
-        # 分批插入数据
+    def _execute_batch_insert(
+            self,
+            db_name: str,
+            table_name: str,
+            data: List[Dict],
+            set_typ: Dict[str, str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            batch_size: int,
+            batch_id: Optional[str],
+            transaction_mode: str
+    ) -> Tuple[int, int, int]:
+        """执行批量插入操作"""
+        # 获取所有列名（排除id列）
+        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
+
+        total_inserted = 0
+        total_skipped = 0
+        total_failed = 0
+
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 for i in range(0, len(data), batch_size):
                     batch = data[i:i + batch_size]
-                    batch_inserted = 0
-                    batch_skipped = 0
-                    batch_failed = 0
-
-                    if transaction_mode == 'batch':
-                        # 批量模式特殊处理 - 尝试逐行插入但保持事务
-                        try:
-                            for row_idx, row in enumerate(batch, 1):
-                                try:
-                                    # 准备参数
-                                    row_values = [row.get(col) for col in all_columns]
-                                    if check_duplicate and not update_on_duplicate:
-                                        row_values += [row.get(col) for col in duplicate_columns]
-
-                                    cursor.execute(sql, row_values)
-
-                                    if check_duplicate:
-                                        # 检查是否实际插入了行
-                                        if cursor.rowcount > 0:
-                                            batch_inserted += 1
-                                        else:
-                                            batch_skipped += 1
-                                    else:
-                                        batch_inserted += 1
-
-                                except Exception as e:
-                                    batch_failed += 1
-                                    logger.error(sys._getframe().f_code.co_name, {
-                                        '库': db_name,
-                                        '表': table_name,
-                                        '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
-                                        'error_type': type(e).__name__,
-                                        '单行插入失败': str(e),
-                                        '数据类型': set_typ,
-                                        '是否排重': check_duplicate,
-                                        '排重列': duplicate_columns,
-                                        '事务提交模式': transaction_mode,
-                                        '处理方式': '继续处理剩余行'
-                                    })
-                                    continue  # 继续处理下一行
-
-                            # 批量模式最后统一提交
-                            conn.commit()
-
-                        except Exception as e:
-                            # 如果整个批量操作失败，回滚
-                            conn.rollback()
-                            batch_failed = len(batch)  # 标记整个批次失败
-                            logger.error(sys._getframe().f_code.co_name, {
-                                '库': db_name,
-                                '表': table_name,
-                                '批次': f'{batch_id}  {i+1}/{len(data)}',
-                                'error_type': type(e).__name__,
-                                '批量操作失败': str(e),
-                                '事务提交模式': transaction_mode,
-                                '处理方式': '整个批次回滚'
-                            })
-
-                    else:  # row 或 hybrid 模式
-                        for row_idx, row in enumerate(batch, 1):
-                            try:
-                                # 准备参数
-                                row_values = [row.get(col) for col in all_columns]
-                                if check_duplicate and not update_on_duplicate:
-                                    row_values += [row.get(col) for col in duplicate_columns]
-
-                                cursor.execute(sql, row_values)
-
-                                if check_duplicate:
-                                    # 检查是否实际插入了行
-                                    if cursor.rowcount > 0:
-                                        batch_inserted += 1
-                                    else:
-                                        batch_skipped += 1
-                                else:
-                                    batch_inserted += 1
-
-                                # 根据模式决定提交时机
-                                if transaction_mode == 'row':
-                                    conn.commit()  # 逐行提交
-                                elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
-                                    conn.commit()  # 每100行提交一次
-
-                            except Exception as e:
-                                conn.rollback()
-                                batch_failed += 1
-                                logger.error(sys._getframe().f_code.co_name, {
-                                    '库': db_name,
-                                    '表': table_name,
-                                    '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
-                                    'error_type': type(e).__name__,
-                                    '单行插入失败': str(e),
-                                    '数据类型': set_typ,
-                                    '是否排重': check_duplicate,
-                                    '排重列': duplicate_columns,
-                                    '事务提交模式': transaction_mode,
-                                })
-
-                        # 混合模式最后统一提交
-                        if transaction_mode == 'hybrid':
-                            conn.commit()
+                    batch_inserted, batch_skipped, batch_failed = self._process_batch(
+                        conn, cursor, db_name, table_name, batch, all_columns,
+                        sql, check_duplicate, duplicate_columns, batch_id,
+                        transaction_mode, i, len(data)
+                    )
 
                     # 更新总统计
                     total_inserted += batch_inserted
                     total_skipped += batch_skipped
                     total_failed += batch_failed
 
-                    logger.debug(sys._getframe().f_code.co_name, {
+        return total_inserted, total_skipped, total_failed
+
+    def _process_batch(
+            self,
+            conn,
+            cursor,
+            db_name: str,
+            table_name: str,
+            batch: List[Dict],
+            all_columns: List[str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]],
+            batch_id: Optional[str],
+            transaction_mode: str,
+            batch_index: int,
+            total_data_length: int
+    ) -> Tuple[int, int, int]:
+        """处理单个批次的数据插入"""
+        batch_inserted = 0
+        batch_skipped = 0
+        batch_failed = 0
+
+        if transaction_mode == 'batch':
+            # 批量模式特殊处理 - 尝试逐行插入但保持事务
+            try:
+                for row_idx, row in enumerate(batch, 1):
+                    result = self._process_single_row(
+                        cursor, row, all_columns, sql,
+                        check_duplicate, duplicate_columns
+                    )
+                    if result == 'inserted':
+                        batch_inserted += 1
+                    elif result == 'skipped':
+                        batch_skipped += 1
+                    else:
+                        batch_failed += 1
+
+                # 批量模式最后统一提交
+                conn.commit()
+
+            except Exception as e:
+                # 如果整个批量操作失败，回滚
+                conn.rollback()
+                batch_failed = len(batch)  # 标记整个批次失败
+                logger.error(sys._getframe().f_code.co_name, {
+                    '库': db_name,
+                    '表': table_name,
+                    '批次': f'{batch_id}  {batch_index + 1}/{total_data_length}',
+                    'error_type': type(e).__name__,
+                    '批量操作失败': str(e),
+                    '事务提交模式': transaction_mode,
+                    '处理方式': '整个批次回滚'
+                })
+
+        else:  # row 或 hybrid 模式
+            for row_idx, row in enumerate(batch, 1):
+                try:
+                    result = self._process_single_row(
+                        cursor, row, all_columns, sql,
+                        check_duplicate, duplicate_columns
+                    )
+                    if result == 'inserted':
+                        batch_inserted += 1
+                    elif result == 'skipped':
+                        batch_skipped += 1
+                    else:
+                        batch_failed += 1
+
+                    # 根据模式决定提交时机
+                    if transaction_mode == 'row':
+                        conn.commit()  # 逐行提交
+                    elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
+                        conn.commit()  # 每100行提交一次
+
+                except Exception as e:
+                    conn.rollback()
+                    batch_failed += 1
+                    logger.error(sys._getframe().f_code.co_name, {
                         '库': db_name,
                         '表': table_name,
-                        '批次': batch_id,
-                        '批次处理完成': i // batch_size + 1,
-                        '总批次': (len(data) + batch_size - 1) // batch_size,
-                        '数据量': len(batch),
-                        '插入': batch_inserted,
-                        '跳过': batch_skipped,
-                        '失败': batch_failed,
+                        '批次/当前行': f'{batch_id}  {row_idx}/{len(batch)}',
+                        'error_type': type(e).__name__,
+                        '单行插入失败': str(e),
+                        '数据类型': set_typ,
+                        '是否排重': check_duplicate,
+                        '排重列': duplicate_columns,
                         '事务提交模式': transaction_mode,
                     })
 
-        logger.info('插入完成', {
+            # 混合模式最后统一提交
+            if transaction_mode == 'hybrid':
+                conn.commit()
+
+        logger.debug(sys._getframe().f_code.co_name, {
             '库': db_name,
             '表': table_name,
-            '完成总计': len(data),
-            '插入': total_inserted,
-            '跳过': total_skipped,
-            '失败': total_failed,
+            '批次': batch_id,
+            '批次处理完成': batch_index // len(batch) + 1,
+            '总批次': (total_data_length + len(batch) - 1) // len(batch),
+            '数据量': len(batch),
+            '插入': batch_inserted,
+            '跳过': batch_skipped,
+            '失败': batch_failed,
             '事务提交模式': transaction_mode,
         })
 
+        return batch_inserted, batch_skipped, batch_failed
+
+    def _process_single_row(
+            self,
+            cursor,
+            row: Dict,
+            all_columns: List[str],
+            sql: str,
+            check_duplicate: bool,
+            duplicate_columns: Optional[List[str]]
+    ) -> str:
+        """处理单行数据插入"""
+        try:
+            # 准备参数
+            row_values = [row.get(col) for col in all_columns]
+            if check_duplicate:
+                row_values += [row.get(col) for col in duplicate_columns]
+
+            cursor.execute(sql, row_values)
+
+            if check_duplicate:
+                # 检查是否实际插入了行
+                return 'inserted' if cursor.rowcount > 0 else 'skipped'
+            return 'inserted'
+
+        except Exception as e:
+            logger.error(sys._getframe().f_code.co_name, {
+                'error_type': type(e).__name__,
+                '单行插入失败': str(e),
+                '是否排重': check_duplicate,
+                '排重列': duplicate_columns,
+                '处理方式': '继续处理剩余行'
+            })
+            return 'failed'
+
     def close(self):
         """
         关闭连接池并清理资源
@@ -1385,6 +1460,7 @@ class MySQLUploader:
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.close()
 
+
 def main():
     uploader = MySQLUploader(
         username='root',

{mdbq-3.10.1.dist-info → mdbq-3.10.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.10.1
+Version: 3.10.3
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.10.1.dist-info → mdbq-3.10.3.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=IV5Hh9sLBiZlaqbX0_vhJtOQad8a2sOKBbx_A0k8fik,18
+mdbq/__version__.py,sha256=S1pYeTgXo5MtZqzwck9ASp8x1pB1PZ33oC1NI7fY9dI,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
@@ -12,7 +12,7 @@ mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=Hvnc-Oqg0HloXTzu868k9Zh_aSMeVgfWgrqgeUWdJA0,58037
+mdbq/mysql/uploader.py,sha256=9wgFxsiFSAngdX2pWj57jElaspwqfPtadC-xQqvweUc,59066
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=YyPWa_nOH1zs8wgTDcgzn5w8szGKWPyWzmWMVIPkFnU,21638
-mdbq-3.10.1.dist-info/METADATA,sha256=CQR_RVCq58Ui1Llc2n_u3q0kzL8FJU_SKptHBVSswCM,364
-mdbq-3.10.1.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.10.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.10.1.dist-info/RECORD,,
+mdbq-3.10.3.dist-info/METADATA,sha256=TM8JAb8gTTte7N0agKbaDWZ14bmkl66dgCxIbLTqCbc,364
+mdbq-3.10.3.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.10.3.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.10.3.dist-info/RECORD,,