mdbq 3.9.17__tar.gz → 3.10.0__tar.gz

{mdbq-3.9.17 → mdbq-3.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.17
+Version: 3.10.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.10.0/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '3.10.0'

{mdbq-3.9.17 → mdbq-3.10.0}/mdbq/mysql/uploader.py

@@ -698,7 +698,8 @@ class MySQLUploader:
             date_column: Optional[str],
             indexes: Optional[List[str]],
             batch_id: Optional[str] = None,
-            update_on_duplicate: bool = False
+            update_on_duplicate: bool = False,
+            transaction_mode: str = "batch"
     ):
         """实际执行表上传的方法"""
         # 检查表是否存在
@@ -739,7 +740,8 @@ class MySQLUploader:
             db_name, table_name, data, set_typ,
             check_duplicate, duplicate_columns,
             batch_id=batch_id,
-            update_on_duplicate=update_on_duplicate
+            update_on_duplicate=update_on_duplicate,
+            transaction_mode=transaction_mode
         )
 
     def _infer_data_type(self, value: Any) -> str:
@@ -932,7 +934,8 @@ class MySQLUploader:
             partition_date_column: str = '日期',
             auto_create: bool = True,
             indexes: Optional[List[str]] = None,
-            update_on_duplicate: bool = False
+            update_on_duplicate: bool = False,
+            transaction_mode: str = "batch"
     ):
         """
         上传数据到数据库的主入口方法
@@ -950,6 +953,11 @@ class MySQLUploader:
         :param auto_create: 表不存在时是否自动创建，默认为True
         :param indexes: 需要创建索引的列列表，可选
         :param update_on_duplicate: 遇到重复数据时是否更新旧数据(默认为False)
+        :param transaction_mode: 事务提交模式，可选值：
+        - 'row'     : 逐行提交事务（错误隔离性好）
+        - 'batch'   : 整批提交事务（性能最优）
+        - 'hybrid'  : 混合模式（每N行提交，平衡性能与安全性）
+            默认值为 'batch'
         :raises: 可能抛出各种验证和数据库相关异常
         """
         upload_start = time.time()
@@ -1035,7 +1043,7 @@ class MySQLUploader:
                             db_name, part_table, part_data, filtered_set_typ,
                             primary_keys, check_duplicate, duplicate_columns,
                             allow_null, auto_create, partition_date_column,
-                            indexes, batch_id, update_on_duplicate
+                            indexes, batch_id, update_on_duplicate, transaction_mode
                         )
                     except Exception as e:
                         logger.error(sys._getframe().f_code.co_name, {
@@ -1051,7 +1059,7 @@ class MySQLUploader:
                     db_name, table_name, prepared_data, filtered_set_typ,
                     primary_keys, check_duplicate, duplicate_columns,
                     allow_null, auto_create, partition_date_column,
-                    indexes, batch_id, update_on_duplicate
+                    indexes, batch_id, update_on_duplicate, transaction_mode
                 )
 
             success_flag = True
@@ -1083,7 +1091,8 @@ class MySQLUploader:
             duplicate_columns: Optional[List[str]] = None,
             batch_size: int = 1000,
             batch_id: Optional[str] = None,
-            update_on_duplicate: bool = False
+            update_on_duplicate: bool = False,
+            transaction_mode: str = "batch"
     ):
         """
         实际执行数据插入的方法
@@ -1097,15 +1106,37 @@ class MySQLUploader:
         :param batch_size: 批量插入大小，默认为1000
         :param update_on_duplicate: 遇到重复数据时是否更新旧数据(默认为False)
         :param batch_id: 批次ID用于日志追踪，可选
+        :param transaction_mode: 事务提交模式，可选值：
+        - 'row'     : 逐行提交事务（错误隔离性好）
+        - 'batch'   : 整批提交事务（性能最优）
+        - 'hybrid'  : 混合模式（每N行提交，平衡性能与安全性）
+            默认值为 'batch'
         """
         if not data:
             return
 
+        valid_modes = ('row', 'batch', 'hybrid')
+        if transaction_mode.lower() not in valid_modes:
+            logger.error(sys._getframe().f_code.co_name, {
+                '库': db_name,
+                '表': table_name,
+                '参数异常': f'transaction_mode -> {transaction_mode}',
+                '可选值': valid_modes,
+                '自动使用默认模式': 'batch'
+            })
+            transaction_mode = 'batch'
+            # return
+
         # 获取所有列名（排除id列）
         all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
         safe_columns = [self._validate_identifier(col) for col in all_columns]
         placeholders = ','.join(['%s'] * len(safe_columns))
 
+        # 初始化统计变量
+        total_inserted = 0
+        total_skipped = 0
+        total_failed = 0
+
         # 构建基础SQL语句
         if check_duplicate:
             if not duplicate_columns:
@@ -1116,10 +1147,7 @@ class MySQLUploader:
             conditions = []
             for col in duplicate_columns:
                 col_type = set_typ.get(col, '').lower()
-
-                # 处理DECIMAL类型，使用ROUND确保精度一致
                 if col_type.startswith('decimal'):
-                    # 提取小数位数，如DECIMAL(10,2)提取2
                     scale_match = re.search(r'decimal\(\d+,(\d+)\)', col_type)
                     scale = int(scale_match.group(1)) if scale_match else 2
                     conditions.append(f"ROUND(`{self._validate_identifier(col)}`, {scale}) = ROUND(%s, {scale})")
@@ -1137,10 +1165,6 @@ class MySQLUploader:
                     VALUES ({placeholders})
                     ON DUPLICATE KEY UPDATE {update_clause}
                     """
-
-                # 注意：在update_on_duplicate模式下，row_values只需要插入数据，不需要排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns]
             else:
                 sql = f"""INSERT INTO `{db_name}`.`{table_name}` 
                     (`{'`,`'.join(safe_columns)}`) 
@@ -1151,10 +1175,6 @@ class MySQLUploader:
                         WHERE {where_clause}
                     )
                     """
-
-                # 在check_duplicate模式下，row_values需要插入数据+排重列值
-                def prepare_values(row):
-                    return [row.get(col) for col in all_columns] + [row.get(col) for col in duplicate_columns]
         else:
             sql = f"""
                 INSERT INTO `{db_name}`.`{table_name}` 
@@ -1162,35 +1182,44 @@ class MySQLUploader:
                 VALUES ({placeholders})
                 """
 
-            # 普通模式下，row_values只需要插入数据
-            def prepare_values(row):
-                return [row.get(col) for col in all_columns]
-
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0  # 失败计数器
-
         # 分批插入数据
         with self._get_connection() as conn:
             with conn.cursor() as cursor:
                 for i in range(0, len(data), batch_size):
-                    batch_start = time.time()
                     batch = data[i:i + batch_size]
-                    successful_rows = 0  # 当前批次成功数
+                    batch_inserted = 0
+                    batch_skipped = 0
+                    batch_failed = 0
 
-                    for row in batch:
+                    for row_idx, row in enumerate(batch, 1):
                         try:
                             # 准备参数
-                            row_values = prepare_values(row)
+                            row_values = [row.get(col) for col in all_columns]
+                            if check_duplicate and not update_on_duplicate:
+                                row_values += [row.get(col) for col in duplicate_columns]
+
                             cursor.execute(sql, row_values)
-                            successful_rows += 1
-                            conn.commit()  # 每次成功插入后提交
 
-                        except Exception as e:
-                            conn.rollback()  # 回滚当前行的事务
-                            total_failed += 1
+                            if check_duplicate:
+                                # 检查是否实际插入了行
+                                if cursor.rowcount > 0:
+                                    batch_inserted += 1
+                                else:
+                                    batch_skipped += 1
+                            else:
+                                batch_inserted += 1
+
+                            # 根据模式决定提交时机
+                            if transaction_mode == 'row':
+                                conn.commit()  # 逐行提交
+                            elif transaction_mode == 'hybrid' and row_idx % 100 == 0:
+                                conn.commit()  # 每100行提交一次
 
-                            # 记录失败行详细信息
+                        except Exception as e:
+                            # if transaction_mode == 'row':
+                            #     conn.rollback()
+                            conn.rollback()
+                            batch_failed += 1
                             logger.error(sys._getframe().f_code.co_name, {
                                 '库': db_name,
                                 '表': table_name,
@@ -1199,31 +1228,30 @@ class MySQLUploader:
                                 '单行插入失败': str(e),
                                 '数据类型': set_typ,
                                 '是否排重': check_duplicate,
-                                '排重列': duplicate_columns
+                                '排重列': duplicate_columns,
+                                '事务提交模式': transaction_mode,
                             })
-                            continue  # 跳过当前行，继续处理下一行
-
-                    # 更新统计信息
-                    if check_duplicate:
-                        cursor.execute("SELECT ROW_COUNT()")
-                        affected_rows = cursor.rowcount
-                        total_inserted += affected_rows
-                        total_skipped += len(batch) - affected_rows - (len(batch) - successful_rows)
-                    else:
-                        total_inserted += successful_rows
 
-                    batch_elapsed = round(time.time() - batch_start, 2)
+                    # 批量模式最后统一提交
+                    if transaction_mode in ('batch', 'hybrid'):
+                        conn.commit()
+
+                    # 更新总统计
+                    total_inserted += batch_inserted
+                    total_skipped += batch_skipped
+                    total_failed += batch_failed
+
                     logger.debug(sys._getframe().f_code.co_name, {
                         '库': db_name,
                         '表': table_name,
                         '批次': batch_id,
                         '批次处理完成': i // batch_size + 1,
-                        'total_batches': (len(data) + batch_size - 1) // batch_size,
-                        'batch_size': len(batch),
-                        'successful_rows': successful_rows,
-                        'failed_rows': len(batch) - successful_rows,
-                        '耗时': batch_elapsed,
-                        'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
+                        '总批次': (len(data) + batch_size - 1) // batch_size,
+                        '数据量': len(batch),
+                        '插入': batch_inserted,
+                        '跳过': batch_skipped,
+                        '失败': batch_failed,
+                        '事务提交模式': transaction_mode,
                     })
 
         logger.info('插入完成', {
@@ -1232,7 +1260,8 @@ class MySQLUploader:
             '完成总计': len(data),
             '插入': total_inserted,
             '跳过': total_skipped,
-            '失败': total_failed
+            '失败': total_failed,
+            '事务提交模式': transaction_mode,
         })
 
     def close(self):
@@ -1343,6 +1372,7 @@ def main():
         partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
         auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
         indexes=[],  # 指定索引列
+        transaction_mode='row',  # 事务模式
     )
 
     uploader.close()

{mdbq-3.9.17 → mdbq-3.10.0}/mdbq/spider/aikucun.py

@@ -28,10 +28,23 @@ config_file = os.path.join(dir_path, 'spd.txt')
 content = config.read_config(file_path=config_file)
 username, password, host, port = content['username'], content['password'], content['host'], content['port']
 
-m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
+# m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
+uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-logger = spider_logging.setup_logging()
+logger = mylogger.MyLogger(
+    name='aikucun',
+    logging_mode='file',
+    log_level='info',
+    log_file='aikucun.log',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=1,  # 采样50%的DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段列表
+    enable_metrics=False,  # 是否启用性能指标
+)
 
 
 def keep_connect(_db_name, _config, max_try: int=10):
@@ -175,10 +188,26 @@ class AikuCun:
             '更新时间': 'timestamp'
         }
         # 更新至数据库记录
-        m_engine.dict_to_mysql(
+        # m_engine.dict_to_mysql(
+        #     db_name=self.db_name,
+        #     table_name=self.table_name,
+        #     dict_data=self.token,
+        # )
+        uld.upload_data(
             db_name=self.db_name,
             table_name=self.table_name,
-            dict_data=self.token,
+            data=self.token,
+            set_typ={},
+            primary_keys=[],
+            check_duplicate=False,
+            update_on_duplicate=False,
+            duplicate_columns=[],
+            allow_null=False,
+            partition_by=None,
+            partition_date_column='日期',
+            auto_create=True,
+            indexes=[],
+            transaction_mode='row',  # 事务模式
         )
 
     def get_data_from_bbx(self, start_date=None, end_date=None, item_type='spu', page_num=1, page_size=300):
@@ -400,14 +429,30 @@ class AikuCun:
             drop_dup = ['日期', '平台', '店铺名称', '商品款号', '访客量']
         else:
             drop_dup = ['日期', '平台', '店铺名称', '条码']
-        m_engine.insert_many_dict(
+        # m_engine.insert_many_dict(
+        #     db_name=db_name,
+        #     table_name=table_name,
+        #     dict_data_list=_results,
+        #     icm_update=drop_dup,  # 唯一组合键
+        #     # unique_main_key=['人群id'],
+        #     set_typ=set_typ,
+        #     allow_not_null=False,  # 创建允许插入空值的列
+        # )
+        uld.upload_data(
             db_name=db_name,
             table_name=table_name,
-            dict_data_list=_results,
-            icm_update=drop_dup,  # 唯一组合键
-            # unique_main_key=['人群id'],
-            set_typ=set_typ,
-            allow_not_null=False,  # 创建允许插入空值的列
+            data=_results,
+            set_typ=set_typ,  # 定义列和数据类型
+            primary_keys=[],  # 创建唯一主键
+            check_duplicate=True,  # 检查重复数据
+            update_on_duplicate=False,  # 遇到重复时更新数据，默认 False 跳过
+            duplicate_columns=drop_dup,  # 指定排重的组合键
+            allow_null=False,  # 允许插入空值
+            partition_by=None,  # 按年/月分表
+            partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
+            auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
+            indexes=[],  # 指定索引列
+            transaction_mode='row',  # 事务模式
         )
 
     def get_sign(self):

{mdbq-3.9.17 → mdbq-3.10.0}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.17
+Version: 3.10.0
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-3.9.17/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '3.9.17'