mdbq 3.9.13__py3-none-any.whl → 3.9.14__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.13'
+VERSION = '3.9.14'

mdbq/log/mylogger.py

@@ -222,7 +222,7 @@ class MyLogger:
                         'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                         'level': record.levelname,
                         'message': record.getMessage(),
-                        'name': record.name,
+                        # 'name': record.name,
                         # 'module': record.module,
                         # 'function': record.funcName,
                     }

mdbq/mysql/uploader.py

@@ -138,7 +138,6 @@ class MySQLUploader:
         if hasattr(self, 'pool') and self.pool is not None and self._check_pool_health():
             return self.pool
 
-        start_time = time.time()
         self.pool = None
 
         pool_params = {
@@ -172,18 +171,14 @@ class MySQLUploader:
 
         try:
             pool = PooledDB(**pool_params)
-            elapsed = round(time.time() - start_time, 2)
             logger.info("连接池创建成功", {
-                'pool_size': self.pool_size,
-                '耗时': elapsed
+                '连接池': self.pool_size
             })
             return pool
         except Exception as e:
-            elapsed = round(time.time() - start_time, 2)
             self.pool = None
             logger.error("连接池创建失败", {
-                'error': str(e),
-                '耗时': elapsed
+                'error': str(e)
             })
             raise ConnectionError(f"连接池创建失败: {str(e)}")
 
@@ -198,7 +193,6 @@ class MySQLUploader:
         @wraps(func)
         def wrapper(*args, **kwargs):
             last_exception = None
-            start_time = time.time()
             operation = func.__name__
 
             logger.debug(f"开始执行操作: {operation}", {
@@ -209,18 +203,15 @@ class MySQLUploader:
             for attempt in range(self.max_retries):
                 try:
                     result = func(*args, **kwargs)
-                    elapsed = round(time.time() - start_time, 2)
 
                     if attempt > 0:
                         logger.info("操作成功(重试后)", {
                             'operation': operation,
-                            'attempts': attempt + 1,
-                            '耗时': elapsed
+                            'attempts': attempt + 1
                         })
                     else:
                         logger.debug("操作成功", {
-                            'operation': operation,
-                            '耗时': elapsed
+                            'operation': operation
                         })
 
                     return result
@@ -252,15 +243,11 @@ class MySQLUploader:
                                 'error': str(reconnect_error)
                             })
                     else:
-                        elapsed = round(time.time() - start_time, 2)
-                        error_details['耗时'] = elapsed
-                        logger.error(f"操作最终失败 {error_details}")
+                        logger.error(f"操作最终失败")
 
                 except pymysql.IntegrityError as e:
-                    elapsed = round(time.time() - start_time, 2)
                     logger.error("完整性约束错误", {
                         'operation': operation,
-                        '耗时': elapsed,
                         'error_code': e.args[0] if e.args else None,
                         'error_message': e.args[1] if len(e.args) > 1 else None
                     })
@@ -268,10 +255,8 @@ class MySQLUploader:
 
                 except Exception as e:
                     last_exception = e
-                    elapsed = round(time.time() - start_time, 2)
                     logger.error("发生意外错误", {
                         'operation': operation,
-                        '耗时': elapsed,
                         'error_type': type(e).__name__,
                         'error_message': str(e),
                         'error_args': e.args if hasattr(e, 'args') else None
@@ -334,9 +319,9 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
                 conn.commit()
-                logger.info(f"{db_name} 数据库已创建")
+                logger.info(f"`{db_name}` 数据库已创建")
         except Exception as e:
-            logger.error(f"{db_name}: 无法创建数据库 {str(e)}")
+            logger.error(f"无法创建数据库: `{db_name}` -> {str(e)}")
             conn.rollback()
             raise
 
@@ -354,22 +339,10 @@ class MySQLUploader:
             # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
             date_obj = self._validate_datetime(date_value, True)
         except ValueError:
-            error_msg = f"`{table_name}` 无效的日期格式1: {date_value}"
+            error_msg = f"`{table_name}` 无效的日期格式1: `{date_value}`"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
-        # try:
-        #     # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
-        #     date_obj = self._validate_datetime(date_value, True)
-        # except ValueError:
-        #     try:
-        #         # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
-        #         date_obj = self._validate_datetime(date_value, True)
-        #     except ValueError:
-        #         error_msg = f"无效的日期格式1: {date_value}"
-        #         logger.error(error_msg)
-        #         raise ValueError(error_msg)
-
         if partition_by == 'year':
             return f"{table_name}_{date_obj.year}"
         elif partition_by == 'month':
@@ -388,7 +361,7 @@ class MySQLUploader:
         :raises ValueError: 当标识符无效时抛出
         """
         if not identifier or not isinstance(identifier, str):
-            error_msg = f"无效的标识符: {identifier}"
+            error_msg = f"无效的标识符: `{identifier}`"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
@@ -399,7 +372,7 @@ class MySQLUploader:
         cleaned = re.sub(r'_+', '_', cleaned).strip('_')
 
         if not cleaned:
-            error_msg = f"无法清理异常标识符: {identifier}"
+            error_msg = f"无法清理异常标识符: `{identifier}`"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
@@ -409,7 +382,7 @@ class MySQLUploader:
             'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
         }
         if cleaned.lower() in mysql_keywords:
-            logger.debug(f"存在MySQL保留字: {cleaned}")
+            logger.debug(f"存在MySQL保留字: `{cleaned}`")
             return f"`{cleaned}`"
 
         return cleaned
@@ -522,7 +495,7 @@ class MySQLUploader:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
-                    logger.info(f"{db_name}.{table_name}: 数据表已创建")
+                    logger.info(f"`{db_name}`.`{table_name}`: 数据表已创建")
 
                 # 添加普通索引
                 index_statements = []
@@ -551,10 +524,10 @@ class MySQLUploader:
                             logger.debug(f"Executed index statement: {stmt}", )
 
                 conn.commit()
-                logger.info(f"{db_name}.{table_name}: 索引已添加")
+                logger.info(f"索引已添加: `{db_name}`.`{table_name}` -> `{indexes}`")
 
         except Exception as e:
-            logger.error(f"{db_name}.{table_name}: 建表失败: {str(e)}")
+            logger.error(f"`{db_name}`.`{table_name}`: 建表失败: {str(e)}")
             conn.rollback()
             raise
 
@@ -587,7 +560,7 @@ class MySQLUploader:
                     return datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
             except ValueError:
                 continue
-        raise ValueError(f"无效的日期格式2: {value}")
+        raise ValueError(f"无效的日期格式2: `{value}`")
 
     def _validate_value(self, value: Any, column_type: str, allow_null: bool) -> Any:
         """
@@ -625,7 +598,7 @@ class MySQLUploader:
                     try:
                         return int(float(value))
                     except ValueError:
-                        raise ValueError(f"`{value}` 无法转为整数")
+                        raise ValueError(f"`{value}` -> 无法转为整数")
                 return int(value) if value is not None else None
             elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
                 if isinstance(value, str):
@@ -639,7 +612,7 @@ class MySQLUploader:
                     try:
                         return self._validate_datetime(value)  # 使用专门的日期验证方法
                     except ValueError as e:
-                        raise ValueError(f"无效日期格式: {value} - {str(e)}")
+                        raise ValueError(f"无效日期格式: `{value}` -> {str(e)}")
                 return str(value)
             elif 'char' in column_type_lower or 'text' in column_type_lower:
                 # 防止SQL注入
@@ -678,7 +651,7 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql, (db_name, table_name))
                     set_typ = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
-                    logger.debug(f"{db_name}.{table_name}: 获取表的列信息: {set_typ}")
+                    logger.debug(f"`{db_name}`.`{table_name}`: 获取表的列信息: `{set_typ}`")
                     return set_typ
         except Exception as e:
             logger.error(f"无法获取表列信息: {str(e)}")
@@ -707,21 +680,21 @@ class MySQLUploader:
                 self._create_table(db_name, table_name, set_typ, primary_keys, date_column, indexes,
                                    allow_null=allow_null)
             else:
-                error_msg = f"数据表不存在: '{db_name}.{table_name}'"
+                error_msg = f"数据表不存在: `{db_name}`.`{table_name}`"
                 logger.error(error_msg)
                 raise ValueError(error_msg)
 
         # 获取表结构并验证
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
-            error_msg = f"获取列失败 '{db_name}.{table_name}'"
+            error_msg = f"获取列失败 `{db_name}`.`{table_name}`"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 验证数据列与表列匹配
         for col in set_typ:
             if col not in table_columns:
-                error_msg = f"列不存在: '{col}' -> '{db_name}.{table_name}'"
+                error_msg = f"列不存在: `{col}` -> `{db_name}`.`{table_name}`"
                 logger.error(error_msg)
                 raise ValueError(error_msg)
 
@@ -869,7 +842,7 @@ class MySQLUploader:
                 if sample_values:
                     inferred_type = self._infer_data_type(sample_values[0])
                     filtered_set_typ[col] = inferred_type
-                    logger.debug(f"自动推断列 `{col}` 的数据类型为: {inferred_type}")
+                    logger.debug(f"自动推断列 `{col}` 的数据类型为: `{inferred_type}`")
                 else:
                     # 没有样本值，使用默认类型
                     filtered_set_typ[col] = 'VARCHAR(255)'
@@ -941,13 +914,13 @@ class MySQLUploader:
         batch_id = f"batch_{int(time.time() * 1000)}"
         success_flag = False
 
-        logger.info("开始上传数据", {
-            '批次号': batch_id,
+        logger.info("开始上传", {
             '库': db_name,
             '表': table_name,
+            '批次': batch_id,
             '分表方式': partition_by,
             '排重': check_duplicate,
-            '总计行数': len(data) if hasattr(data, '__len__') else 1,
+            '传入总计': len(data) if hasattr(data, '__len__') else 1,
             '自动建表': auto_create
         })
 
@@ -959,7 +932,7 @@ class MySQLUploader:
             if partition_by:
                 partition_by = str(partition_by).lower()
                 if partition_by not in ['year', 'month']:
-                    error_msg = "分表方式必须是 'year' 或 'month'"
+                    error_msg = "分表方式必须是 'year' 或 'month' 或 'None'"
                     logger.error(error_msg)
                     raise ValueError(error_msg)
 
@@ -971,7 +944,7 @@ class MySQLUploader:
                 if auto_create:
                     self._create_database(db_name)
                 else:
-                    error_msg = f"数据库不存在: '{db_name}'"
+                    error_msg = f"数据库不存在: `{db_name}`"
                     logger.error(error_msg)
                     raise ValueError(error_msg)
 
@@ -981,8 +954,12 @@ class MySQLUploader:
                 for row in prepared_data:
                     try:
                         if partition_date_column not in row:
-                            error_msg = f"异常缺失列 '{partition_date_column}'"
-                            logger.error(error_msg)
+                            logger.error('缺失列',{
+                                '库': db_name,
+                                '表': table_name,
+                                '批次': batch_id,
+                                '异常缺失列': partition_date_column,
+                            })
                             continue  # 跳过当前行
 
                         part_table = self._get_partition_table_name(
@@ -1011,7 +988,7 @@ class MySQLUploader:
                         )
                     except Exception as e:
                         logger.error("分表上传失败", {
-                            'partition_table': part_table,
+                            '分表': part_table,
                             'error': str(e)
                         })
                         continue  # 跳过当前分表，继续处理其他分表
@@ -1027,16 +1004,17 @@ class MySQLUploader:
             success_flag = True
 
         except Exception as e:
-            logger.error("上传过程中发生全局错误", {
+            logger.error("上传过程发生全局错误", {
                 'error': str(e),
                 'error_type': type(e).__name__
             })
         finally:
-            elapsed = round(time.time() - upload_start, 2)
             logger.info("上传处理完成", {
-                '批次号': batch_id,
+                '库': db_name,
+                '表': table_name,
+                '批次': batch_id,
                 'success': success_flag,
-                '耗时': elapsed,
+                '耗时': round(time.time() - upload_start, 2),
                 '数据行': initial_row_count
             })
 
@@ -1158,17 +1136,16 @@ class MySQLUploader:
                             total_failed += 1
 
                             # 记录失败行详细信息
-                            error_details = {
-                                '批次号': batch_id,
+                            logger.error(f"单行插入失败", {
                                 '库': db_name,
                                 '表': table_name,
+                                '批次': batch_id,
                                 'error_type': type(e).__name__,
                                 'error_message': str(e),
                                 '数据类型': set_typ,
                                 '是否排重': check_duplicate,
                                 '排重列': duplicate_columns
-                            }
-                            logger.error(f"单行插入失败: {error_details}")
+                            })
                             continue  # 跳过当前行，继续处理下一行
 
                     # 更新统计信息
@@ -1181,8 +1158,10 @@ class MySQLUploader:
                         total_inserted += successful_rows
 
                     batch_elapsed = round(time.time() - batch_start, 2)
-                    batch_info = {
-                        '批次号': batch_id,
+                    logger.debug("批次处理完成", {
+                        '库': db_name,
+                        '表': table_name,
+                        '批次': batch_id,
                         'batch_index': i // batch_size + 1,
                         'total_batches': (len(data) + batch_size - 1) // batch_size,
                         'batch_size': len(batch),
@@ -1190,14 +1169,15 @@ class MySQLUploader:
                         'failed_rows': len(batch) - successful_rows,
                         '耗时': batch_elapsed,
                         'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
-                    }
-                    logger.debug(f"批次处理完成 {batch_info}")
+                    })
 
-        logger.info("数据插入完成", {
-            '总数据行': len(data),
-            '插入行数': total_inserted,
-            '跳过行数': total_skipped,
-            '失败行数': total_failed
+        logger.info('插入完成', {
+            '库': db_name,
+            '表': table_name,
+            '完成总计': len(data),
+            '插入': total_inserted,
+            '跳过': total_skipped,
+            '失败': total_failed
         })
 
     def close(self):
@@ -1205,8 +1185,6 @@ class MySQLUploader:
         关闭连接池并清理资源
         :raises: 可能抛出关闭连接时的异常
         """
-        close_start = time.time()
-
         try:
             if hasattr(self, 'pool') and self.pool is not None:
                 # 更安全的关闭方式
@@ -1219,15 +1197,10 @@ class MySQLUploader:
 
                 self.pool = None
 
-                elapsed = round(time.time() - close_start, 2)
-                logger.info("连接池已关闭", {
-                    '耗时': elapsed
-                })
+                logger.info("连接池已关闭")
         except Exception as e:
-            elapsed = round(time.time() - close_start, 2)
             logger.error("关闭连接池失败", {
-                'error': str(e),
-                '耗时': elapsed
+                'error': str(e)
             })
             raise
 
@@ -1261,7 +1234,7 @@ class MySQLUploader:
                         if attempt < max_retries - 1:
                             time.sleep(delay * (attempt + 1))
                             continue
-                        raise logger.error(f"操作重试{max_retries}次后失败")
+                        raise logger.error(f"操作重试 {max_retries} 次后失败")
                     except Exception as e:
                         raise logger.error(f"操作失败: {str(e)}")
                 raise last_exception if last_exception else logger.error("操作重试失败，未知错误")

{mdbq-3.9.13.dist-info → mdbq-3.9.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.13
+Version: 3.9.14
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.13.dist-info → mdbq-3.9.14.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=hHYZV4WZxzfyOitDWrTp2i2d39HoIUIWNO-u9W_Hsrg,18
+mdbq/__version__.py,sha256=uU6jCmJep8DNWxJdEknjL2d2SQyXppESNa8rNLyG0II,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=jHCVO7KPQrg2kcCaIrakHivZmFBJyy-24sIn2rsbK4Y,24440
+mdbq/log/mylogger.py,sha256=hhUPq-IZREEDPLYDLENZyXJvwSGuOa_GX6QhY_Km-M4,24442
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=Fzaqbjg5g3HdNl50jInIrdurdzcgR2CCzdKLVImD1-Q,55339
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=QgwOCuJjaafr1cSBesfNz-rHspseMLNHrlYH3zmt0Ug,52421
+mdbq/mysql/uploader.py,sha256=QTSdxtvc9eyELfMi25zu1OhpHel-GKXbi-JiaK7m3JI,51108
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.13.dist-info/METADATA,sha256=3zm3JzLWPxjLvf0g3YSD-MBl2OUv5x4GukNymFgpebs,364
-mdbq-3.9.13.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.13.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.13.dist-info/RECORD,,
+mdbq-3.9.14.dist-info/METADATA,sha256=rxLTylQ7pJhOg_qsrLDNyT1rwgzefivgPjkc-UcrQEg,364
+mdbq-3.9.14.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.14.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.14.dist-info/RECORD,,