mdbq 3.9.7__py3-none-any.whl → 3.9.9__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '3.9.7'
+VERSION = '3.9.9'

mdbq/log/mylogger.py

@@ -9,6 +9,8 @@ import threading
 import queue
 from typing import Optional, Dict, Any, List, Callable, Union
 import atexit
+import traceback
+import inspect
 
 try:
     import psutil
@@ -123,6 +125,7 @@ class MyLogger:
         # 定时刷新相关
         self._flush_thread = None
         self._last_flush_time = 0
+        self._start_flush_thread()
 
         # 创建日志记录器
         self.logger = logging.getLogger(name)
@@ -219,12 +222,9 @@ class MyLogger:
                         'timestamp': datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                         'level': record.levelname,
                         'message': record.getMessage(),
-                        # 'module': record.module,
-                        'function': record.funcName,
-                        # 'line': record.lineno,
-                        # 'thread': record.threadName,
-                        # 'process': record.processName,
                         'name': record.name,
+                        # 'module': record.module,
+                        # 'function': record.funcName,
                     }
 
                     # 添加额外字段
@@ -347,6 +347,45 @@ class MyLogger:
         )
         self._async_thread.start()
 
+    def log_error_handler(retry_times=0, fallback_level='error'):
+        """
+        日志错误处理装饰器
+
+        参数:
+        - retry_times: 异常时重试次数
+        - fallback_level: 降级日志级别
+        """
+
+        def decorator(log_method):
+            def wrapper(self, level: str, message: str, extra: Optional[Dict] = None):
+                last_exception = None
+                for attempt in range(retry_times + 1):
+                    try:
+                        return log_method(self, level, message, extra)
+                    except Exception as e:
+                        last_exception = e
+                        if attempt < retry_times:
+                            time.sleep(0.1 * (attempt + 1))  # 简单的退避策略
+                            continue
+
+                        try:
+                            # 降级处理
+                            logging.basicConfig()
+                            fallback_logger = logging.getLogger(f"{getattr(self, 'name', 'mylogger')}_fallback")
+                            fallback_msg = f"[降级处理] {message}"[:1000]
+                            getattr(fallback_logger, fallback_level)(
+                                f"日志记录失败(尝试{attempt + 1}次): {e}\n原始消息: {fallback_msg}"
+                            )
+                        except:
+                            sys.stderr.write(f"严重: 日志系统完全失败 - {last_exception}\n")
+
+                return None
+
+            return wrapper
+
+        return decorator
+
+    @log_error_handler(retry_times=1, fallback_level='warning')
     def _sync_log(self, level: str, message: str, extra: Optional[Dict] = None):
         """同步日志记录"""
         if not hasattr(self.logger, level.lower()):
@@ -375,7 +414,8 @@ class MyLogger:
             log_extra['context_data'] = self._context.data.copy()
 
         # 添加敏感字段过滤
-        log_extra['过滤'] = self.sensitive_fields
+        if self.sensitive_fields:
+            log_extra['过滤'] = self.sensitive_fields
 
         # 应用日志采样
         if self.sample_rate < 1.0 and level.lower() in ('debug', 'info'):
@@ -441,28 +481,6 @@ class MyLogger:
         if hasattr(self._context, 'data'):
             self._context.data.clear()
 
-    def shutdown(self):
-        """关闭日志记录器，确保所有日志被刷新"""
-        if self.enable_async:
-            self._stop_event.set()
-            # 等待队列清空
-            while not self._log_queue.empty():
-                time.sleep(0.1)
-            if self._async_thread and self._async_thread.is_alive():
-                self._async_thread.join(timeout=2)
-            if self._flush_thread and self._flush_thread.is_alive():
-                self._flush_thread.join(timeout=2)
-
-        # 确保所有handler被刷新
-        self._flush_handlers()
-
-        # 关闭所有handler
-        for handler in self.logger.handlers:
-            try:
-                handler.close()
-            except:
-                pass
-
     def debug(self, message: str, extra: Optional[Dict] = None):
         """记录调试信息"""
         self.log('debug', message, extra)
@@ -487,41 +505,46 @@ class MyLogger:
         """记录异常信息"""
         if not extra:
             extra = {}
-        # # 获取异常发生的实际位置
+
+        # # 获取完整的异常堆栈
         # tb = exc_info.__traceback__
+        # while tb.tb_next:
+        #     tb = tb.tb_next  # 获取最内层的堆栈帧
         #
-        # if tb:
-        #     extra.update({
-        #         'module': tb.tb_frame.f_globals.get('__name__', ''),
-        #         'function': tb.tb_frame.f_code.co_name,
-        #         'line': tb.tb_lineno,
-        #         'file': tb.tb_frame.f_code.co_filename
-        #     })
-        # extra['异常'] = str(exc_info)
-        # extra['类型'] = exc_info.__class__.__name__
-        # self.log('error', message, extra)
-
-        # 获取完整的异常堆栈
-        tb = exc_info.__traceback__
-        while tb.tb_next:
-            tb = tb.tb_next  # 获取最内层的堆栈帧
-
-        extra.update({
-            'module': tb.tb_frame.f_globals.get('__name__', ''),
-            'function': tb.tb_frame.f_code.co_name,
-            'line': tb.tb_lineno,
-            'file': tb.tb_frame.f_code.co_filename,
-            '异常': str(exc_info),
-            '类型': exc_info.__class__.__name__,
-            '堆栈': self._format_traceback(exc_info)
-        })
+        # extra.update({
+        #     'module': tb.tb_frame.f_globals.get('__name__', ''),
+        #     'function': tb.tb_frame.f_code.co_name,
+        #     'line': tb.tb_lineno,
+        #     'file': tb.tb_frame.f_code.co_filename,
+        #     '异常': str(exc_info),
+        #     '类型': exc_info.__class__.__name__,
+        #     '堆栈': self._format_traceback(exc_info)
+        # })
+
+        # 使用inspect获取调用栈
+        frame = inspect.currentframe()
+        try:
+            # 向上追溯2层（1层是exception方法本身，2层是实际调用位置）
+            caller_frame = frame.f_back.f_back
+            extra.update({
+                'module': caller_frame.f_globals.get('__name__', ''),
+                'function': caller_frame.f_code.co_name,
+                'line': caller_frame.f_lineno,
+                'file': caller_frame.f_code.co_filename,
+                '异常': str(exc_info),
+                '类型': exc_info.__class__.__name__,
+                '堆栈': self._format_traceback(exc_info)
+            })
+        finally:
+            del frame  # 避免循环引用
 
         # 直接使用logger的error方法记录，保留原始调用栈
         self.log('error', message, extra)
 
     def _format_traceback(self, exc_info):
         """格式化异常堆栈"""
-        import traceback
+        if exc_info is None:
+            return ""
         return ''.join(traceback.format_exception(type(exc_info), exc_info, exc_info.__traceback__))
 
     def timeit(self, message: str = "Execution time"):
@@ -579,17 +602,40 @@ class MyLogger:
                 except:
                     pass
 
+    def shutdown(self):
+        """关闭日志记录器，确保所有日志被刷新"""
+        if self.enable_async:
+            self._stop_event.set()
+            # 等待队列清空
+            while not self._log_queue.empty():
+                time.sleep(0.1)
+            if self._async_thread and self._async_thread.is_alive():
+                self._async_thread.join(timeout=0.5)
+
+        # 确保所有handler被刷新
+        if self._flush_thread:
+            self._flush_handlers()
+            if self._flush_thread.is_alive():
+                self._flush_thread.join(timeout=0.5)
+
+        # 关闭所有handler
+        for handler in self.logger.handlers:
+            try:
+                handler.close()
+            except:
+                pass
+
 def main():
     # 创建日志记录器
     logger = MyLogger(
         name='my_app',
         logging_mode='both',
         log_level='DEBUG',
-        log_file='/Users/xigua/Downloads/my_app.log',
+        log_file='my_app.log',
         log_format='json',
         max_log_size=50,
         backup_count=5,
-        enable_async=True,  # 是否启用异步日志
+        enable_async=False,  # 是否启用异步日志
         sample_rate=1,  # 采样50%的DEBUG/INFO日志
         sensitive_fields=[],  #  敏感字段列表
         enable_metrics=False,  # 是否启用性能指标
@@ -603,4 +649,3 @@ def main():
 
 if __name__ == '__main__':
     pass
-    main()

mdbq/mysql/deduplicator.py

@@ -17,14 +17,14 @@ from collections import defaultdict
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
     name='deduplicator',
-    logging_mode='none',
-    log_level='error',
+    logging_mode='both',
+    log_level='info',
     log_file='deduplicator.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=0.5,  # 采样50%的DEBUG/INFO日志
+    sample_rate=1,  # 采样50%的DEBUG/INFO日志
     sensitive_fields=[],  #  敏感字段列表
 )
 
@@ -89,6 +89,9 @@ class MySQLDeduplicator:
         :param retry_interval: 重试间隔(秒)
         :param pool_size: 连接池大小
         """
+        # 连接池状态标志
+        self._closed = False
+
         # 初始化连接池
         self.pool = PooledDB(
             creator=pymysql,
@@ -113,13 +116,12 @@ class MySQLDeduplicator:
         self._processing_tables = set()  # 正在处理的表集合
 
         # 系统数据库列表
-        self.SYSTEM_DATABASES = {
-            'information_schema', 'mysql',
-            'performance_schema', 'sys'
-        }
+        self.SYSTEM_DATABASES = {'information_schema', 'mysql', 'performance_schema', 'sys'}
 
     def _get_connection(self):
         """从连接池获取连接"""
+        if self._closed:
+            raise ConnectionError("连接池已关闭")
         try:
             conn = self.pool.connection()
             logger.debug("成功获取数据库连接")
@@ -263,7 +265,9 @@ class MySQLDeduplicator:
 
             # 构建去重SQL
             column_list = ', '.join([f'`{col}`' for col in use_columns])
-            temp_table = f"temp_{table}_{int(time.time())}"
+            # temp_table = f"temp_{table}_{int(time.time())}"
+            temp_table = f"temp_{table}_dedup_{os.getpid()}"  # 使用进程ID构建临时表
+            temp_table = re.sub(r'[^a-zA-Z0-9_]', '_', temp_table)  # 确保表名合法
 
             # 使用临时表方案处理去重，避免锁表问题
             create_temp_sql = f"""
@@ -556,13 +560,12 @@ class MySQLDeduplicator:
     def close(self):
         """关闭连接池"""
         try:
-            if hasattr(self, 'pool') and self.pool:
+            if hasattr(self, 'pool') and self.pool and not self._closed:
                 self.pool.close()
+                self._closed = True
                 logger.info("数据库连接池已关闭")
         except Exception as e:
             logger.error(f"关闭连接池时出错: {str(e)}", {'error_type': type(e).__name__})
-        finally:
-            self.pool = None
 
     def __enter__(self):
         return self
@@ -574,7 +577,7 @@ class MySQLDeduplicator:
 def main():
     deduplicator = MySQLDeduplicator(
         username='root',
-        password='pw',
+        password='188988yang188',
         host='localhost',
         port=3306
     )

mdbq/mysql/uploader.py

@@ -17,20 +17,26 @@ from collections import OrderedDict
 warnings.filterwarnings('ignore')
 logger = mylogger.MyLogger(
     name='uploader',
-    logging_mode='none',
-    log_level='error',
+    logging_mode='both',
+    log_level='info',
     log_file='uploader.log',
     log_format='json',
     max_log_size=50,
     backup_count=5,
     enable_async=False,  # 是否启用异步日志
-    sample_rate=0.5,  # 采样50%的DEBUG/INFO日志
+    sample_rate=1,  # 采样50%的DEBUG/INFO日志
     sensitive_fields=[],  #  敏感字段列表
 )
 
 
 def count_decimal_places(num_str):
-    """ 计算小数位数, 允许科学计数法 """
+    """
+    计算数字字符串的小数位数，支持科学计数法
+
+    :param num_str: 数字字符串
+    :return: 返回元组(整数位数, 小数位数)
+    :raises: 无显式抛出异常，但正则匹配失败时返回(0, 0)
+    """
     match = re.match(r'^[-+]?\d+(\.\d+)?([eE][-+]?\d+)?$', str(num_str))
     if match:
         # 如果是科学计数法
@@ -53,8 +59,13 @@ def count_decimal_places(num_str):
 
 
 class StatementCache(OrderedDict):
-    """LRU缓存策略"""
+    """基于OrderedDict实现的LRU缓存策略，用于缓存SQL语句"""
     def __init__(self, maxsize=100):
+        """
+        初始化缓存
+
+        :param maxsize: 最大缓存大小，默认为100
+        """
         super().__init__()
         self.maxsize = maxsize
 
@@ -119,7 +130,12 @@ class MySQLUploader:
         self.pool = self._create_connection_pool()
 
     def _create_connection_pool(self) -> PooledDB:
-        """创建数据库连接池"""
+        """
+        创建数据库连接池
+
+        :return: PooledDB连接池实例
+        :raises ConnectionError: 当连接池创建失败时抛出
+        """
         if hasattr(self, 'pool') and self.pool is not None and self._check_pool_health():
             return self.pool
 
@@ -157,22 +173,29 @@ class MySQLUploader:
 
         try:
             pool = PooledDB(**pool_params)
-            elapsed = time.time() - start_time
+            elapsed = round(time.time() - start_time, 2)
             logger.info("连接池创建成功", {
                 'pool_size': self.pool_size,
-                'time_elapsed': elapsed
+                '耗时': elapsed
             })
             return pool
         except Exception as e:
-            elapsed = time.time() - start_time
+            elapsed = round(time.time() - start_time, 2)
             self.pool = None
             logger.error("连接池创建失败", {
                 'error': str(e),
-                'time_elapsed': elapsed
+                '耗时': elapsed
             })
             raise ConnectionError(f"连接池创建失败: {str(e)}")
 
     def _execute_with_retry(self, func):
+        """
+        带重试机制的装饰器，用于数据库操作
+
+        :param func: 被装饰的函数
+        :return: 装饰后的函数
+        :raises: 可能抛出原始异常或最后一次重试的异常
+        """
         @wraps(func)
         def wrapper(*args, **kwargs):
             last_exception = None
@@ -187,18 +210,18 @@ class MySQLUploader:
             for attempt in range(self.max_retries):
                 try:
                     result = func(*args, **kwargs)
-                    elapsed = time.time() - start_time
+                    elapsed = round(time.time() - start_time, 2)
 
                     if attempt > 0:
                         logger.info("操作成功(重试后)", {
                             'operation': operation,
                             'attempts': attempt + 1,
-                            'time_elapsed': elapsed
+                            '耗时': elapsed
                         })
                     else:
                         logger.debug("操作成功", {
                             'operation': operation,
-                            'time_elapsed': elapsed
+                            '耗时': elapsed
                         })
 
                     return result
@@ -230,15 +253,15 @@ class MySQLUploader:
                                 'error': str(reconnect_error)
                             })
                     else:
-                        elapsed = time.time() - start_time
-                        error_details['time_elapsed'] = elapsed
+                        elapsed = round(time.time() - start_time, 2)
+                        error_details['耗时'] = elapsed
                         logger.error(f"操作最终失败 {error_details}")
 
                 except pymysql.IntegrityError as e:
-                    elapsed = time.time() - start_time
+                    elapsed = round(time.time() - start_time, 2)
                     logger.error("完整性约束错误", {
                         'operation': operation,
-                        'time_elapsed': elapsed,
+                        '耗时': elapsed,
                         'error_code': e.args[0] if e.args else None,
                         'error_message': e.args[1] if len(e.args) > 1 else None
                     })
@@ -246,10 +269,10 @@ class MySQLUploader:
 
                 except Exception as e:
                     last_exception = e
-                    elapsed = time.time() - start_time
+                    elapsed = round(time.time() - start_time, 2)
                     logger.error("发生意外错误", {
                         'operation': operation,
-                        'time_elapsed': elapsed,
+                        '耗时': elapsed,
                         'error_type': type(e).__name__,
                         'error_message': str(e),
                         'error_args': e.args if hasattr(e, 'args') else None
@@ -261,7 +284,12 @@ class MySQLUploader:
         return wrapper
 
     def _get_connection(self):
-        """从连接池获取连接"""
+        """
+        从连接池获取数据库连接
+
+        :return: 数据库连接对象
+        :raises ConnectionError: 当获取连接失败时抛出
+        """
         try:
             conn = self.pool.connection()
             logger.debug("获取数据库连接")
@@ -271,7 +299,13 @@ class MySQLUploader:
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     def _check_database_exists(self, db_name: str) -> bool:
-        """检查数据库是否存在"""
+        """
+        检查数据库是否存在
+
+        :param db_name: 数据库名称
+        :return: 存在返回True，否则返回False
+        :raises: 可能抛出数据库相关异常
+        """
         db_name = self._validate_identifier(db_name)
         sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
 
@@ -287,7 +321,12 @@ class MySQLUploader:
             raise
 
     def _create_database(self, db_name: str):
-        """创建数据库"""
+        """
+        创建数据库
+
+        :param db_name: 要创建的数据库名称
+        :raises: 可能抛出数据库相关异常
+        """
         db_name = self._validate_identifier(db_name)
         sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
 
@@ -329,18 +368,17 @@ class MySQLUploader:
         elif partition_by == 'month':
             return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
         else:
-            error_msg = "partition_by must be 'year' or 'month'"
+            error_msg = "分表方式必须是 'year' 或 'month'"
             logger.error(error_msg)
             raise ValueError(error_msg)
 
     def _validate_identifier(self, identifier: str) -> str:
         """
-        验证并清理数据库标识符(数据库名、表名、列名)
-        防止SQL注入和非法字符
+        验证并清理数据库标识符(表名、列名等)
 
         :param identifier: 要验证的标识符
         :return: 清理后的安全标识符
-        :raises ValueError: 如果标识符无效
+        :raises ValueError: 当标识符无效时抛出
         """
         if not identifier or not isinstance(identifier, str):
             error_msg = f"无效的标识符: {identifier}"
@@ -366,7 +404,14 @@ class MySQLUploader:
         return cleaned
 
     def _check_table_exists(self, db_name: str, table_name: str) -> bool:
-        """检查表是否存在"""
+        """
+        检查表是否存在
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :return: 存在返回True，否则返回False
+        :raises: 可能抛出数据库相关异常
+        """
         cache_key = f"{db_name}.{table_name}"
         if cache_key in self._table_metadata_cache:
             cached_time, result = self._table_metadata_cache[cache_key]
@@ -410,9 +455,11 @@ class MySQLUploader:
         :param db_name: 数据库名
         :param table_name: 表名
         :param set_typ: 列名和数据类型字典 {列名: 数据类型}
-        :param primary_keys: 主键列列表
-        :param date_column: 日期列名，如果存在将设置为索引
-        :param indexes: 需要创建索引的列列表
+        :param primary_keys: 主键列列表，可选
+        :param date_column: 日期列名，可选，如果存在将设置为索引
+        :param indexes: 需要创建索引的列列表，可选
+        :param allow_null: 是否允许空值，默认为False
+        :raises: 可能抛出数据库相关异常
         """
         db_name = self._validate_identifier(db_name)
         table_name = self._validate_identifier(table_name)
@@ -501,7 +548,14 @@ class MySQLUploader:
             raise
 
     def _validate_datetime(self, value, date_type=False):
-        """date_type: 返回字符串类型或者日期类型"""
+        """
+        验证并标准化日期时间格式
+
+        :param value: 日期时间值
+        :param date_type: 是否返回日期类型(True)或字符串(False)
+        :return: 标准化后的日期时间字符串或日期对象
+        :raises ValueError: 当日期格式无效时抛出
+        """
         formats = [
             '%Y-%m-%d %H:%M:%S',
             '%Y-%m-%d',
@@ -524,26 +578,42 @@ class MySQLUploader:
                 continue
         raise ValueError(f"无效的日期格式2: {value}")
 
-    def _validate_value(self, value: Any, column_type: str) -> Any:
+    def _validate_value(self, value: Any, column_type: str, allow_null: bool) -> Any:
         """
-        验证并清理数据值，根据列类型进行适当转换
+        根据列类型验证并转换数据值
 
         :param value: 要验证的值
         :param column_type: 列的数据类型
-        :return: 清理后的值
-        :raises ValueError: 如果值转换失败
+        :param allow_null: 是否允许空值
+        :return: 转换后的值
+        :raises ValueError: 当值转换失败时抛出
         """
         if value is None:
+            if not allow_null:
+                return 'none'
             return None
 
         try:
             column_type_lower = column_type.lower()
 
+            # 处理百分比值
+            if isinstance(value, str) and '%' in value:
+                try:
+                    # 移除百分号并转换为小数
+                    percent_value = float(value.strip().replace('%', ''))
+                    decimal_value = percent_value / 100
+                    return decimal_value
+                except ValueError:
+                    pass  # 如果不是有效的百分比数字，继续正常处理
+
             if 'int' in column_type_lower:
                 if isinstance(value, (str, bytes)) and not value.strip().isdigit():
                     raise ValueError("非数字字符串无法转换为整数")
                 return int(value)
             elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
+                if isinstance(value, str):
+                    # 处理可能包含逗号的数字字符串
+                    value = value.replace(',', '')
                 return float(value) if value is not None else None
             elif '日期' in column_type_lower or 'time' in column_type_lower:
                 if isinstance(value, (datetime.datetime, pd.Timestamp)):
@@ -570,7 +640,14 @@ class MySQLUploader:
             raise ValueError(error_msg)
 
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
-        """获取表的列名和数据类型"""
+        """
+        获取表的列名和数据类型
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :return: 列名和数据类型字典 {列名: 数据类型}
+        :raises: 可能抛出数据库相关异常
+        """
         db_name = self._validate_identifier(db_name)
         table_name = self._validate_identifier(table_name)
         sql = """
@@ -639,7 +716,7 @@ class MySQLUploader:
 
     def _infer_data_type(self, value: Any) -> str:
         """
-        根据值推断合适的数据类型
+        根据值推断合适的MySQL数据类型
 
         :param value: 要推断的值
         :return: MySQL数据类型字符串
@@ -647,6 +724,10 @@ class MySQLUploader:
         if value is None:
             return 'VARCHAR(255)'  # 默认字符串类型
 
+        # 检查是否是百分比字符串
+        if isinstance(value, str) and '%' in value:
+            return 'DECIMAL(10,4)'  # 百分比统一使用DECIMAL(10,4)
+
         if isinstance(value, bool):
             return 'TINYINT(1)'
         elif isinstance(value, int):
@@ -661,7 +742,10 @@ class MySQLUploader:
             else:
                 return 'BIGINT'
         elif isinstance(value, float):
-            return 'DECIMAL(10,2)'
+            # 计算小数位数
+            num_str = str(value)
+            _, decimal_places = count_decimal_places(num_str)
+            return f'DECIMAL(20,{min(decimal_places, 6)})'  # 限制最大6位小数
         elif isinstance(value, (datetime.datetime, pd.Timestamp)):
             return 'DATETIME'
         elif isinstance(value, datetime.date):
@@ -694,15 +778,15 @@ class MySQLUploader:
             data: Union[Dict, List[Dict], pd.DataFrame],
             set_typ: Dict[str, str],
             allow_null: bool = False
-    ) -> List[Dict]:
+    ) -> Tuple[List[Dict], Dict[str, str]]:
         """
         准备要上传的数据，验证并转换数据类型
 
-        :param data: 输入数据
+        :param data: 输入数据，可以是字典、字典列表或DataFrame
         :param set_typ: 列名和数据类型字典 {列名: 数据类型}
         :param allow_null: 是否允许空值
-        :return: 待上传的数据列表和对应的数据类型
-        :raises ValueError: 如果数据验证失败
+        :return: 元组(准备好的数据列表, 过滤后的列类型字典)
+        :raises ValueError: 当数据验证失败时抛出
         """
         # 统一数据格式为字典列表
         if isinstance(data, pd.DataFrame):
@@ -764,7 +848,7 @@ class MySQLUploader:
                     prepared_row[col_name] = None
                 else:
                     try:
-                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name])
+                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name], allow_null)
                     except ValueError as e:
                         error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
                         logger.error(error_msg)
@@ -790,7 +874,21 @@ class MySQLUploader:
             indexes: Optional[List[str]] = None
     ):
         """
-        上传数据到数据库
+        上传数据到数据库的主入口方法
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要上传的数据
+        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
+        :param primary_keys: 主键列列表，可选
+        :param check_duplicate: 是否检查重复数据，默认为False
+        :param duplicate_columns: 用于检查重复的列，可选
+        :param allow_null: 是否允许空值，默认为False
+        :param partition_by: 分表方式('year'或'month')，可选
+        :param partition_date_column: 用于分表的日期列名，默认为'日期'
+        :param auto_create: 表不存在时是否自动创建，默认为True
+        :param indexes: 需要创建索引的列列表，可选
+        :raises: 可能抛出各种验证和数据库相关异常
         """
         upload_start = time.time()
         initial_row_count = len(data) if hasattr(data, '__len__') else 1
@@ -799,21 +897,21 @@ class MySQLUploader:
         success_flag = False
 
         logger.info("开始上传数据", {
-            'batch_id': batch_id,
+            '批次号': batch_id,
             'database': db_name,
             'table': table_name,
-            'partition_by': partition_by,
-            'check_duplicate': check_duplicate,
-            'row_count': len(data) if hasattr(data, '__len__') else 1,
-            'auto_create': auto_create
+            '分表方式': partition_by,
+            '是否排重': check_duplicate,
+            '总计行数': len(data) if hasattr(data, '__len__') else 1,
+            '自动建表': auto_create
         })
 
         try:
-            # 验证参数
-            if not set_typ:
-                error_msg = "列的数据类型缺失"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
+            # # 验证参数
+            # if not set_typ:
+            #     error_msg = "列的数据类型缺失"
+            #     logger.error(error_msg)
+            #     raise ValueError(error_msg)
 
             if partition_by and partition_by not in ['year', 'month']:
                 error_msg = "分表方式必须是 'year' 或 'month'"
@@ -821,7 +919,7 @@ class MySQLUploader:
                 raise ValueError(error_msg)
 
             # 准备数据
-            prepared_data, set_typ = self._prepare_data(data, set_typ, allow_null)
+            prepared_data, filtered_set_typ = self._prepare_data(data, set_typ, allow_null)
 
             # 检查数据库是否存在
             if not self._check_database_exists(db_name):
@@ -861,7 +959,7 @@ class MySQLUploader:
                 for part_table, part_data in partitioned_data.items():
                     try:
                         self._upload_to_table(
-                            db_name, part_table, part_data, set_typ,
+                            db_name, part_table, part_data, filtered_set_typ,
                             primary_keys, check_duplicate, duplicate_columns,
                             allow_null, auto_create, partition_date_column,
                             indexes, batch_id
@@ -875,7 +973,7 @@ class MySQLUploader:
             else:
                 # 不分表，直接上传
                 self._upload_to_table(
-                    db_name, table_name, prepared_data, set_typ,
+                    db_name, table_name, prepared_data, filtered_set_typ,
                     primary_keys, check_duplicate, duplicate_columns,
                     allow_null, auto_create, partition_date_column,
                     indexes, batch_id
@@ -889,12 +987,12 @@ class MySQLUploader:
                 'error_type': type(e).__name__
             })
         finally:
-            elapsed = time.time() - upload_start
+            elapsed = round(time.time() - upload_start, 2)
             logger.info("上传处理完成", {
-                'batch_id': batch_id,
+                '批次号': batch_id,
                 'success': success_flag,
-                'time_elapsed': elapsed,
-                'initial_row_count': initial_row_count
+                '耗时': elapsed,
+                '数据行': initial_row_count
             })
 
     def _insert_data(
@@ -909,17 +1007,16 @@ class MySQLUploader:
             batch_id: Optional[str] = None
     ):
         """
-        插入数据到表中
-
-        参数:
-        db_name: 数据库名
-        table_name: 表名
-        data: 要插入的数据列表
-        set_typ: 列名和数据类型字典 {列名: 数据类型}
-        check_duplicate: 是否检查重复
-        duplicate_columns: 用于检查重复的列(为空时检查所有列)
-        batch_size: 批量插入大小
-        batch_id: 批次ID用于日志追踪
+        实际执行数据插入的方法
+
+        :param db_name: 数据库名
+        :param table_name: 表名
+        :param data: 要插入的数据列表
+        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
+        :param check_duplicate: 是否检查重复数据，默认为False
+        :param duplicate_columns: 用于检查重复的列，可选
+        :param batch_size: 批量插入大小，默认为1000
+        :param batch_id: 批次ID用于日志追踪，可选
         """
         if not data:
             return
@@ -998,13 +1095,13 @@ class MySQLUploader:
 
                             # 记录失败行详细信息
                             error_details = {
-                                'batch_id': batch_id,
+                                '批次号': batch_id,
                                 'database': db_name,
                                 'table': table_name,
                                 'error_type': type(e).__name__,
                                 'error_message': str(e),
                                 'column_types': set_typ,
-                                'duplicate_check': check_duplicate,
+                                '是否排重': check_duplicate,
                                 'duplicate_columns': duplicate_columns
                             }
                             logger.error(f"单行插入失败: {error_details}")
@@ -1019,28 +1116,31 @@ class MySQLUploader:
                     else:
                         total_inserted += successful_rows
 
-                    batch_elapsed = time.time() - batch_start
+                    batch_elapsed = round(time.time() - batch_start, 2)
                     batch_info = {
-                        'batch_id': batch_id,
+                        '批次号': batch_id,
                         'batch_index': i // batch_size + 1,
                         'total_batches': (len(data) + batch_size - 1) // batch_size,
                         'batch_size': len(batch),
                         'successful_rows': successful_rows,
                         'failed_rows': len(batch) - successful_rows,
-                        'time_elapsed': batch_elapsed,
+                        '耗时': batch_elapsed,
                         'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
                     }
                     logger.debug(f"批次处理完成 {batch_info}")
 
         logger.info("数据插入完成", {
-            'total_rows': len(data),
-            'inserted_rows': total_inserted,
-            'skipped_rows': total_skipped,
-            'failed_rows': total_failed
+            '总数据行': len(data),
+            '插入行数': total_inserted,
+            '跳过行数': total_skipped,
+            '失败行数': total_failed
         })
 
     def close(self):
-        """关闭连接池并记录最终指标"""
+        """
+        关闭连接池并清理资源
+        :raises: 可能抛出关闭连接时的异常
+        """
         close_start = time.time()
 
         try:
@@ -1057,18 +1157,22 @@ class MySQLUploader:
 
                 elapsed = round(time.time() - close_start, 2)
                 logger.info("连接池已关闭", {
-                    'close_time_elapsed': elapsed
+                    '耗时': elapsed
                 })
         except Exception as e:
             elapsed = round(time.time() - close_start, 2)
             logger.error("关闭连接池失败", {
                 'error': str(e),
-                'close_time_elapsed': elapsed
+                '耗时': elapsed
             })
             raise
 
     def _check_pool_health(self):
-        """定期检查连接池健康状态"""
+        """
+        检查连接池健康状态
+
+        :return: 连接池健康返回True，否则返回False
+        """
         try:
             conn = self.pool.connection()
             conn.ping(reconnect=True)
@@ -1093,10 +1197,10 @@ class MySQLUploader:
                         if attempt < max_retries - 1:
                             time.sleep(delay * (attempt + 1))
                             continue
-                        raise MySQLUploaderError(f"操作重试{max_retries}次后失败") from e
+                        raise logger.error(f"操作重试{max_retries}次后失败")
                     except Exception as e:
-                        raise MySQLUploaderError(f"操作失败: {str(e)}") from e
-                raise last_exception if last_exception else MySQLUploaderError("未知错误")
+                        raise logger.error(f"操作失败: {str(e)}")
+                raise last_exception if last_exception else logger.error("操作重试失败，未知错误")
 
             return wrapper
 

{mdbq-3.9.7.dist-info → mdbq-3.9.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 3.9.7
+Version: 3.9.9
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.9.7.dist-info → mdbq-3.9.9.dist-info}/RECORD

@@ -1,18 +1,18 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=-IZp-xcG1aWVfit9XDMRHuI2-WEcYyXeULDeK2w-mPI,17
+mdbq/__version__.py,sha256=Z38j4uvZuqpFYiUEq0FTd82-1Y90RoVwpNEDWVHNTkk,17
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/optimize.py,sha256=2oalzD9weZhDclUC22OLxYa8Zj7KnmsGUoUau_Jlyc4,19796
 mdbq/aggregation/query_data.py,sha256=5_OzjGR5Sq00q-EgAYmSE5V9i4Solw9y4hkldl4mvt8,179808
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
-mdbq/log/mylogger.py,sha256=T1s94-2ERWjFkzSKJKH0EgtqWdy0DE_OGb5-Ds5vYgk,22596
+mdbq/log/mylogger.py,sha256=jHCVO7KPQrg2kcCaIrakHivZmFBJyy-24sIn2rsbK4Y,24440
 mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=sd5R-6Y00yGQ2PFTW3jkXPvJ-_OFEQZCjXM99nRHa8Q,21670
+mdbq/mysql/deduplicator.py,sha256=brhX3eyE8-kn3nAYweKfBbAkXiNcyw_pL4CTyPqmPBg,21983
 mdbq/mysql/mysql.py,sha256=jTcizvUtRdwMhWK2i_LA9yDPmcifLjUzVhwTbC3wfJk,119785
 mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
-mdbq/mysql/uploader.py,sha256=tb84-rJtrwla7SeBXL9EWzn0vIKnsE_9FebGNIoKrmU,45043
+mdbq/mysql/uploader.py,sha256=mIgUnV7MwIkrbG-dchMkMzWo_N-XrQROLWTGGGuD_ts,49171
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
@@ -25,7 +25,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=OhyEv1VyAKTOHjLDM37iNDQeRg5OnrNoKODoG2VxHes,19806
-mdbq-3.9.7.dist-info/METADATA,sha256=yX7vEbqUQIMoaJXh6VGpWKyYa5Ge0-ePKoxET8Y6LBM,363
-mdbq-3.9.7.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-3.9.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.9.7.dist-info/RECORD,,
+mdbq-3.9.9.dist-info/METADATA,sha256=F6RAyI8aGmpT-VLwVeY7jw13qemIce-PMH2Ri335GAE,363
+mdbq-3.9.9.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-3.9.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.9.9.dist-info/RECORD,,