mdbq 3.9.6__py3-none-any.whl → 3.9.7__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -12,6 +12,7 @@ import os
 import logging
 import logging.handlers
 from mdbq.other import otk
+from mdbq.log import mylogger
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
 import json
@@ -27,7 +28,18 @@ warnings.filterwarnings('ignore')
 建表流程:
 建表规范:
 """
-logger = logging.getLogger(__name__)
+logger = mylogger.MyLogger(
+    name='mysql',
+    logging_mode='both',
+    log_level='info',
+    log_file='mysql.log',
+    log_format='json',
+    max_log_size=50,
+    backup_count=5,
+    enable_async=False,  # 是否启用异步日志
+    sample_rate=0.5,  # 采样50%的DEBUG/INFO日志
+    sensitive_fields=[],  #  敏感字段列表
+)
 
 
 def count_decimal_places(num_str):
@@ -1136,19 +1148,13 @@ class MySQLUploader:
             port: int = 3306,
             charset: str = 'utf8mb4',
             collation: str = 'utf8mb4_0900_ai_ci',  # utf8mb4_0900_ai_ci: 该排序规则对大小写不敏感, utf8mb4_0900_as_cs/utf8mb4_bin: 对大小写敏感
-            logging_mode: str = 'console',  # 'both'(控制台+文件), 'console'(仅控制台), 'file'(仅文件), 'none'(禁用)
-            log_level: str = 'INFO',  # 默认日志级别
-            log_file: str = 'mysql_upload.log',  # 日志文件路径
-            max_log_size: int = 50,  # 日志文件大小(MB)
-            backup_count: int = 5,  # 保留的日志文件数量
             max_retries: int = 10,
             retry_interval: int = 10,
             pool_size: int = 5,
             connect_timeout: int = 10,
             read_timeout: int = 30,
             write_timeout: int = 30,
-            ssl: Optional[Dict] = None,
-            enable_metrics: bool = True  # 是否启用性能指标收集
+            ssl: Optional[Dict] = None
     ):
         """
         :param username: 数据库用户名
@@ -1157,11 +1163,7 @@ class MySQLUploader:
         :param port: 数据库端口，默认为3306
         :param charset: 字符集，默认为utf8mb4
         :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
-        :param logging_mode: 日志模式，可选 'both'(控制台+文件), 'console'(仅控制台), 'file'(仅文件), 'none'(禁用)
-        :param log_level: 日志级别，默认为INFO
-        :param log_file: 日志文件路径
-        :param max_log_size: 日志文件最大大小(MB)，默认为50
-        :param backup_count: 保留的日志备份数量，默认为5
+
         :param max_retries: 最大重试次数，默认为10
         :param retry_interval: 重试间隔(秒)，默认为10
         :param pool_size: 连接池大小，默认为5
@@ -1169,7 +1171,6 @@ class MySQLUploader:
         :param read_timeout: 读取超时(秒)，默认为30
         :param write_timeout: 写入超时(秒)，默认为30
         :param ssl: SSL配置字典，默认为None
-        :param enable_metrics: 是否启用性能指标收集，默认为True
         """
         self.username = username
         self.password = password
@@ -1186,178 +1187,12 @@ class MySQLUploader:
         self.ssl = ssl
         self._prepared_statements = StatementCache(maxsize=100)
         self._max_cached_statements = 100
-        self.enable_metrics = enable_metrics
-        self.metrics = {
-            'total_uploads': 0,
-            'successful_uploads': 0,
-            'failed_uploads': 0,
-            'total_rows': 0,
-            'successful_rows': 0,
-            'failed_rows': 0,
-            'total_retries': 0,
-            'total_execution_time': 0.0,
-            'connection_usage': [],
-            'memory_usage': [],
-            'cpu_usage': []
-        }
-        self._last_metrics_time = 0
-        self._metrics_cache = {}  # 缓存最近一次的系统指标
-        self.metrics_interval = 30  # 指标采集频率控制
-        self._table_metadata_cache = {}  # 元信息缓存
-        self.metadata_cache_ttl = 300  # 元信息缓存频率控制
-
-        # 初始化日志系统
-        self._init_logging(logging_mode, log_level, log_file, max_log_size, backup_count)
+        self._table_metadata_cache = {}
+        self.metadata_cache_ttl = 300  # 5分钟缓存时间
 
         # 创建连接池
         self.pool = self._create_connection_pool()
 
-    def _init_logging(
-        self,
-        logging_mode: str,
-        log_level: str,
-        log_file: str,
-        max_log_size: int,
-        backup_count: int
-    ):
-        """初始化结构化日志配置"""
-        if logging_mode.lower() == 'none':
-            self.logger = None
-            return
-
-        valid_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
-        level = log_level.upper() if log_level.upper() in valid_levels else 'INFO'
-
-        # 创建格式化器 - 使用结构化JSON格式
-        class StructuredFormatter(logging.Formatter):
-            def format(self, record):
-                log_data = {
-                    'time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
-                    'level': record.levelname,
-                    'message': record.getMessage(),
-                    # 'logger': record.name,
-                    'module': record.module,
-                    'line': record.lineno,
-                    # 'process': record.process
-                }
-
-                # 添加异常信息
-                if record.exc_info:
-                    log_data['exception'] = self.formatException(record.exc_info)
-
-                # 过滤敏感信息
-                if hasattr(record, 'password'):
-                    log_data['message'] = log_data['message'].replace(self.password, '***')
-
-                return json.dumps(log_data, ensure_ascii=False)
-
-        # 创建日志记录器
-        self.logger = logging.getLogger('upload')
-        self.logger.setLevel(level)
-
-        # 防止重复添加handler
-        if self.logger.handlers:
-            for handler in self.logger.handlers[:]:
-                self.logger.removeHandler(handler)
-
-        formatter = StructuredFormatter()
-        mode = logging_mode.lower()
-
-        # 根据模式添加相应的handler
-        if mode in ('both', 'console'):
-            console_handler = logging.StreamHandler()
-            console_handler.setFormatter(formatter)
-            self.logger.addHandler(console_handler)
-
-        if mode in ('both', 'file'):
-            file_handler = logging.handlers.RotatingFileHandler(
-                filename=log_file,
-                maxBytes=max_log_size * 1024 * 1024,
-                backupCount=backup_count,
-                encoding='utf-8'
-            )
-            file_handler.setFormatter(formatter)
-            self.logger.addHandler(file_handler)
-
-    def _record_metrics(self, metric_name: str, value: Any = 1, is_timing: bool = False):
-        """记录性能指标"""
-        if not self.enable_metrics:
-            return
-
-        # 对于频繁调用的指标，使用更高效的数据结构
-        if metric_name in ('total_uploads', 'successful_uploads', 'failed_uploads'):
-            self.metrics[metric_name] = self.metrics.get(metric_name, 0) + value
-            return
-
-        if metric_name not in self.metrics:
-            self.metrics[metric_name] = []
-
-        if is_timing:
-            # 如果是时间指标，记录时间戳和值
-            self.metrics[metric_name].append({
-                'timestamp': datetime.datetime.now().isoformat(),
-                'value': value
-            })
-        else:
-            # 其他指标直接累加
-            if isinstance(self.metrics[metric_name], (int, float)):
-                self.metrics[metric_name] += value
-            elif isinstance(self.metrics[metric_name], list):
-                self.metrics[metric_name].append({
-                    'timestamp': datetime.datetime.now().isoformat(),
-                    'value': value
-                })
-
-    def _get_system_metrics(self):
-        """获取系统资源使用指标"""
-        if not self.enable_metrics:
-            return {}
-
-        metrics = {
-            'memory': psutil.virtual_memory().percent,
-            'cpu': psutil.cpu_percent(),
-        }
-
-        # 更安全的连接数获取方式
-        if hasattr(self, 'pool') and self.pool is not None:
-            try:
-                # 对于不同的连接池实现可能有不同的属性名
-                if hasattr(self.pool, '_connections'):
-                    connections = self.pool._connections
-                    metrics['connections'] = len(connections) if hasattr(connections, '__len__') else 0
-                else:
-                    metrics['connections'] = 0
-            except Exception:
-                metrics['connections'] = 0
-        else:
-            metrics['connections'] = 0
-
-        return metrics
-
-    def _log_with_metrics(self, level: str, message: str, extra: Optional[Dict] = None):
-        """日志记录"""
-        if not self.logger:
-            return
-
-        if len(message) > 500:
-            message = message[:500] + '...'
-
-        now = time.time()
-        if now - self._last_metrics_time > self.metrics_interval:
-            self._metrics_cache = self._get_system_metrics()
-            # 使用缓存的指标
-            log_extra = {'metrics': self._metrics_cache}
-            self._last_metrics_time = now
-        else:
-            # 记录系统指标
-            metrics = self._get_system_metrics()
-            log_extra = {'metrics': metrics}
-
-        if extra:
-            log_extra.update(extra)
-
-        getattr(self.logger, level.lower())(message, extra={'extra_data': log_extra})
-
     def _create_connection_pool(self) -> PooledDB:
         """创建数据库连接池"""
         if hasattr(self, 'pool') and self.pool is not None and self._check_pool_health():
@@ -1386,7 +1221,7 @@ class MySQLUploader:
             required_keys = {'ca', 'cert', 'key'}
             if not all(k in self.ssl for k in required_keys):
                 error_msg = "SSL配置必须包含ca、cert和key"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
             pool_params['ssl'] = {
                 'ca': self.ssl['ca'],
@@ -1398,17 +1233,15 @@ class MySQLUploader:
         try:
             pool = PooledDB(**pool_params)
             elapsed = time.time() - start_time
-            self._record_metrics('connection_pool_creation_time', elapsed, is_timing=True)
-            self._log_with_metrics('info', "连接池创建成功", {
+            logger.info("连接池创建成功", {
                 'pool_size': self.pool_size,
                 'time_elapsed': elapsed
             })
             return pool
         except Exception as e:
             elapsed = time.time() - start_time
-            self._record_metrics('connection_pool_failures', 1)
             self.pool = None
-            self._log_with_metrics('error', "连接池创建失败", {
+            logger.error("连接池创建失败", {
                 'error': str(e),
                 'time_elapsed': elapsed
             })
@@ -1421,7 +1254,7 @@ class MySQLUploader:
             start_time = time.time()
             operation = func.__name__
 
-            self._log_with_metrics('debug', f"开始执行操作: {operation}", {
+            logger.debug(f"开始执行操作: {operation}", {
                 'attempt': 1,
                 'max_retries': self.max_retries
             })
@@ -1432,14 +1265,13 @@ class MySQLUploader:
                     elapsed = time.time() - start_time
 
                     if attempt > 0:
-                        self._record_metrics('total_retries', attempt)
-                        self._log_with_metrics('info', "操作成功(重试后)", {
+                        logger.info("操作成功(重试后)", {
                             'operation': operation,
                             'attempts': attempt + 1,
                             'time_elapsed': elapsed
                         })
                     else:
-                        self._log_with_metrics('debug', "操作成功", {
+                        logger.debug("操作成功", {
                             'operation': operation,
                             'time_elapsed': elapsed
                         })
@@ -1448,7 +1280,6 @@ class MySQLUploader:
 
                 except (pymysql.OperationalError, pymysql.err.MySQLError) as e:
                     last_exception = e
-                    self._record_metrics('database_errors', 1)
 
                     # 记录详细的MySQL错误信息
                     error_details = {
@@ -1462,26 +1293,25 @@ class MySQLUploader:
                     if attempt < self.max_retries - 1:
                         wait_time = self.retry_interval * (attempt + 1)
                         error_details['wait_time'] = wait_time
-                        self._log_with_metrics('warning', f"数据库操作失败，准备重试 {error_details}", )
+                        logger.warning(f"数据库操作失败，准备重试 {error_details}", )
                         time.sleep(wait_time)
 
                         # 尝试重新连接
                         try:
                             self.pool = self._create_connection_pool()
-                            self._log_with_metrics('info', "成功重新建立数据库连接")
+                            logger.info("成功重新建立数据库连接")
                         except Exception as reconnect_error:
-                            self._log_with_metrics('error', "重连失败", {
+                            logger.error("重连失败", {
                                 'error': str(reconnect_error)
                             })
                     else:
                         elapsed = time.time() - start_time
                         error_details['time_elapsed'] = elapsed
-                        self._log_with_metrics('error', f"操作最终失败 {error_details}")
+                        logger.error(f"操作最终失败 {error_details}")
 
                 except pymysql.IntegrityError as e:
                     elapsed = time.time() - start_time
-                    self._record_metrics('integrity_errors', 1)
-                    self._log_with_metrics('error', "完整性约束错误", {
+                    logger.error("完整性约束错误", {
                         'operation': operation,
                         'time_elapsed': elapsed,
                         'error_code': e.args[0] if e.args else None,
@@ -1492,8 +1322,7 @@ class MySQLUploader:
                 except Exception as e:
                     last_exception = e
                     elapsed = time.time() - start_time
-                    self._record_metrics('unexpected_errors', 1)
-                    self._log_with_metrics('error', "发生意外错误", {
+                    logger.error("发生意外错误", {
                         'operation': operation,
                         'time_elapsed': elapsed,
                         'error_type': type(e).__name__,
@@ -1510,10 +1339,10 @@ class MySQLUploader:
         """从连接池获取连接"""
         try:
             conn = self.pool.connection()
-            self._log_with_metrics('debug', "获取数据库连接")
+            logger.debug("获取数据库连接")
             return conn
         except Exception as e:
-            self._log_with_metrics("error", f'{e}')
+            logger.error(f'{e}')
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     def _check_database_exists(self, db_name: str) -> bool:
@@ -1526,10 +1355,10 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql, (db_name,))
                     exists = bool(cursor.fetchone())
-                    self._log_with_metrics('debug', f"{db_name} 数据库已存在: {exists}")
+                    logger.debug(f"{db_name} 数据库已存在: {exists}")
                     return exists
         except Exception as e:
-            self._log_with_metrics('error', f"检查数据库是否存在时出错: {str(e)}")
+            logger.error(f"检查数据库是否存在时出错: {str(e)}")
             raise
 
     def _create_database(self, db_name: str):
@@ -1542,9 +1371,9 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
                 conn.commit()
-                self._log_with_metrics('info', f"{db_name} 数据库已创建")
+                logger.info(f"{db_name} 数据库已创建")
         except Exception as e:
-            self._log_with_metrics('error', f"{db_name}: 无法创建数据库 {str(e)}")
+            logger.error(f"{db_name}: 无法创建数据库 {str(e)}")
             conn.rollback()
             raise
 
@@ -1567,7 +1396,7 @@ class MySQLUploader:
                 date_obj = self._validate_datetime(date_value, True)
             except ValueError:
                 error_msg = f"无效的日期格式1: {date_value}"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
 
         if partition_by == 'year':
@@ -1576,7 +1405,7 @@ class MySQLUploader:
             return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
         else:
             error_msg = "partition_by must be 'year' or 'month'"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
     def _validate_identifier(self, identifier: str) -> str:
@@ -1590,14 +1419,14 @@ class MySQLUploader:
         """
         if not identifier or not isinstance(identifier, str):
             error_msg = f"无效的标识符: {identifier}"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 移除非法字符，只保留字母、数字、下划线和美元符号
         cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '', identifier)
         if not cleaned:
             error_msg = f"无法清理异常标识符: {identifier}"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 检查是否为MySQL保留字
@@ -1606,7 +1435,7 @@ class MySQLUploader:
             'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
         }
         if cleaned.lower() in mysql_keywords:
-            self._log_with_metrics('debug', f"存在MySQL保留字: {cleaned}")
+            logger.debug(f"存在MySQL保留字: {cleaned}")
             return f"`{cleaned}`"
 
         return cleaned
@@ -1633,7 +1462,7 @@ class MySQLUploader:
                     cursor.execute(sql, (db_name, table_name))
                     result = bool(cursor.fetchone())
         except Exception as e:
-            self._log_with_metrics('error', f"检查数据表是否存在时发生未知错误: {e}", )
+            logger.error(f"检查数据表是否存在时发生未知错误: {e}", )
             raise
 
         # 执行查询并缓存结果
@@ -1665,7 +1494,7 @@ class MySQLUploader:
 
         if not set_typ:
             error_msg = "No columns specified for table creation"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 构建列定义SQL
@@ -1710,7 +1539,7 @@ class MySQLUploader:
             with self._get_connection() as conn:
                 with conn.cursor() as cursor:
                     cursor.execute(sql)
-                    self._log_with_metrics('info', f"{db_name}.{table_name}: 数据表已创建")
+                    logger.info(f"{db_name}.{table_name}: 数据表已创建")
 
                 # 添加普通索引
                 index_statements = []
@@ -1736,13 +1565,13 @@ class MySQLUploader:
                     with conn.cursor() as cursor:
                         for stmt in index_statements:
                             cursor.execute(stmt)
-                            self._log_with_metrics('debug', f"Executed index statement: {stmt}", )
+                            logger.debug(f"Executed index statement: {stmt}", )
 
                 conn.commit()
-                self._log_with_metrics('info', f"{db_name}.{table_name}: 索引已添加")
+                logger.info(f"{db_name}.{table_name}: 索引已添加")
 
         except Exception as e:
-            self._log_with_metrics('error', f"{db_name}.{table_name}: 建表失败: {str(e)}")
+            logger.error(f"{db_name}.{table_name}: 建表失败: {str(e)}")
             conn.rollback()
             raise
 
@@ -1812,7 +1641,7 @@ class MySQLUploader:
                 return value
         except (ValueError, TypeError) as e:
             error_msg = f"数据类型转换异常 {value} to type {column_type}: {str(e)}"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
     def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
@@ -1831,10 +1660,10 @@ class MySQLUploader:
                 with conn.cursor() as cursor:
                     cursor.execute(sql, (db_name, table_name))
                     set_typ = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
-                    self._log_with_metrics('debug', f"{db_name}.{table_name}: 获取表的列信息: {set_typ}")
+                    logger.debug(f"{db_name}.{table_name}: 获取表的列信息: {set_typ}")
                     return set_typ
         except Exception as e:
-            self._log_with_metrics('error', f"无法获取表列信息: {str(e)}")
+            logger.error(f"无法获取表列信息: {str(e)}")
             raise
 
     def _upload_to_table(
@@ -1860,21 +1689,21 @@ class MySQLUploader:
                                    allow_null=allow_null)
             else:
                 error_msg = f"数据表不存在: '{db_name}.{table_name}'"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
 
         # 获取表结构并验证
         table_columns = self._get_table_columns(db_name, table_name)
         if not table_columns:
             error_msg = f"获取列失败 '{db_name}.{table_name}'"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 验证数据列与表列匹配
         for col in set_typ:
             if col not in table_columns:
                 error_msg = f"列不存在: '{col}' -> '{db_name}.{table_name}'"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
 
         # 插入数据
@@ -1957,7 +1786,7 @@ class MySQLUploader:
                 data.columns = [col.lower() for col in data.columns]
                 data = data.replace({pd.NA: None}).to_dict('records')
             except Exception as e:
-                self._log_with_metrics("error", f"数据转字典时发生错误: {e}", )
+                logger.error(f"数据转字典时发生错误: {e}", )
                 raise ValueError(f"数据转字典时发生错误: {e}")
         elif isinstance(data, dict):
             data = [{k.lower(): v for k, v in data.items()}]
@@ -1966,7 +1795,7 @@ class MySQLUploader:
             data = [{k.lower(): v for k, v in item.items()} for item in data]
         else:
             error_msg = "数据结构必须是字典、列表、字典列表或dataframe"
-            self._log_with_metrics('error', error_msg)
+            logger.error(error_msg)
             raise ValueError(error_msg)
 
         # 将set_typ的键转为小写
@@ -1988,11 +1817,11 @@ class MySQLUploader:
                 if sample_values:
                     inferred_type = self._infer_data_type(sample_values[0])
                     filtered_set_typ[col] = inferred_type
-                    self._log_with_metrics('debug', f"自动推断列'{col}'的数据类型为: {inferred_type}")
+                    logger.debug(f"自动推断列'{col}'的数据类型为: {inferred_type}")
                 else:
                     # 没有样本值，使用默认类型
                     filtered_set_typ[col] = 'VARCHAR(255)'
-                    self._log_with_metrics('debug', f"为列'{col}'使用默认数据类型: VARCHAR(255)")
+                    logger.debug(f"为列'{col}'使用默认数据类型: VARCHAR(255)")
 
         prepared_data = []
         for row_idx, row in enumerate(data, 1):
@@ -2005,7 +1834,7 @@ class MySQLUploader:
                 if col_name not in row:
                     if not allow_null:
                         error_msg = f"Row {row_idx}: Missing required column '{col_name}' in data"
-                        self._log_with_metrics('error', error_msg)
+                        logger.error(error_msg)
                         raise ValueError(error_msg)
                     prepared_row[col_name] = None
                 else:
@@ -2013,11 +1842,11 @@ class MySQLUploader:
                         prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name])
                     except ValueError as e:
                         error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
-                        self._log_with_metrics('error', error_msg)
+                        logger.error(error_msg)
                         raise ValueError(error_msg)
             prepared_data.append(prepared_row)
 
-        self._log_with_metrics('debug', f"已准备 {len(prepared_data)} 行数据")
+        logger.debug(f"已准备 {len(prepared_data)} 行数据")
         return prepared_data, filtered_set_typ
 
     def upload_data(
@@ -2039,14 +1868,12 @@ class MySQLUploader:
         上传数据到数据库
         """
         upload_start = time.time()
-        self._record_metrics('total_uploads', 1)
         initial_row_count = len(data) if hasattr(data, '__len__') else 1
-        self.metrics['total_rows'] += len(data) if hasattr(data, '__len__') else 1
 
         batch_id = f"batch_{int(time.time() * 1000)}"
         success_flag = False
 
-        self._log_with_metrics('info', "开始上传数据", {
+        logger.info("开始上传数据", {
             'batch_id': batch_id,
             'database': db_name,
             'table': table_name,
@@ -2060,12 +1887,12 @@ class MySQLUploader:
             # 验证参数
             if not set_typ:
                 error_msg = "列的数据类型缺失"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
 
             if partition_by and partition_by not in ['year', 'month']:
                 error_msg = "分表方式必须是 'year' 或 'month'"
-                self._log_with_metrics('error', error_msg)
+                logger.error(error_msg)
                 raise ValueError(error_msg)
 
             # 准备数据
@@ -2077,7 +1904,7 @@ class MySQLUploader:
                     self._create_database(db_name)
                 else:
                     error_msg = f"数据库不存在: '{db_name}'"
-                    self._log_with_metrics('error', error_msg)
+                    logger.error(error_msg)
                     raise ValueError(error_msg)
 
             # 处理分表逻辑
@@ -2087,7 +1914,7 @@ class MySQLUploader:
                     try:
                         if partition_date_column not in row:
                             error_msg = f"异常缺失列 '{partition_date_column}'"
-                            self._log_with_metrics('error', error_msg)
+                            logger.error(error_msg)
                             continue  # 跳过当前行
 
                         part_table = self._get_partition_table_name(
@@ -2099,7 +1926,7 @@ class MySQLUploader:
                             partitioned_data[part_table] = []
                         partitioned_data[part_table].append(row)
                     except Exception as e:
-                        self._log_with_metrics('error', "分表处理失败", {
+                        logger.error("分表处理失败", {
                             'row_data': row,
                             'error': str(e)
                         })
@@ -2115,7 +1942,7 @@ class MySQLUploader:
                             indexes, batch_id
                         )
                     except Exception as e:
-                        self._log_with_metrics('error', "分表上传失败", {
+                        logger.error("分表上传失败", {
                             'partition_table': part_table,
                             'error': str(e)
                         })
@@ -2132,25 +1959,17 @@ class MySQLUploader:
             success_flag = True
 
         except Exception as e:
-            self._log_with_metrics('error', "上传过程中发生全局错误", {
+            logger.error("上传过程中发生全局错误", {
                 'error': str(e),
                 'error_type': type(e).__name__
             })
         finally:
             elapsed = time.time() - upload_start
-            self._record_metrics('upload_execution_time', elapsed, is_timing=True)
-
-            if success_flag:
-                self._record_metrics('successful_uploads', 1)
-            else:
-                self._record_metrics('failed_uploads', 1)
-
-            self._log_with_metrics('info', "上传处理完成", {
+            logger.info("上传处理完成", {
                 'batch_id': batch_id,
                 'success': success_flag,
                 'time_elapsed': elapsed,
-                'initial_row_count': initial_row_count,
-                'processed_rows': self.metrics['successful_rows'] + self.metrics['failed_rows']
+                'initial_row_count': initial_row_count
             })
 
     def _insert_data(
@@ -2263,7 +2082,7 @@ class MySQLUploader:
                                 'duplicate_check': check_duplicate,
                                 'duplicate_columns': duplicate_columns
                             }
-                            self._log_with_metrics('error', f"单行插入失败: {error_details}")
+                            logger.error(f"单行插入失败: {error_details}")
                             continue  # 跳过当前行，继续处理下一行
 
                     # 更新统计信息
@@ -2276,8 +2095,6 @@ class MySQLUploader:
                         total_inserted += successful_rows
 
                     batch_elapsed = time.time() - batch_start
-                    self._record_metrics('batch_execution_time', batch_elapsed, is_timing=True)
-
                     batch_info = {
                         'batch_id': batch_id,
                         'batch_index': i // batch_size + 1,
@@ -2288,68 +2105,42 @@ class MySQLUploader:
                         'time_elapsed': batch_elapsed,
                         'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
                     }
-                    self._log_with_metrics('debug', f"批次处理完成 {batch_info}")
+                    logger.debug(f"批次处理完成 {batch_info}")
 
-        # 更新全局指标
-        self.metrics['failed_rows'] += total_failed
-        self._log_with_metrics('info', "数据插入完成", {
+        logger.info("数据插入完成", {
             'total_rows': len(data),
             'inserted_rows': total_inserted,
             'skipped_rows': total_skipped,
             'failed_rows': total_failed
         })
 
-    def get_metrics(self) -> Dict:
-        """获取当前性能指标"""
-        metrics = self.metrics.copy()
-
-        # 添加当前系统指标
-        metrics.update({
-            'current_time': datetime.datetime.now().isoformat(),
-            'system': self._get_system_metrics(),
-            'connection_pool': {
-                'size': self.pool_size,
-                'active': len(self.pool._connections) if hasattr(self.pool, '_connections') else 0
-            }
-        })
-
-        return metrics
-
     def close(self):
         """关闭连接池并记录最终指标"""
         close_start = time.time()
 
         try:
             if hasattr(self, 'pool') and self.pool is not None:
-                # 记录关闭前的连接池状态
-                active_connections = self._get_system_metrics().get('connections', 0)
-
                 # 更安全的关闭方式
                 try:
                     self.pool.close()
                 except Exception as e:
-                    self._log_with_metrics('warning', "关闭连接池时出错", {
+                    logger.warning("关闭连接池时出错", {
                         'error': str(e)
                     })
 
                 self.pool = None
 
-                elapsed = time.time() - close_start
-                self._log_with_metrics('info', "连接池已关闭", {
-                    'active_connections_before_close': active_connections,
+                elapsed = round(time.time() - close_start, 2)
+                logger.info("连接池已关闭", {
                     'close_time_elapsed': elapsed
                 })
         except Exception as e:
-            elapsed = time.time() - close_start
-            self._log_with_metrics('error', "关闭连接池失败", {
+            elapsed = round(time.time() - close_start, 2)
+            logger.error("关闭连接池失败", {
                 'error': str(e),
                 'close_time_elapsed': elapsed
             })
             raise
-        finally:
-            # 记录最终性能指标
-            if hasattr(self, 'logger') and self.logger and self.enable_metrics:
-                self._log_with_metrics('debug', "最终性能指标", self.get_metrics())
 
     def _check_pool_health(self):
         """定期检查连接池健康状态"""
@@ -2358,8 +2149,8 @@ class MySQLUploader:
             conn.ping(reconnect=True)
             conn.close()
             return True
-        except Exception:
-            self._log_with_metrics('warning', "连接池健康检查失败", {
+        except Exception as e:
+            logger.warning("连接池健康检查失败", {
                 'error': str(e)
             })
             return False
@@ -2428,9 +2219,6 @@ class MySQLDeduplicator:
             max_workers: int = 1,
             batch_size: int = 1000,
             skip_system_dbs: bool = True,
-            logging_mode: str = 'console',
-            log_level: str = 'INFO',
-            log_file: str = 'mysql_deduplicate.log',
             max_retries: int = 3,
             retry_interval: int = 5,
             pool_size: int = 5
@@ -2446,9 +2234,6 @@ class MySQLDeduplicator:
         :param max_workers: 最大工作线程数，默认为1(单线程)
         :param batch_size: 批量处理大小，默认为1000
         :param skip_system_dbs: 是否跳过系统数据库，默认为True
-        :param logging_mode: 日志模式('console', 'file', 'both', 'none')
-        :param log_level: 日志级别('DEBUG', 'INFO', 'WARNING', 'ERROR')
-        :param log_file: 日志文件路径
         :param max_retries: 最大重试次数
         :param retry_interval: 重试间隔(秒)
         :param pool_size: 连接池大小
@@ -2476,69 +2261,20 @@ class MySQLDeduplicator:
         self._lock = threading.Lock()
         self._processing_tables = set()  # 正在处理的表集合
 
-        # 初始化日志
-        self._init_logging(logging_mode, log_level, log_file)
-
         # 系统数据库列表
         self.SYSTEM_DATABASES = {
             'information_schema', 'mysql',
             'performance_schema', 'sys'
         }
 
-    def _init_logging(
-            self,
-            logging_mode: str,
-            log_level: str,
-            log_file: str
-    ):
-        """初始化日志配置"""
-        self.logger = logging.getLogger('mysql_deduplicator')
-        self.logger.setLevel(log_level.upper())
-
-        # 防止重复添加handler
-        if self.logger.handlers:
-            for handler in self.logger.handlers[:]:
-                self.logger.removeHandler(handler)
-
-        formatter = logging.Formatter(
-            '%(asctime)s - %(levelname)s - %(message)s',
-            datefmt='%Y-%m-%d %H:%M:%S'
-        )
-
-        mode = logging_mode.lower()
-        if mode in ('both', 'console'):
-            console_handler = logging.StreamHandler()
-            console_handler.setFormatter(formatter)
-            self.logger.addHandler(console_handler)
-
-        if mode in ('both', 'file'):
-            file_handler = logging.FileHandler(
-                filename=log_file,
-                encoding='utf-8'
-            )
-            file_handler.setFormatter(formatter)
-            self.logger.addHandler(file_handler)
-
-    def _log(self, level: str, message: str, extra: Optional[Dict] = None):
-        """统一的日志记录方法"""
-        if not hasattr(self.logger, level.lower()):
-            return
-
-        # 简化日志内容，避免过长
-        if len(message) > 500:
-            message = message[:500] + '...'
-
-        log_method = getattr(self.logger, level.lower())
-        log_method(message, extra=extra)
-
     def _get_connection(self):
         """从连接池获取连接"""
         try:
             conn = self.pool.connection()
-            self._log('debug', "成功获取数据库连接")
+            logger.debug("成功获取数据库连接")
             return conn
         except Exception as e:
-            self._log('error', f"获取数据库连接失败: {str(e)}")
+            logger.error(f"获取数据库连接失败: {str(e)}")
             raise ConnectionError(f"连接数据库失败: {str(e)}")
 
     @staticmethod
@@ -2555,16 +2291,14 @@ class MySQLDeduplicator:
                     last_exception = e
                     if attempt < self.max_retries:
                         wait_time = self.retry_interval * (attempt + 1)
-                        self._log('warning',
-                                  f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
-                                  {'error': str(e), 'wait_time': wait_time})
+                        logger.warning(
+                            f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
+                            {'error': str(e), 'wait_time': wait_time})
                         time.sleep(wait_time)
                         continue
                 except Exception as e:
                     last_exception = e
-                    self._log('error',
-                              f"操作失败: {str(e)}",
-                              {'error_type': type(e).__name__})
+                    logger.error(f"操作失败: {str(e)}", {'error_type': type(e).__name__})
                     break
 
             if last_exception:
@@ -2620,7 +2354,7 @@ class MySQLDeduplicator:
 
         with self._lock:
             if key in self._processing_tables:
-                self._log('debug', f"表 {key} 正在被其他线程处理，跳过")
+                logger.debug(f"表 {key} 正在被其他线程处理，跳过")
                 return False
             self._processing_tables.add(key)
             return True
@@ -2653,12 +2387,12 @@ class MySQLDeduplicator:
             return (0, 0)
 
         try:
-            self._log('info', f"开始处理表: {database}.{table}")
+            logger.info(f"开始处理表: {database}.{table}")
 
             # 获取实际列名
             all_columns = self._get_table_columns(database, table)
             if not all_columns:
-                self._log('warning', f"表 {database}.{table} 没有有效列(可能只有id列)，跳过")
+                logger.warning(f"表 {database}.{table} 没有有效列(可能只有id列)，跳过")
                 return (0, 0)
 
             # 使用指定列或所有列
@@ -2666,14 +2400,14 @@ class MySQLDeduplicator:
             invalid_columns = set(use_columns) - set(all_columns)
 
             if invalid_columns:
-                self._log('warning',
-                          f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
-                          {'invalid_columns': invalid_columns}
-                          )
+                logger.warning(
+                    f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
+                    {'invalid_columns': invalid_columns}
+                )
                 use_columns = [col for col in use_columns if col in all_columns]
 
             if not use_columns:
-                self._log('error', f"表 {database}.{table} 没有有效的去重列")
+                logger.error(f"表 {database}.{table} 没有有效的去重列")
                 return (0, 0)
 
             # 构建去重SQL
@@ -2706,31 +2440,31 @@ class MySQLDeduplicator:
                     dup_count = cursor.fetchone()['cnt']
 
                     if dup_count == 0:
-                        self._log('info', f"表 {database}.{table} 没有重复数据")
+                        logger.info(f"表 {database}.{table} 没有重复数据")
                         cursor.execute(drop_temp_sql)
                         conn.commit()
                         return (0, 0)
 
-                    self._log('info',
-                              f"表 {database}.{table} 发现 {dup_count} 组重复数据",
-                              {'columns': use_columns}
-                              )
+                    logger.info(
+                        f"表 {database}.{table} 发现 {dup_count} 组重复数据",
+                        {'columns': use_columns}
+                    )
 
                     if not dry_run:
                         # 执行实际删除
                         cursor.execute(delete_dup_sql)
                         affected_rows = cursor.rowcount
                         conn.commit()
-                        self._log('info',
-                                  f"表 {database}.{table} 已删除 {affected_rows} 行重复数据",
-                                  {'columns': use_columns}
-                                  )
+                        logger.info(
+                            f"表 {database}.{table} 已删除 {affected_rows} 行重复数据",
+                            {'columns': use_columns}
+                        )
                     else:
                         affected_rows = 0
-                        self._log('info',
-                                  f"[模拟运行] 表 {database}.{table} 将删除 {dup_count} 组重复数据",
-                                  {'columns': use_columns}
-                                  )
+                        logger.info(
+                            f"[模拟运行] 表 {database}.{table} 将删除 {dup_count} 组重复数据",
+                            {'columns': use_columns}
+                        )
 
                     # 清理临时表
                     cursor.execute(drop_temp_sql)
@@ -2739,10 +2473,10 @@ class MySQLDeduplicator:
                     return (dup_count, affected_rows)
 
         except Exception as e:
-            self._log('error',
-                      f"处理表 {database}.{table} 时出错: {str(e)}",
-                      {'error_type': type(e).__name__}
-                      )
+            logger.error(
+                f"处理表 {database}.{table} 时出错: {str(e)}",
+                {'error_type': type(e).__name__}
+            )
             return (0, 0)
         finally:
             self._release_table_lock(database, table)
@@ -2766,15 +2500,15 @@ class MySQLDeduplicator:
         try:
             # 检查表是否存在
             if not self._check_table_exists(database, table):
-                self._log('warning', f"表 {database}.{table} 不存在，跳过")
+                logger.warning(f"表 {database}.{table} 不存在，跳过")
                 return (0, 0)
 
             return self._deduplicate_table(database, table, columns, dry_run)
         except Exception as e:
-            self._log('error',
-                      f"处理表 {database}.{table} 时发生全局错误: {str(e)}",
-                      {'error_type': type(e).__name__}
-                      )
+            logger.error(
+                f"处理表 {database}.{table} 时发生全局错误: {str(e)}",
+                {'error_type': type(e).__name__}
+            )
             return (0, 0)
 
     def deduplicate_database(
@@ -2800,19 +2534,19 @@ class MySQLDeduplicator:
         try:
             # 检查数据库是否存在
             if not self._check_database_exists(database):
-                self._log('warning', f"数据库 {database} 不存在，跳过")
+                logger.warning(f"数据库 {database} 不存在，跳过")
                 return results
 
             # 获取要处理的表
             target_tables = tables or self._get_tables(database)
             if not target_tables:
-                self._log('info', f"数据库 {database} 中没有表，跳过")
+                logger.info(f"数据库 {database} 中没有表，跳过")
                 return results
 
-            self._log('info',
-                      f"开始处理数据库 {database} 中的 {len(target_tables)} 张表",
-                      {'tables': target_tables}
-                      )
+            logger.info(
+                f"开始处理数据库 {database} 中的 {len(target_tables)} 张表",
+                {'tables': target_tables}
+            )
 
             if parallel and self.max_workers > 1:
                 # 并行处理
@@ -2833,10 +2567,10 @@ class MySQLDeduplicator:
                             dup_count, affected_rows = future.result()
                             results[table] = (dup_count, affected_rows)
                         except Exception as e:
-                            self._log('error',
-                                      f"处理表 {database}.{table} 时出错: {str(e)}",
-                                      {'error_type': type(e).__name__}
-                                      )
+                            logger.error(
+                                f"处理表 {database}.{table} 时出错: {str(e)}",
+                                {'error_type': type(e).__name__}
+                            )
                             results[table] = (0, 0)
             else:
                 # 串行处理
@@ -2851,18 +2585,15 @@ class MySQLDeduplicator:
             total_dup = sum(r[0] for r in results.values())
             total_del = sum(r[1] for r in results.values())
 
-            self._log('info',
-                      f"数据库 {database} 处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                      {'results': results}
-                      )
+            logger.info(
+                f"数据库 {database} 处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
+                {'results': results}
+            )
 
             return results
 
         except Exception as e:
-            self._log('error',
-                      f"处理数据库 {database} 时发生全局错误: {str(e)}",
-                      {'error_type': type(e).__name__}
-                      )
+            logger.error(f"处理数据库 {database} 时发生全局错误: {str(e)}", {'error_type': type(e).__name__})
             return results
 
     def deduplicate_all(
@@ -2889,13 +2620,10 @@ class MySQLDeduplicator:
             # 获取要处理的数据库
             target_dbs = databases or self._get_databases()
             if not target_dbs:
-                self._log('warning', "没有可处理的数据库")
+                logger.warning("没有可处理的数据库")
                 return all_results
 
-            self._log('info',
-                      f"开始处理 {len(target_dbs)} 个数据库",
-                      {'databases': target_dbs}
-                      )
+            logger.info(f"开始处理 {len(target_dbs)} 个数据库", {'databases': target_dbs})
 
             if parallel and self.max_workers > 1:
                 # 并行处理数据库
@@ -2917,10 +2645,7 @@ class MySQLDeduplicator:
                             db_results = future.result()
                             all_results[db] = db_results
                         except Exception as e:
-                            self._log('error',
-                                      f"处理数据库 {db} 时出错: {str(e)}",
-                                      {'error_type': type(e).__name__}
-                                      )
+                            logger.error(f"处理数据库 {db} 时出错: {str(e)}", {'error_type': type(e).__name__})
                             all_results[db] = {}
             else:
                 # 串行处理数据库
@@ -2942,18 +2667,15 @@ class MySQLDeduplicator:
                 for r in db.values()
             )
 
-            self._log('info',
-                      f"所有数据库处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                      {'total_results': all_results}
-                      )
+            logger.info(
+                f"所有数据库处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
+                {'total_results': all_results}
+            )
 
             return all_results
 
         except Exception as e:
-            self._log('error',
-                      f"全局处理时发生错误: {str(e)}",
-                      {'error_type': type(e).__name__}
-                      )
+            logger.error(f"全局处理时发生错误: {str(e)}", {'error_type': type(e).__name__})
             return all_results
 
     @_retry_on_failure
@@ -2985,12 +2707,9 @@ class MySQLDeduplicator:
         try:
             if hasattr(self, 'pool') and self.pool:
                 self.pool.close()
-                self._log('info', "数据库连接池已关闭")
+                logger.info("数据库连接池已关闭")
         except Exception as e:
-            self._log('error',
-                      f"关闭连接池时出错: {str(e)}",
-                      {'error_type': type(e).__name__}
-                      )
+            logger.error(f"关闭连接池时出错: {str(e)}", {'error_type': type(e).__name__})
         finally:
             self.pool = None
 
@@ -3004,11 +2723,9 @@ class MySQLDeduplicator:
 def main():
     uploader = MySQLUploader(
         username='root',
-        password='188988yang188',
+        password='pw',
         host='localhost',
         port=3306,
-        logging_mode='console',
-        log_level='info'
     )
 
     # 定义列和数据类型
@@ -3050,24 +2767,22 @@ def main():
 def main2():
     deduplicator = MySQLDeduplicator(
         username='root',
-        password='1',
+        password='pw',
         host='localhost',
         port=3306
     )
 
-    # # 全库去重(单线程)
-    # deduplicator.deduplicate_all()
+    # 全库去重(单线程)
+    deduplicator.deduplicate_all()
 
     # # 指定数据库去重(多线程)
     # deduplicator.deduplicate_database('my_db', parallel=True)
 
-    # 指定表去重(使用特定列)
-    deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
+    # # 指定表去重(使用特定列)
+    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
 
     # 关闭连接
     deduplicator.close()
 
 if __name__ == '__main__':
     pass
-
-    main2()