mdbq 3.9.10__py3-none-any.whl → 3.9.11__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -5,23 +5,12 @@ import time
 from functools import wraps
 import warnings
 import pymysql
-import numpy as np
 import pandas as pd
 from sqlalchemy import create_engine
 import os
-import logging
-import logging.handlers
 from mdbq.other import otk
 from mdbq.log import mylogger
-from typing import Union, List, Dict, Optional, Any, Tuple, Set
-from dbutils.pooled_db import PooledDB
 import json
-import psutil
-from collections import OrderedDict
-import threading
-import concurrent.futures
-from collections import defaultdict
-
 
 warnings.filterwarnings('ignore')
 """
@@ -187,7 +176,7 @@ class MysqlUpload:
                     if str(cut_data).lower() == 'year':
                         table_name = f'{table_name}_{__y}'
                     elif str(cut_data).lower() == 'month':
-                        table_name = f'{table_name}_{__y_m}'
+                        table_name = f'{table_name}-{__y_m}'
                     else:
                         logger.info(f'参数不正确，cut_data应为 year 或 month ')
                 except Exception as e:
@@ -1127,1662 +1116,5 @@ class OptimizeDatas:
             self.connection.close()
 
 
-class StatementCache(OrderedDict):
-    """LRU缓存策略"""
-    def __init__(self, maxsize=100):
-        super().__init__()
-        self.maxsize = maxsize
-
-    def __setitem__(self, key, value):
-        super().__setitem__(key, value)
-        if len(self) > self.maxsize:
-            self.popitem(last=False)
-
-
-class MySQLUploader:
-    def __init__(
-            self,
-            username: str,
-            password: str,
-            host: str = 'localhost',
-            port: int = 3306,
-            charset: str = 'utf8mb4',
-            collation: str = 'utf8mb4_0900_ai_ci',  # utf8mb4_0900_ai_ci: 该排序规则对大小写不敏感, utf8mb4_0900_as_cs/utf8mb4_bin: 对大小写敏感
-            max_retries: int = 10,
-            retry_interval: int = 10,
-            pool_size: int = 5,
-            connect_timeout: int = 10,
-            read_timeout: int = 30,
-            write_timeout: int = 30,
-            ssl: Optional[Dict] = None
-    ):
-        """
-        :param username: 数据库用户名
-        :param password: 数据库密码
-        :param host: 数据库主机地址，默认为localhost
-        :param port: 数据库端口，默认为3306
-        :param charset: 字符集，默认为utf8mb4
-        :param collation: 排序规则，默认为utf8mb4_0900_ai_ci
-
-        :param max_retries: 最大重试次数，默认为10
-        :param retry_interval: 重试间隔(秒)，默认为10
-        :param pool_size: 连接池大小，默认为5
-        :param connect_timeout: 连接超时(秒)，默认为10
-        :param read_timeout: 读取超时(秒)，默认为30
-        :param write_timeout: 写入超时(秒)，默认为30
-        :param ssl: SSL配置字典，默认为None
-        """
-        self.username = username
-        self.password = password
-        self.host = host
-        self.port = port
-        self.charset = charset
-        self.collation = collation
-        self.max_retries = max(max_retries, 1)
-        self.retry_interval = max(retry_interval, 1)
-        self.pool_size = max(pool_size, 1)
-        self.connect_timeout = connect_timeout
-        self.read_timeout = read_timeout
-        self.write_timeout = write_timeout
-        self.ssl = ssl
-        self._prepared_statements = StatementCache(maxsize=100)
-        self._max_cached_statements = 100
-        self._table_metadata_cache = {}
-        self.metadata_cache_ttl = 300  # 5分钟缓存时间
-
-        # 创建连接池
-        self.pool = self._create_connection_pool()
-
-    def _create_connection_pool(self) -> PooledDB:
-        """创建数据库连接池"""
-        if hasattr(self, 'pool') and self.pool is not None and self._check_pool_health():
-            return self.pool
-
-        start_time = time.time()
-        self.pool = None
-
-        pool_params = {
-            'creator': pymysql,
-            'host': self.host,
-            'port': self.port,
-            'user': self.username,
-            'password': self.password,
-            'charset': self.charset,
-            'cursorclass': pymysql.cursors.DictCursor,
-            'maxconnections': self.pool_size,
-            'ping': 7,
-            'connect_timeout': self.connect_timeout,
-            'read_timeout': self.read_timeout,
-            'write_timeout': self.write_timeout,
-            'autocommit': False
-        }
-
-        if self.ssl:
-            required_keys = {'ca', 'cert', 'key'}
-            if not all(k in self.ssl for k in required_keys):
-                error_msg = "SSL配置必须包含ca、cert和key"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-            pool_params['ssl'] = {
-                'ca': self.ssl['ca'],
-                'cert': self.ssl['cert'],
-                'key': self.ssl['key'],
-                'check_hostname': self.ssl.get('check_hostname', False)
-            }
-
-        try:
-            pool = PooledDB(**pool_params)
-            elapsed = time.time() - start_time
-            logger.info("连接池创建成功", {
-                'pool_size': self.pool_size,
-                'time_elapsed': elapsed
-            })
-            return pool
-        except Exception as e:
-            elapsed = time.time() - start_time
-            self.pool = None
-            logger.error("连接池创建失败", {
-                'error': str(e),
-                'time_elapsed': elapsed
-            })
-            raise ConnectionError(f"连接池创建失败: {str(e)}")
-
-    def _execute_with_retry(self, func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            last_exception = None
-            start_time = time.time()
-            operation = func.__name__
-
-            logger.debug(f"开始执行操作: {operation}", {
-                'attempt': 1,
-                'max_retries': self.max_retries
-            })
-
-            for attempt in range(self.max_retries):
-                try:
-                    result = func(*args, **kwargs)
-                    elapsed = time.time() - start_time
-
-                    if attempt > 0:
-                        logger.info("操作成功(重试后)", {
-                            'operation': operation,
-                            'attempts': attempt + 1,
-                            'time_elapsed': elapsed
-                        })
-                    else:
-                        logger.debug("操作成功", {
-                            'operation': operation,
-                            'time_elapsed': elapsed
-                        })
-
-                    return result
-
-                except (pymysql.OperationalError, pymysql.err.MySQLError) as e:
-                    last_exception = e
-
-                    # 记录详细的MySQL错误信息
-                    error_details = {
-                        'operation': operation,
-                        'error_code': e.args[0] if e.args else None,
-                        'error_message': e.args[1] if len(e.args) > 1 else None,
-                        'attempt': attempt + 1,
-                        'max_retries': self.max_retries
-                    }
-
-                    if attempt < self.max_retries - 1:
-                        wait_time = self.retry_interval * (attempt + 1)
-                        error_details['wait_time'] = wait_time
-                        logger.warning(f"数据库操作失败，准备重试 {error_details}", )
-                        time.sleep(wait_time)
-
-                        # 尝试重新连接
-                        try:
-                            self.pool = self._create_connection_pool()
-                            logger.info("成功重新建立数据库连接")
-                        except Exception as reconnect_error:
-                            logger.error("重连失败", {
-                                'error': str(reconnect_error)
-                            })
-                    else:
-                        elapsed = time.time() - start_time
-                        error_details['time_elapsed'] = elapsed
-                        logger.error(f"操作最终失败 {error_details}")
-
-                except pymysql.IntegrityError as e:
-                    elapsed = time.time() - start_time
-                    logger.error("完整性约束错误", {
-                        'operation': operation,
-                        'time_elapsed': elapsed,
-                        'error_code': e.args[0] if e.args else None,
-                        'error_message': e.args[1] if len(e.args) > 1 else None
-                    })
-                    raise e
-
-                except Exception as e:
-                    last_exception = e
-                    elapsed = time.time() - start_time
-                    logger.error("发生意外错误", {
-                        'operation': operation,
-                        'time_elapsed': elapsed,
-                        'error_type': type(e).__name__,
-                        'error_message': str(e),
-                        'error_args': e.args if hasattr(e, 'args') else None
-                    })
-                    break
-
-            raise last_exception if last_exception else Exception("发生未知错误")
-
-        return wrapper
-
-    def _get_connection(self):
-        """从连接池获取连接"""
-        try:
-            conn = self.pool.connection()
-            logger.debug("获取数据库连接")
-            return conn
-        except Exception as e:
-            logger.error(f'{e}')
-            raise ConnectionError(f"连接数据库失败: {str(e)}")
-
-    def _check_database_exists(self, db_name: str) -> bool:
-        """检查数据库是否存在"""
-        db_name = self._validate_identifier(db_name)
-        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
-
-        try:
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute(sql, (db_name,))
-                    exists = bool(cursor.fetchone())
-                    logger.debug(f"{db_name} 数据库已存在: {exists}")
-                    return exists
-        except Exception as e:
-            logger.error(f"检查数据库是否存在时出错: {str(e)}")
-            raise
-
-    def _create_database(self, db_name: str):
-        """创建数据库"""
-        db_name = self._validate_identifier(db_name)
-        sql = f"CREATE DATABASE IF NOT EXISTS `{db_name}` CHARACTER SET {self.charset} COLLATE {self.collation}"
-
-        try:
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute(sql)
-                conn.commit()
-                logger.info(f"{db_name} 数据库已创建")
-        except Exception as e:
-            logger.error(f"{db_name}: 无法创建数据库 {str(e)}")
-            conn.rollback()
-            raise
-
-    def _get_partition_table_name(self, table_name: str, date_value: str, partition_by: str) -> str:
-        """
-        获取分表名称
-
-        :param table_name: 基础表名
-        :param date_value: 日期值
-        :param partition_by: 分表方式 ('year' 或 'month')
-        :return: 分表名称
-        :raises ValueError: 如果日期格式无效或分表方式无效
-        """
-        try:
-            # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d %H:%M:%S')
-            date_obj = self._validate_datetime(date_value, True)
-        except ValueError:
-            try:
-                # date_obj = datetime.datetime.strptime(date_value, '%Y-%m-%d')
-                date_obj = self._validate_datetime(date_value, True)
-            except ValueError:
-                error_msg = f"无效的日期格式1: {date_value}"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-
-        if partition_by == 'year':
-            return f"{table_name}_{date_obj.year}"
-        elif partition_by == 'month':
-            return f"{table_name}_{date_obj.year}_{date_obj.month:02d}"
-        else:
-            error_msg = "partition_by must be 'year' or 'month'"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-    def _validate_identifier(self, identifier: str) -> str:
-        """
-        验证并清理数据库标识符(数据库名、表名、列名)
-        防止SQL注入和非法字符
-
-        :param identifier: 要验证的标识符
-        :return: 清理后的安全标识符
-        :raises ValueError: 如果标识符无效
-        """
-        if not identifier or not isinstance(identifier, str):
-            error_msg = f"无效的标识符: {identifier}"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 移除非法字符，只保留字母、数字、下划线和美元符号
-        cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '', identifier)
-        if not cleaned:
-            error_msg = f"无法清理异常标识符: {identifier}"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 检查是否为MySQL保留字
-        mysql_keywords = {
-            'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
-            'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
-        }
-        if cleaned.lower() in mysql_keywords:
-            logger.debug(f"存在MySQL保留字: {cleaned}")
-            return f"`{cleaned}`"
-
-        return cleaned
-
-    def _check_table_exists(self, db_name: str, table_name: str) -> bool:
-        """检查表是否存在"""
-        cache_key = f"{db_name}.{table_name}"
-        if cache_key in self._table_metadata_cache:
-            cached_time, result = self._table_metadata_cache[cache_key]
-            if time.time() - cached_time < self.metadata_cache_ttl:
-                return result
-
-        db_name = self._validate_identifier(db_name)
-        table_name = self._validate_identifier(table_name)
-        sql = """
-                SELECT TABLE_NAME 
-                FROM INFORMATION_SCHEMA.TABLES 
-                WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
-                """
-
-        try:
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute(sql, (db_name, table_name))
-                    result = bool(cursor.fetchone())
-        except Exception as e:
-            logger.error(f"检查数据表是否存在时发生未知错误: {e}", )
-            raise
-
-        # 执行查询并缓存结果
-        self._table_metadata_cache[cache_key] = (time.time(), result)
-        return result
-
-    def _create_table(
-            self,
-            db_name: str,
-            table_name: str,
-            set_typ: Dict[str, str],
-            primary_keys: Optional[List[str]] = None,
-            date_column: Optional[str] = None,
-            indexes: Optional[List[str]] = None,
-            allow_null: bool = False
-    ):
-        """
-        创建数据表
-
-        :param db_name: 数据库名
-        :param table_name: 表名
-        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
-        :param primary_keys: 主键列列表
-        :param date_column: 日期列名，如果存在将设置为索引
-        :param indexes: 需要创建索引的列列表
-        """
-        db_name = self._validate_identifier(db_name)
-        table_name = self._validate_identifier(table_name)
-
-        if not set_typ:
-            error_msg = "No columns specified for table creation"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 构建列定义SQL
-        column_defs = ["`id` INT NOT NULL AUTO_INCREMENT"]
-
-        # 添加其他列定义
-        for col_name, col_type in set_typ.items():
-            # 跳过id列，因为已经在前面添加了
-            if col_name.lower() == 'id':
-                continue
-            safe_col_name = self._validate_identifier(col_name)
-            col_def = f"`{safe_col_name}` {col_type}"
-
-            # 根据allow_null决定是否添加NOT NULL约束
-            if not allow_null and not col_type.lower().startswith('json'):
-                col_def += " NOT NULL"
-
-            column_defs.append(col_def)
-
-        # 添加主键定义
-        if primary_keys:
-            # 确保id在主键中
-            if 'id' not in [pk.lower() for pk in primary_keys]:
-                primary_keys = ['id'] + primary_keys
-        else:
-            # 如果没有指定主键，则使用id作为主键
-            primary_keys = ['id']
-
-        # 添加主键定义
-        safe_primary_keys = [self._validate_identifier(pk) for pk in primary_keys]
-        primary_key_sql = f", PRIMARY KEY (`{'`,`'.join(safe_primary_keys)}`)"
-
-        # 构建完整SQL
-        sql = f"""
-        CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
-            {','.join(column_defs)}
-            {primary_key_sql}
-        ) ENGINE=InnoDB DEFAULT CHARSET={self.charset} COLLATE={self.collation}
-        """
-
-        try:
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute(sql)
-                    logger.info(f"{db_name}.{table_name}: 数据表已创建")
-
-                # 添加普通索引
-                index_statements = []
-
-                # 日期列索引
-                if date_column and date_column in set_typ:
-                    safe_date_col = self._validate_identifier(date_column)
-                    index_statements.append(
-                        f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_date_col}` (`{safe_date_col}`)"
-                    )
-
-                # 其他索引
-                if indexes:
-                    for idx_col in indexes:
-                        if idx_col in set_typ:
-                            safe_idx_col = self._validate_identifier(idx_col)
-                            index_statements.append(
-                                f"ALTER TABLE `{db_name}`.`{table_name}` ADD INDEX `idx_{safe_idx_col}` (`{safe_idx_col}`)"
-                            )
-
-                # 执行所有索引创建语句
-                if index_statements:
-                    with conn.cursor() as cursor:
-                        for stmt in index_statements:
-                            cursor.execute(stmt)
-                            logger.debug(f"Executed index statement: {stmt}", )
-
-                conn.commit()
-                logger.info(f"{db_name}.{table_name}: 索引已添加")
-
-        except Exception as e:
-            logger.error(f"{db_name}.{table_name}: 建表失败: {str(e)}")
-            conn.rollback()
-            raise
-
-    def _validate_datetime(self, value, date_type=False):
-        """date_type: 返回字符串类型或者日期类型"""
-        formats = [
-            '%Y-%m-%d %H:%M:%S',
-            '%Y-%m-%d',
-            '%Y/%m/%d %H:%M:%S',
-            '%Y/%m/%d',
-            '%Y%m%d',
-            '%Y-%m-%dT%H:%M:%S',
-            '%Y-%m-%d %H:%M:%S.%f',
-            '%Y/%-m/%-d',  # 2023/1/8
-            '%Y-%m-%-d',  # 2023-01-8
-            '%Y-%-m-%-d'  # 2023-1-8
-        ]
-        for fmt in formats:
-            try:
-                if date_type:
-                    return pd.to_datetime(datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d'))
-                else:
-                    return datetime.datetime.strptime(value, fmt).strftime('%Y-%m-%d %H:%M:%S')
-            except ValueError:
-                continue
-        raise ValueError(f"无效的日期格式2: {value}")
-
-    def _validate_value(self, value: Any, column_type: str) -> Any:
-        """
-        验证并清理数据值，根据列类型进行适当转换
-
-        :param value: 要验证的值
-        :param column_type: 列的数据类型
-        :return: 清理后的值
-        :raises ValueError: 如果值转换失败
-        """
-        if value is None:
-            return None
-
-        try:
-            column_type_lower = column_type.lower()
-
-            if 'int' in column_type_lower:
-                if isinstance(value, (str, bytes)) and not value.strip().isdigit():
-                    raise ValueError("非数字字符串无法转换为整数")
-                return int(value)
-            elif any(t in column_type_lower for t in ['float', 'double', 'decimal']):
-                return float(value) if value is not None else None
-            elif '日期' in column_type_lower or 'time' in column_type_lower:
-                if isinstance(value, (datetime.datetime, pd.Timestamp)):
-                    return value.strftime('%Y-%m-%d %H:%M:%S')
-                elif isinstance(value, str):
-                    try:
-                        return self._validate_datetime(value)  # 使用专门的日期验证方法
-                    except ValueError as e:
-                        raise ValueError(f"无效日期格式: {value} - {str(e)}")
-                return str(value)
-            elif 'char' in column_type_lower or 'text' in column_type_lower:
-                # 防止SQL注入
-                if isinstance(value, str):
-                    return value.replace('\\', '\\\\').replace("'", "\\'")
-                return str(value)
-            elif 'json' in column_type_lower:
-                import json
-                return json.dumps(value) if value is not None else None
-            else:
-                return value
-        except (ValueError, TypeError) as e:
-            error_msg = f"数据类型转换异常 {value} to type {column_type}: {str(e)}"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-    def _get_table_columns(self, db_name: str, table_name: str) -> Dict[str, str]:
-        """获取表的列名和数据类型"""
-        db_name = self._validate_identifier(db_name)
-        table_name = self._validate_identifier(table_name)
-        sql = """
-        SELECT COLUMN_NAME, DATA_TYPE 
-        FROM INFORMATION_SCHEMA.COLUMNS 
-        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
-        ORDER BY ORDINAL_POSITION
-        """
-
-        try:
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    cursor.execute(sql, (db_name, table_name))
-                    set_typ = {row['COLUMN_NAME']: row['DATA_TYPE'] for row in cursor.fetchall()}
-                    logger.debug(f"{db_name}.{table_name}: 获取表的列信息: {set_typ}")
-                    return set_typ
-        except Exception as e:
-            logger.error(f"无法获取表列信息: {str(e)}")
-            raise
-
-    def _upload_to_table(
-            self,
-            db_name: str,
-            table_name: str,
-            data: List[Dict],
-            set_typ: Dict[str, str],
-            primary_keys: Optional[List[str]],
-            check_duplicate: bool,
-            duplicate_columns: Optional[List[str]],
-            allow_null: bool,
-            auto_create: bool,
-            date_column: Optional[str],
-            indexes: Optional[List[str]],
-            batch_id: Optional[str] = None
-    ):
-        """实际执行表上传的方法"""
-        # 检查表是否存在
-        if not self._check_table_exists(db_name, table_name):
-            if auto_create:
-                self._create_table(db_name, table_name, set_typ, primary_keys, date_column, indexes,
-                                   allow_null=allow_null)
-            else:
-                error_msg = f"数据表不存在: '{db_name}.{table_name}'"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-
-        # 获取表结构并验证
-        table_columns = self._get_table_columns(db_name, table_name)
-        if not table_columns:
-            error_msg = f"获取列失败 '{db_name}.{table_name}'"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 验证数据列与表列匹配
-        for col in set_typ:
-            if col not in table_columns:
-                error_msg = f"列不存在: '{col}' -> '{db_name}.{table_name}'"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-
-        # 插入数据
-        self._insert_data(
-            db_name, table_name, data, set_typ,
-            check_duplicate, duplicate_columns
-        )
-
-    def _infer_data_type(self, value: Any) -> str:
-        """
-        根据值推断合适的数据类型
-
-        :param value: 要推断的值
-        :return: MySQL数据类型字符串
-        """
-        if value is None:
-            return 'VARCHAR(255)'  # 默认字符串类型
-
-        if isinstance(value, bool):
-            return 'TINYINT(1)'
-        elif isinstance(value, int):
-            # if -128 <= value <= 127:
-            #     return 'TINYINT'
-            # elif -32768 <= value <= 32767:
-            #     return 'SMALLINT'
-            # elif -8388608 <= value <= 8388607:
-            #     return 'MEDIUMINT'
-            if -2147483648 <= value <= 2147483647:
-                return 'INT'
-            else:
-                return 'BIGINT'
-        elif isinstance(value, float):
-            return 'DECIMAL(10,2)'
-        elif isinstance(value, (datetime.datetime, pd.Timestamp)):
-            return 'DATETIME'
-        elif isinstance(value, datetime.date):
-            return 'DATE'
-        elif isinstance(value, (list, dict)):
-            return 'JSON'
-        elif isinstance(value, str):
-            # 尝试判断是否是日期时间
-            try:
-                self._validate_datetime(value)
-                return 'DATETIME'
-            except ValueError:
-                pass
-
-            # 根据字符串长度选择合适类型
-            length = len(value)
-            if length <= 255:
-                return 'VARCHAR(255)'
-            elif length <= 65535:
-                return 'TEXT'
-            elif length <= 16777215:
-                return 'MEDIUMTEXT'
-            else:
-                return 'LONGTEXT'
-        else:
-            return 'VARCHAR(255)'
-
-    def _prepare_data(
-            self,
-            data: Union[Dict, List[Dict], pd.DataFrame],
-            set_typ: Dict[str, str],
-            allow_null: bool = False
-    ) -> List[Dict]:
-        """
-        准备要上传的数据，验证并转换数据类型
-
-        :param data: 输入数据
-        :param set_typ: 列名和数据类型字典 {列名: 数据类型}
-        :param allow_null: 是否允许空值
-        :return: 待上传的数据列表和对应的数据类型
-        :raises ValueError: 如果数据验证失败
-        """
-        # 统一数据格式为字典列表
-        if isinstance(data, pd.DataFrame):
-            try:
-                # 将列名转为小写
-                data.columns = [col.lower() for col in data.columns]
-                data = data.replace({pd.NA: None}).to_dict('records')
-            except Exception as e:
-                logger.error(f"数据转字典时发生错误: {e}", )
-                raise ValueError(f"数据转字典时发生错误: {e}")
-        elif isinstance(data, dict):
-            data = [{k.lower(): v for k, v in data.items()}]
-        elif isinstance(data, list) and all(isinstance(item, dict) for item in data):
-            # 将列表中的每个字典键转为小写
-            data = [{k.lower(): v for k, v in item.items()} for item in data]
-        else:
-            error_msg = "数据结构必须是字典、列表、字典列表或dataframe"
-            logger.error(error_msg)
-            raise ValueError(error_msg)
-
-        # 将set_typ的键转为小写
-        set_typ = {k.lower(): v for k, v in set_typ.items()}
-
-        # 获取数据中实际存在的列名
-        data_columns = set()
-        if data:
-            data_columns = set(data[0].keys())
-
-        # 过滤set_typ，只保留数据中存在的列
-        filtered_set_typ = {}
-        for col in data_columns:
-            if col in set_typ:
-                filtered_set_typ[col] = set_typ[col]
-            else:
-                # 如果列不在set_typ中，尝试推断类型
-                sample_values = [row[col] for row in data if col in row and row[col] is not None][:10]
-                if sample_values:
-                    inferred_type = self._infer_data_type(sample_values[0])
-                    filtered_set_typ[col] = inferred_type
-                    logger.debug(f"自动推断列'{col}'的数据类型为: {inferred_type}")
-                else:
-                    # 没有样本值，使用默认类型
-                    filtered_set_typ[col] = 'VARCHAR(255)'
-                    logger.debug(f"为列'{col}'使用默认数据类型: VARCHAR(255)")
-
-        prepared_data = []
-        for row_idx, row in enumerate(data, 1):
-            prepared_row = {}
-            for col_name in filtered_set_typ:
-                # 跳过id列，不允许外部传入id
-                if col_name.lower() == 'id':
-                    continue
-
-                if col_name not in row:
-                    if not allow_null:
-                        error_msg = f"Row {row_idx}: Missing required column '{col_name}' in data"
-                        logger.error(error_msg)
-                        raise ValueError(error_msg)
-                    prepared_row[col_name] = None
-                else:
-                    try:
-                        prepared_row[col_name] = self._validate_value(row[col_name], filtered_set_typ[col_name])
-                    except ValueError as e:
-                        error_msg = f"Row {row_idx}, column '{col_name}': {str(e)}"
-                        logger.error(error_msg)
-                        raise ValueError(error_msg)
-            prepared_data.append(prepared_row)
-
-        logger.debug(f"已准备 {len(prepared_data)} 行数据")
-        return prepared_data, filtered_set_typ
-
-    def upload_data(
-            self,
-            db_name: str,
-            table_name: str,
-            data: Union[Dict, List[Dict], pd.DataFrame],
-            set_typ: Dict[str, str],
-            primary_keys: Optional[List[str]] = None,
-            check_duplicate: bool = False,
-            duplicate_columns: Optional[List[str]] = None,
-            allow_null: bool = False,
-            partition_by: Optional[str] = None,
-            partition_date_column: str = '日期',
-            auto_create: bool = True,
-            indexes: Optional[List[str]] = None
-    ):
-        """
-        上传数据到数据库
-        """
-        upload_start = time.time()
-        initial_row_count = len(data) if hasattr(data, '__len__') else 1
-
-        batch_id = f"batch_{int(time.time() * 1000)}"
-        success_flag = False
-
-        logger.info("开始上传数据", {
-            'batch_id': batch_id,
-            'database': db_name,
-            'table': table_name,
-            'partition_by': partition_by,
-            'check_duplicate': check_duplicate,
-            'row_count': len(data) if hasattr(data, '__len__') else 1,
-            'auto_create': auto_create
-        })
-
-        try:
-            # 验证参数
-            if not set_typ:
-                error_msg = "列的数据类型缺失"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-
-            if partition_by and partition_by not in ['year', 'month']:
-                error_msg = "分表方式必须是 'year' 或 'month'"
-                logger.error(error_msg)
-                raise ValueError(error_msg)
-
-            # 准备数据
-            prepared_data, set_typ = self._prepare_data(data, set_typ, allow_null)
-
-            # 检查数据库是否存在
-            if not self._check_database_exists(db_name):
-                if auto_create:
-                    self._create_database(db_name)
-                else:
-                    error_msg = f"数据库不存在: '{db_name}'"
-                    logger.error(error_msg)
-                    raise ValueError(error_msg)
-
-            # 处理分表逻辑
-            if partition_by:
-                partitioned_data = {}
-                for row in prepared_data:
-                    try:
-                        if partition_date_column not in row:
-                            error_msg = f"异常缺失列 '{partition_date_column}'"
-                            logger.error(error_msg)
-                            continue  # 跳过当前行
-
-                        part_table = self._get_partition_table_name(
-                            table_name,
-                            str(row[partition_date_column]),
-                            partition_by
-                        )
-                        if part_table not in partitioned_data:
-                            partitioned_data[part_table] = []
-                        partitioned_data[part_table].append(row)
-                    except Exception as e:
-                        logger.error("分表处理失败", {
-                            'row_data': row,
-                            'error': str(e)
-                        })
-                        continue  # 跳过当前行
-
-                # 对每个分表执行上传
-                for part_table, part_data in partitioned_data.items():
-                    try:
-                        self._upload_to_table(
-                            db_name, part_table, part_data, set_typ,
-                            primary_keys, check_duplicate, duplicate_columns,
-                            allow_null, auto_create, partition_date_column,
-                            indexes, batch_id
-                        )
-                    except Exception as e:
-                        logger.error("分表上传失败", {
-                            'partition_table': part_table,
-                            'error': str(e)
-                        })
-                        continue  # 跳过当前分表，继续处理其他分表
-            else:
-                # 不分表，直接上传
-                self._upload_to_table(
-                    db_name, table_name, prepared_data, set_typ,
-                    primary_keys, check_duplicate, duplicate_columns,
-                    allow_null, auto_create, partition_date_column,
-                    indexes, batch_id
-                )
-
-            success_flag = True
-
-        except Exception as e:
-            logger.error("上传过程中发生全局错误", {
-                'error': str(e),
-                'error_type': type(e).__name__
-            })
-        finally:
-            elapsed = time.time() - upload_start
-            logger.info("上传处理完成", {
-                'batch_id': batch_id,
-                'success': success_flag,
-                'time_elapsed': elapsed,
-                'initial_row_count': initial_row_count
-            })
-
-    def _insert_data(
-            self,
-            db_name: str,
-            table_name: str,
-            data: List[Dict],
-            set_typ: Dict[str, str],
-            check_duplicate: bool = False,
-            duplicate_columns: Optional[List[str]] = None,
-            batch_size: int = 1000,
-            batch_id: Optional[str] = None
-    ):
-        """
-        插入数据到表中
-
-        参数:
-        db_name: 数据库名
-        table_name: 表名
-        data: 要插入的数据列表
-        set_typ: 列名和数据类型字典 {列名: 数据类型}
-        check_duplicate: 是否检查重复
-        duplicate_columns: 用于检查重复的列(为空时检查所有列)
-        batch_size: 批量插入大小
-        batch_id: 批次ID用于日志追踪
-        """
-        if not data:
-            return
-
-        # 获取所有列名（排除id列）
-        all_columns = [col for col in set_typ.keys() if col.lower() != 'id']
-        safe_columns = [self._validate_identifier(col) for col in all_columns]
-        placeholders = ','.join(['%s'] * len(safe_columns))
-
-        # 构建基础SQL语句
-        if check_duplicate:
-            if not duplicate_columns:
-                duplicate_columns = all_columns
-            else:
-                duplicate_columns = [col for col in duplicate_columns if col != 'id']
-
-            conditions = []
-            for col in duplicate_columns:
-                col_type = set_typ.get(col, '').lower()
-
-                # 处理DECIMAL类型，使用ROUND确保精度一致
-                if col_type.startswith('decimal'):
-                    # 提取小数位数，如DECIMAL(10,2)提取2
-                    scale_match = re.search(r'decimal\(\d+,(\d+)\)', col_type)
-                    scale = int(scale_match.group(1)) if scale_match else 2
-                    conditions.append(f"ROUND(`{self._validate_identifier(col)}`, {scale}) = ROUND(%s, {scale})")
-                else:
-                    conditions.append(f"`{self._validate_identifier(col)}` = %s")
-
-            where_clause = " AND ".join(conditions)
-
-            sql = f"""
-                    INSERT INTO `{db_name}`.`{table_name}` 
-                    (`{'`,`'.join(safe_columns)}`) 
-                    SELECT {placeholders}
-                    FROM DUAL
-                    WHERE NOT EXISTS (
-                        SELECT 1 FROM `{db_name}`.`{table_name}`
-                        WHERE {where_clause}
-                    )
-                    """
-        else:
-            sql = f"""
-                INSERT INTO `{db_name}`.`{table_name}` 
-                (`{'`,`'.join(safe_columns)}`) 
-                VALUES ({placeholders})
-                """
-
-        total_inserted = 0
-        total_skipped = 0
-        total_failed = 0  # 失败计数器
-
-        # 分批插入数据
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                for i in range(0, len(data), batch_size):
-                    batch_start = time.time()
-                    batch = data[i:i + batch_size]
-                    successful_rows = 0  # 当前批次成功数
-
-                    for row in batch:
-                        try:
-                            # 准备参数
-                            row_values = [row.get(col) for col in all_columns]
-                            # 如果是排重检查，添加排重列值
-                            if check_duplicate:
-                                row_values += [row.get(col) for col in duplicate_columns]
-
-                            cursor.execute(sql, row_values)
-                            successful_rows += 1
-                            conn.commit()  # 每次成功插入后提交
-
-                        except Exception as e:
-                            conn.rollback()  # 回滚当前行的事务
-                            total_failed += 1
-
-                            # 记录失败行详细信息
-                            error_details = {
-                                'batch_id': batch_id,
-                                'database': db_name,
-                                'table': table_name,
-                                'error_type': type(e).__name__,
-                                'error_message': str(e),
-                                'column_types': set_typ,
-                                'duplicate_check': check_duplicate,
-                                'duplicate_columns': duplicate_columns
-                            }
-                            logger.error(f"单行插入失败: {error_details}")
-                            continue  # 跳过当前行，继续处理下一行
-
-                    # 更新统计信息
-                    if check_duplicate:
-                        cursor.execute("SELECT ROW_COUNT()")
-                        affected_rows = cursor.rowcount
-                        total_inserted += affected_rows
-                        total_skipped += len(batch) - affected_rows - (len(batch) - successful_rows)
-                    else:
-                        total_inserted += successful_rows
-
-                    batch_elapsed = time.time() - batch_start
-                    batch_info = {
-                        'batch_id': batch_id,
-                        'batch_index': i // batch_size + 1,
-                        'total_batches': (len(data) + batch_size - 1) // batch_size,
-                        'batch_size': len(batch),
-                        'successful_rows': successful_rows,
-                        'failed_rows': len(batch) - successful_rows,
-                        'time_elapsed': batch_elapsed,
-                        'rows_per_second': successful_rows / batch_elapsed if batch_elapsed > 0 else 0
-                    }
-                    logger.debug(f"批次处理完成 {batch_info}")
-
-        logger.info("数据插入完成", {
-            'total_rows': len(data),
-            'inserted_rows': total_inserted,
-            'skipped_rows': total_skipped,
-            'failed_rows': total_failed
-        })
-
-    def close(self):
-        """关闭连接池并记录最终指标"""
-        close_start = time.time()
-
-        try:
-            if hasattr(self, 'pool') and self.pool is not None:
-                # 更安全的关闭方式
-                try:
-                    self.pool.close()
-                except Exception as e:
-                    logger.warning("关闭连接池时出错", {
-                        'error': str(e)
-                    })
-
-                self.pool = None
-
-                elapsed = round(time.time() - close_start, 2)
-                logger.info("连接池已关闭", {
-                    'close_time_elapsed': elapsed
-                })
-        except Exception as e:
-            elapsed = round(time.time() - close_start, 2)
-            logger.error("关闭连接池失败", {
-                'error': str(e),
-                'close_time_elapsed': elapsed
-            })
-            raise
-
-    def _check_pool_health(self):
-        """定期检查连接池健康状态"""
-        try:
-            conn = self.pool.connection()
-            conn.ping(reconnect=True)
-            conn.close()
-            return True
-        except Exception as e:
-            logger.warning("连接池健康检查失败", {
-                'error': str(e)
-            })
-            return False
-
-    def retry_on_failure(max_retries=3, delay=1):
-        def decorator(func):
-            @wraps(func)
-            def wrapper(*args, **kwargs):
-                last_exception = None
-                for attempt in range(max_retries):
-                    try:
-                        return func(*args, **kwargs)
-                    except (pymysql.OperationalError, pymysql.InterfaceError) as e:
-                        last_exception = e
-                        if attempt < max_retries - 1:
-                            time.sleep(delay * (attempt + 1))
-                            continue
-                        raise MySQLUploaderError(f"操作重试{max_retries}次后失败") from e
-                    except Exception as e:
-                        raise MySQLUploaderError(f"操作失败: {str(e)}") from e
-                raise last_exception if last_exception else MySQLUploaderError("未知错误")
-
-            return wrapper
-
-        return decorator
-
-
-class MySQLDeduplicator:
-    """
-    MySQL数据去重
-
-    功能：
-    1. 自动检测并删除MySQL数据库中的重复数据
-    2. 支持全库扫描或指定表处理
-    3. 支持多线程/多进程安全处理
-    4. 完善的错误处理和日志记录
-
-    使用示例：
-    deduplicator = MySQLDeduplicator(
-        username='root',
-        password='password',
-        host='localhost',
-        port=3306
-    )
-
-    # 全库去重
-    deduplicator.deduplicate_all()
-
-    # 指定数据库去重(多线程)
-    deduplicator.deduplicate_database('my_db', parallel=True)
-
-    # 指定表去重(使用特定列)
-    deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
-
-    # 关闭连接
-    deduplicator.close()
-    """
-
-    def __init__(
-            self,
-            username: str,
-            password: str,
-            host: str = 'localhost',
-            port: int = 3306,
-            charset: str = 'utf8mb4',
-            max_workers: int = 1,
-            batch_size: int = 1000,
-            skip_system_dbs: bool = True,
-            max_retries: int = 3,
-            retry_interval: int = 5,
-            pool_size: int = 5
-    ):
-        """
-        初始化去重处理器
-
-        :param username: 数据库用户名
-        :param password: 数据库密码
-        :param host: 数据库主机，默认为localhost
-        :param port: 数据库端口，默认为3306
-        :param charset: 字符集，默认为utf8mb4
-        :param max_workers: 最大工作线程数，默认为1(单线程)
-        :param batch_size: 批量处理大小，默认为1000
-        :param skip_system_dbs: 是否跳过系统数据库，默认为True
-        :param max_retries: 最大重试次数
-        :param retry_interval: 重试间隔(秒)
-        :param pool_size: 连接池大小
-        """
-        # 初始化连接池
-        self.pool = PooledDB(
-            creator=pymysql,
-            host=host,
-            port=port,
-            user=username,
-            password=password,
-            charset=charset,
-            maxconnections=pool_size,
-            cursorclass=pymysql.cursors.DictCursor
-        )
-
-        # 配置参数
-        self.max_workers = max(1, min(max_workers, 20))  # 限制最大线程数
-        self.batch_size = batch_size
-        self.skip_system_dbs = skip_system_dbs
-        self.max_retries = max_retries
-        self.retry_interval = retry_interval
-
-        # 线程安全控制
-        self._lock = threading.Lock()
-        self._processing_tables = set()  # 正在处理的表集合
-
-        # 系统数据库列表
-        self.SYSTEM_DATABASES = {
-            'information_schema', 'mysql',
-            'performance_schema', 'sys'
-        }
-
-    def _get_connection(self):
-        """从连接池获取连接"""
-        try:
-            conn = self.pool.connection()
-            logger.debug("成功获取数据库连接")
-            return conn
-        except Exception as e:
-            logger.error(f"获取数据库连接失败: {str(e)}")
-            raise ConnectionError(f"连接数据库失败: {str(e)}")
-
-    @staticmethod
-    def _retry_on_failure(func):
-        """重试装饰器"""
-
-        @wraps(func)
-        def wrapper(self, *args, **kwargs):
-            last_exception = None
-            for attempt in range(self.max_retries + 1):
-                try:
-                    return func(self, *args, **kwargs)
-                except (pymysql.OperationalError, pymysql.InterfaceError) as e:
-                    last_exception = e
-                    if attempt < self.max_retries:
-                        wait_time = self.retry_interval * (attempt + 1)
-                        logger.warning(
-                            f"数据库操作失败，准备重试 (尝试 {attempt + 1}/{self.max_retries})",
-                            {'error': str(e), 'wait_time': wait_time})
-                        time.sleep(wait_time)
-                        continue
-                except Exception as e:
-                    last_exception = e
-                    logger.error(f"操作失败: {str(e)}", {'error_type': type(e).__name__})
-                    break
-
-            if last_exception:
-                raise last_exception
-            raise Exception("未知错误")
-
-        return wrapper
-
-    @_retry_on_failure
-    def _get_databases(self) -> List[str]:
-        """获取所有非系统数据库列表"""
-        sql = "SHOW DATABASES"
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql)
-                all_dbs = [row['Database'] for row in cursor.fetchall()]
-
-                if self.skip_system_dbs:
-                    return [db for db in all_dbs if db.lower() not in self.SYSTEM_DATABASES]
-                return all_dbs
-
-    @_retry_on_failure
-    def _get_tables(self, database: str) -> List[str]:
-        """获取指定数据库的所有表"""
-        sql = "SHOW TABLES"
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(f"USE `{database}`")
-                cursor.execute(sql)
-                return [row[f'Tables_in_{database}'] for row in cursor.fetchall()]
-
-    @_retry_on_failure
-    def _get_table_columns(self, database: str, table: str) -> List[str]:
-        """获取表的列名(排除id列)"""
-        sql = """
-        SELECT COLUMN_NAME 
-        FROM INFORMATION_SCHEMA.COLUMNS 
-        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
-        ORDER BY ORDINAL_POSITION
-        """
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (database, table))
-                return [row['COLUMN_NAME'] for row in cursor.fetchall()
-                        if row['COLUMN_NAME'].lower() != 'id']
-
-    def _acquire_table_lock(self, database: str, table: str) -> bool:
-        """获取表处理锁，防止并发处理同一张表"""
-        key = f"{database}.{table}"
-
-        with self._lock:
-            if key in self._processing_tables:
-                logger.debug(f"表 {key} 正在被其他线程处理，跳过")
-                return False
-            self._processing_tables.add(key)
-            return True
-
-    def _release_table_lock(self, database: str, table: str):
-        """释放表处理锁"""
-        key = f"{database}.{table}"
-
-        with self._lock:
-            if key in self._processing_tables:
-                self._processing_tables.remove(key)
-
-    def _deduplicate_table(
-            self,
-            database: str,
-            table: str,
-            columns: Optional[List[str]] = None,
-            dry_run: bool = False
-    ) -> Tuple[int, int]:
-        """
-        执行单表去重
-
-        :param database: 数据库名
-        :param table: 表名
-        :param columns: 用于去重的列(为None时使用所有列)
-        :param dry_run: 是否模拟运行(只统计不实际删除)
-        :return: (重复行数, 删除行数)
-        """
-        if not self._acquire_table_lock(database, table):
-            return (0, 0)
-
-        try:
-            logger.info(f"开始处理表: {database}.{table}")
-
-            # 获取实际列名
-            all_columns = self._get_table_columns(database, table)
-            if not all_columns:
-                logger.warning(f"表 {database}.{table} 没有有效列(可能只有id列)，跳过")
-                return (0, 0)
-
-            # 使用指定列或所有列
-            use_columns = columns or all_columns
-            invalid_columns = set(use_columns) - set(all_columns)
-
-            if invalid_columns:
-                logger.warning(
-                    f"表 {database}.{table} 中不存在以下列: {invalid_columns}，使用有效列",
-                    {'invalid_columns': invalid_columns}
-                )
-                use_columns = [col for col in use_columns if col in all_columns]
-
-            if not use_columns:
-                logger.error(f"表 {database}.{table} 没有有效的去重列")
-                return (0, 0)
-
-            # 构建去重SQL
-            column_list = ', '.join([f'`{col}`' for col in use_columns])
-            temp_table = f"temp_{table}_{int(time.time())}"
-
-            # 使用临时表方案处理去重，避免锁表问题
-            create_temp_sql = f"""
-            CREATE TABLE `{database}`.`{temp_table}` AS
-            SELECT MIN(`id`) as `min_id`, {column_list}, COUNT(*) as `dup_count`
-            FROM `{database}`.`{table}`
-            GROUP BY {column_list}
-            HAVING COUNT(*) > 1
-            """
-
-            delete_dup_sql = f"""
-            DELETE FROM `{database}`.`{table}`
-            WHERE `id` NOT IN (
-                SELECT `min_id` FROM `{database}`.`{temp_table}`
-            ) AND ({' OR '.join([f'`{col}` IS NOT NULL' for col in use_columns])})
-            """
-
-            drop_temp_sql = f"DROP TABLE IF EXISTS `{database}`.`{temp_table}`"
-
-            with self._get_connection() as conn:
-                with conn.cursor() as cursor:
-                    # 创建临时表统计重复数据
-                    cursor.execute(create_temp_sql)
-                    cursor.execute(f"SELECT COUNT(*) as cnt FROM `{database}`.`{temp_table}`")
-                    dup_count = cursor.fetchone()['cnt']
-
-                    if dup_count == 0:
-                        logger.info(f"表 {database}.{table} 没有重复数据")
-                        cursor.execute(drop_temp_sql)
-                        conn.commit()
-                        return (0, 0)
-
-                    logger.info(
-                        f"表 {database}.{table} 发现 {dup_count} 组重复数据",
-                        {'columns': use_columns}
-                    )
-
-                    if not dry_run:
-                        # 执行实际删除
-                        cursor.execute(delete_dup_sql)
-                        affected_rows = cursor.rowcount
-                        conn.commit()
-                        logger.info(
-                            f"表 {database}.{table} 已删除 {affected_rows} 行重复数据",
-                            {'columns': use_columns}
-                        )
-                    else:
-                        affected_rows = 0
-                        logger.info(
-                            f"[模拟运行] 表 {database}.{table} 将删除 {dup_count} 组重复数据",
-                            {'columns': use_columns}
-                        )
-
-                    # 清理临时表
-                    cursor.execute(drop_temp_sql)
-                    conn.commit()
-
-                    return (dup_count, affected_rows)
-
-        except Exception as e:
-            logger.error(
-                f"处理表 {database}.{table} 时出错: {str(e)}",
-                {'error_type': type(e).__name__}
-            )
-            return (0, 0)
-        finally:
-            self._release_table_lock(database, table)
-
-    def deduplicate_table(
-            self,
-            database: str,
-            table: str,
-            columns: Optional[List[str]] = None,
-            dry_run: bool = False
-    ) -> Tuple[int, int]:
-        """
-        对指定表进行去重
-
-        :param database: 数据库名
-        :param table: 表名
-        :param columns: 用于去重的列(为None时使用所有列)
-        :param dry_run: 是否模拟运行(只统计不实际删除)
-        :return: (重复行数, 删除行数)
-        """
-        try:
-            # 检查表是否存在
-            if not self._check_table_exists(database, table):
-                logger.warning(f"表 {database}.{table} 不存在，跳过")
-                return (0, 0)
-
-            return self._deduplicate_table(database, table, columns, dry_run)
-        except Exception as e:
-            logger.error(
-                f"处理表 {database}.{table} 时发生全局错误: {str(e)}",
-                {'error_type': type(e).__name__}
-            )
-            return (0, 0)
-
-    def deduplicate_database(
-            self,
-            database: str,
-            tables: Optional[List[str]] = None,
-            columns_map: Optional[Dict[str, List[str]]] = None,
-            dry_run: bool = False,
-            parallel: bool = False
-    ) -> Dict[str, Tuple[int, int]]:
-        """
-        对指定数据库的所有表进行去重
-
-        :param database: 数据库名
-        :param tables: 要处理的表列表(为None时处理所有表)
-        :param columns_map: 各表使用的去重列 {表名: [列名]}
-        :param dry_run: 是否模拟运行
-        :param parallel: 是否并行处理
-        :return: 字典 {表名: (重复行数, 删除行数)}
-        """
-        results = {}
-
-        try:
-            # 检查数据库是否存在
-            if not self._check_database_exists(database):
-                logger.warning(f"数据库 {database} 不存在，跳过")
-                return results
-
-            # 获取要处理的表
-            target_tables = tables or self._get_tables(database)
-            if not target_tables:
-                logger.info(f"数据库 {database} 中没有表，跳过")
-                return results
-
-            logger.info(
-                f"开始处理数据库 {database} 中的 {len(target_tables)} 张表",
-                {'tables': target_tables}
-            )
-
-            if parallel and self.max_workers > 1:
-                # 并行处理
-                with concurrent.futures.ThreadPoolExecutor(
-                        max_workers=self.max_workers
-                ) as executor:
-                    futures = {}
-                    for table in target_tables:
-                        columns = columns_map.get(table) if columns_map else None
-                        futures[executor.submit(
-                            self.deduplicate_table,
-                            database, table, columns, dry_run
-                        )] = table
-
-                    for future in concurrent.futures.as_completed(futures):
-                        table = futures[future]
-                        try:
-                            dup_count, affected_rows = future.result()
-                            results[table] = (dup_count, affected_rows)
-                        except Exception as e:
-                            logger.error(
-                                f"处理表 {database}.{table} 时出错: {str(e)}",
-                                {'error_type': type(e).__name__}
-                            )
-                            results[table] = (0, 0)
-            else:
-                # 串行处理
-                for table in target_tables:
-                    columns = columns_map.get(table) if columns_map else None
-                    dup_count, affected_rows = self.deduplicate_table(
-                        database, table, columns, dry_run
-                    )
-                    results[table] = (dup_count, affected_rows)
-
-            # 统计结果
-            total_dup = sum(r[0] for r in results.values())
-            total_del = sum(r[1] for r in results.values())
-
-            logger.info(
-                f"数据库 {database} 处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                {'results': results}
-            )
-
-            return results
-
-        except Exception as e:
-            logger.error(f"处理数据库 {database} 时发生全局错误: {str(e)}", {'error_type': type(e).__name__})
-            return results
-
-    def deduplicate_all(
-            self,
-            databases: Optional[List[str]] = None,
-            tables_map: Optional[Dict[str, List[str]]] = None,
-            columns_map: Optional[Dict[str, Dict[str, List[str]]]] = None,
-            dry_run: bool = False,
-            parallel: bool = False
-    ) -> Dict[str, Dict[str, Tuple[int, int]]]:
-        """
-        对所有数据库进行去重
-
-        :param databases: 要处理的数据库列表(为None时处理所有非系统数据库)
-        :param tables_map: 各数据库要处理的表 {数据库名: [表名]}
-        :param columns_map: 各表使用的去重列 {数据库名: {表名: [列名]}}
-        :param dry_run: 是否模拟运行
-        :param parallel: 是否并行处理
-        :return: 嵌套字典 {数据库名: {表名: (重复行数, 删除行数)}}
-        """
-        all_results = defaultdict(dict)
-
-        try:
-            # 获取要处理的数据库
-            target_dbs = databases or self._get_databases()
-            if not target_dbs:
-                logger.warning("没有可处理的数据库")
-                return all_results
-
-            logger.info(f"开始处理 {len(target_dbs)} 个数据库", {'databases': target_dbs})
-
-            if parallel and self.max_workers > 1:
-                # 并行处理数据库
-                with concurrent.futures.ThreadPoolExecutor(
-                        max_workers=self.max_workers
-                ) as executor:
-                    futures = {}
-                    for db in target_dbs:
-                        tables = tables_map.get(db) if tables_map else None
-                        db_columns_map = columns_map.get(db) if columns_map else None
-                        futures[executor.submit(
-                            self.deduplicate_database,
-                            db, tables, db_columns_map, dry_run, False
-                        )] = db
-
-                    for future in concurrent.futures.as_completed(futures):
-                        db = futures[future]
-                        try:
-                            db_results = future.result()
-                            all_results[db] = db_results
-                        except Exception as e:
-                            logger.error(f"处理数据库 {db} 时出错: {str(e)}", {'error_type': type(e).__name__})
-                            all_results[db] = {}
-            else:
-                # 串行处理数据库
-                for db in target_dbs:
-                    tables = tables_map.get(db) if tables_map else None
-                    db_columns_map = columns_map.get(db) if columns_map else None
-                    db_results = self.deduplicate_database(
-                        db, tables, db_columns_map, dry_run, parallel
-                    )
-                    all_results[db] = db_results
-
-            # 统计总体结果
-            total_dup = sum(
-                r[0] for db in all_results.values()
-                for r in db.values()
-            )
-            total_del = sum(
-                r[1] for db in all_results.values()
-                for r in db.values()
-            )
-
-            logger.info(
-                f"所有数据库处理完成 - 共发现 {total_dup} 组重复数据，删除 {total_del} 行",
-                {'total_results': all_results}
-            )
-
-            return all_results
-
-        except Exception as e:
-            logger.error(f"全局处理时发生错误: {str(e)}", {'error_type': type(e).__name__})
-            return all_results
-
-    @_retry_on_failure
-    def _check_database_exists(self, database: str) -> bool:
-        """检查数据库是否存在"""
-        sql = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s"
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (database,))
-                return bool(cursor.fetchone())
-
-    @_retry_on_failure
-    def _check_table_exists(self, database: str, table: str) -> bool:
-        """检查表是否存在"""
-        sql = """
-        SELECT TABLE_NAME 
-        FROM INFORMATION_SCHEMA.TABLES 
-        WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s
-        """
-
-        with self._get_connection() as conn:
-            with conn.cursor() as cursor:
-                cursor.execute(sql, (database, table))
-                return bool(cursor.fetchone())
-
-    def close(self):
-        """关闭连接池"""
-        try:
-            if hasattr(self, 'pool') and self.pool:
-                self.pool.close()
-                logger.info("数据库连接池已关闭")
-        except Exception as e:
-            logger.error(f"关闭连接池时出错: {str(e)}", {'error_type': type(e).__name__})
-        finally:
-            self.pool = None
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.close()
-
-
-def main():
-    uploader = MySQLUploader(
-        username='root',
-        password='pw',
-        host='localhost',
-        port=3306,
-    )
-
-    # 定义列和数据类型
-    set_typ = {
-        'name': 'VARCHAR(255)',
-        'age': 'INT',
-        'salary': 'DECIMAL(10,2)',
-        '日期': 'DATE',
-        'shop': None,
-    }
-
-    # 准备数据
-    data = [
-        {'日期': '2023-01-8', 'name': 'JACk', 'AGE': '24', 'salary': 555.1545},
-        {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 35, 'salary': 100},
-        {'日期': '2023-01-15', 'name': 'Alice', 'AGE': 30, 'salary': 0.0},
-        {'日期': '2023-02-20', 'name': 'Bob', 'AGE': 25, 'salary': 45000.75}
-    ]
-
-    # 上传数据
-    uploader.upload_data(
-        db_name='测试库',
-        table_name='测试表',
-        data=data,
-        set_typ=set_typ,  # 定义列和数据类型
-        primary_keys=[],  # 创建唯一主键
-        check_duplicate=False,  # 检查重复数据
-        duplicate_columns=[],  # 指定排重的组合键
-        allow_null=False,  # 允许插入空值
-        partition_by='year',  # 按月分表
-        partition_date_column='日期',  # 用于分表的日期列名，默认为'日期'
-        auto_create=True,  # 表不存在时自动创建, 默认参数不要更改
-        indexes=[],  # 指定索引列
-    )
-
-    uploader.close()
-
-
-def main2():
-    deduplicator = MySQLDeduplicator(
-        username='root',
-        password='pw',
-        host='localhost',
-        port=3306
-    )
-
-    # 全库去重(单线程)
-    deduplicator.deduplicate_all()
-
-    # # 指定数据库去重(多线程)
-    # deduplicator.deduplicate_database('my_db', parallel=True)
-
-    # # 指定表去重(使用特定列)
-    # deduplicator.deduplicate_table('my_db', 'my_table', columns=['name', 'date'])
-
-    # 关闭连接
-    deduplicator.close()
-
 if __name__ == '__main__':
     pass